From c1885cefd7649c9e16e66f66aa4a4f18417766a8 Mon Sep 17 00:00:00 2001 From: Mike Landau Date: Thu, 29 Oct 2026 14:09:08 -0700 Subject: [PATCH 1/2] perf(nixprofile): read manifest.json instead of running nix profile list Implements item 10 of specs/perf-improvements.md. ProfileListItems ran `nix profile list --json` (~75ms) on every state recompute. For a version 3 manifest (nix >= 2.20), that command reads /manifest.json and prints it back out in the same format, so ProfileListItems now reads the file instead. - Only version 3 is read directly. Versions 1 and 2 keep elements in an array and newer nix derives names for them when reading, so those go to nix, as do unknown versions, unparseable files, and elements nix would reject (missing "active"). - Store paths are sorted and deduplicated, and originalUrl/attrPath are dropped when url is empty, matching what nix prints. - A missing manifest (no profile yet, or a link to a garbage collected generation) returns no items, like nix. A manifest.nix (nix-env profile) goes to nix. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/nix/nixprofile/manifest_test.go | 279 +++++++++++++++++++++++ internal/nix/nixprofile/profile.go | 132 +++++++++-- 2 files changed, 389 insertions(+), 22 deletions(-) create mode 100644 internal/nix/nixprofile/manifest_test.go diff --git a/internal/nix/nixprofile/manifest_test.go b/internal/nix/nixprofile/manifest_test.go new file mode 100644 index 00000000000..a116505dadf --- /dev/null +++ b/internal/nix/nixprofile/manifest_test.go @@ -0,0 +1,279 @@ +// Copyright 2024 Jetify Inc. and contributors. All rights reserved. +// Use of this source code is governed by the license in the LICENSE file. + +package nixprofile + +import ( + "cmp" + "io" + "os" + "os/exec" + "path/filepath" + "reflect" + "slices" + "testing" + + "go.jetify.com/devbox/internal/nix" +) + +const ( + nixpkgsURL = "https://releases.nixos.org/nixpkgs/nixpkgs-26.11pre1104102.e62746b0b472/nixexprs.tar.zst?narHash=sha256-GYpArmwi/iMuX2ov31oCBhAXRMJdNX9kOl2LZHCuR6M%3D" + fdPath = "/nix/store/vx7li94vvv19qqs8dil7vx7fl4by5dqg-fd-10.5.0" + helloPath = "/nix/store/fv6rgzamfwj334kf1xv8ms3y6x8nvirn-hello-2.12.3" + jqBinPath = "/nix/store/vvp6grz676r5jvsry7sf7kxqb4lwdd62-jq-1.8.2-bin" + jqManPath = "/nix/store/makwmvlz85ngimxrznbisy864hvc3g0s-jq-1.8.2-man" +) + +// manifestV3 is a manifest written by nix 2.30 for a profile with a package +// added by store path (which is what devbox does), a package added by flake +// reference, and a package with two outputs. The jq store paths are out of +// order to check that they get sorted like nix does. +const manifestV3 = `{"elements":{` + + `"fd":{"active":true,"priority":5,"storePaths":["` + fdPath + `"]},` + + `"hello":{"active":true,"attrPath":"legacyPackages.aarch64-darwin.hello","originalUrl":"flake:nixpkgs","outputs":null,"priority":5,"storePaths":["` + helloPath + `"],"url":"` + nixpkgsURL + `"},` + + `"jq":{"active":true,"attrPath":"legacyPackages.aarch64-darwin.jq","originalUrl":"flake:nixpkgs","outputs":["bin","man"],"priority":5,"storePaths":["` + jqBinPath + `","` + jqManPath + `"],"url":"` + nixpkgsURL + `"}` + + `},"version":3}` + +// manifestV2 is manifestV3 in the version 2 format (elements in an array). +const manifestV2 = `{"version":2,"elements":[` + + `{"active":true,"priority":5,"storePaths":["` + fdPath + `"]},` + + `{"active":true,"attrPath":"legacyPackages.aarch64-darwin.hello","originalUrl":"flake:nixpkgs","outputs":null,"priority":5,"storePaths":["` + helloPath + `"],"url":"` + nixpkgsURL + `"}` + + `]}` + +// manifestV1 is manifestV2 in the version 1 format (uri instead of url). +const manifestV1 = `{"version":1,"elements":[` + + `{"active":true,"priority":5,"storePaths":["` + fdPath + `"]},` + + `{"active":true,"attrPath":"legacyPackages.aarch64-darwin.hello","originalUri":"flake:nixpkgs","outputs":null,"priority":5,"storePaths":["` + helloPath + `"],"uri":"` + nixpkgsURL + `"}` + + `]}` + +var manifestV3Items = []*NixProfileListItem{ + { + name: "fd", + nixStorePaths: []string{fdPath}, + }, + { + name: "hello", + unlockedReference: "flake:nixpkgs#legacyPackages.aarch64-darwin.hello", + lockedReference: nixpkgsURL + "#legacyPackages.aarch64-darwin.hello", + nixStorePaths: []string{helloPath}, + }, + { + name: "jq", + unlockedReference: "flake:nixpkgs#legacyPackages.aarch64-darwin.jq", + lockedReference: nixpkgsURL + "#legacyPackages.aarch64-darwin.jq", + nixStorePaths: []string{jqManPath, jqBinPath}, + }, +} + +func TestParseManifest(t *testing.T) { + tests := map[string]struct { + manifest string + want []*NixProfileListItem + }{ + "v3": { + manifest: manifestV3, + want: manifestV3Items, + }, + "v3 empty": { + manifest: `{"elements":{},"version":3}`, + want: []*NixProfileListItem{}, + }, + "v3 no url": { + // nix drops the source when url is empty, and treats missing + // storePaths as empty. + manifest: `{"elements":{"x":{"active":false,"attrPath":"a","originalUrl":"flake:nixpkgs","url":""}},"version":3}`, + want: []*NixProfileListItem{{name: "x", nixStorePaths: []string{}}}, + }, + "v3 duplicate store paths": { + manifest: `{"elements":{"fd":{"active":true,"storePaths":["` + fdPath + `","` + fdPath + `"]}},"version":3}`, + want: []*NixProfileListItem{{name: "fd", nixStorePaths: []string{fdPath}}}, + }, + } + for name, test := range tests { + t.Run(name, func(t *testing.T) { + got, ok := parseManifest([]byte(test.manifest)) + if !ok { + t.Fatal("got ok = false, want true") + } + assertItemsEqual(t, got, test.want) + }) + } +} + +func TestParseManifestUnsupported(t *testing.T) { + tests := map[string]string{ + "v1": manifestV1, + "v2": manifestV2, + "v4": `{"elements":{},"version":4}`, + "no version": `{"elements":{}}`, + "v3 elements array": `{"elements":[],"version":3}`, + "v3 missing active": `{"elements":{"fd":{"storePaths":["` + fdPath + `"]}},"version":3}`, + "v3 wrong type": `{"elements":{"fd":{"active":true,"priority":"5"}},"version":3}`, + "invalid json": `{"elements":`, + "not an object": `[]`, + "version not number": `{"elements":{},"version":"3"}`, + } + for name, manifest := range tests { + t.Run(name, func(t *testing.T) { + if _, ok := parseManifest([]byte(manifest)); ok { + t.Error("got ok = true, want false") + } + }) + } +} + +func TestProfileListItemsReadsManifest(t *testing.T) { + stubProfileListCommand(t, false) + + // Lay the profile out like nix does: profile -> profile-1-link -> generation. + dir := t.TempDir() + gen := filepath.Join(dir, "generation") + writeManifest(t, gen, "manifest.json", manifestV3) + if err := os.Symlink(gen, filepath.Join(dir, "profile-1-link")); err != nil { + t.Fatal(err) + } + profile := filepath.Join(dir, "profile") + if err := os.Symlink("profile-1-link", profile); err != nil { + t.Fatal(err) + } + + got, err := ProfileListItems(io.Discard, profile) + if err != nil { + t.Fatal(err) + } + assertItemsEqual(t, got, manifestV3Items) +} + +func TestProfileListItemsNoManifest(t *testing.T) { + stubProfileListCommand(t, false) + + dir := t.TempDir() + dangling := filepath.Join(dir, "dangling") + if err := os.Symlink(filepath.Join(dir, "garbage-collected"), dangling); err != nil { + t.Fatal(err) + } + profiles := map[string]string{ + "missing": filepath.Join(dir, "missing"), + "dangling link": dangling, + "empty dir": t.TempDir(), + } + for name, profile := range profiles { + t.Run(name, func(t *testing.T) { + got, err := ProfileListItems(io.Discard, profile) + if err != nil { + t.Fatal(err) + } + if got == nil || len(got) != 0 { + t.Errorf("got items %v, want empty", got) + } + }) + } +} + +func TestProfileListItemsFallsBackToCommand(t *testing.T) { + tests := map[string]struct { + file string + manifest string + }{ + "v1": {"manifest.json", manifestV1}, + "v2": {"manifest.json", manifestV2}, + "v4": {"manifest.json", `{"elements":{},"version":4}`}, + "bad json": {"manifest.json", `{`}, + "nix-env": {"manifest.nix", `[ ]`}, + "unreadable": {"manifest.json/oops", `{}`}, + } + for name, test := range tests { + t.Run(name, func(t *testing.T) { + stubProfileListCommand(t, true) + profile := t.TempDir() + writeManifest(t, profile, test.file, test.manifest) + + got, err := ProfileListItems(io.Discard, profile) + if err != nil { + t.Fatal(err) + } + if len(got) != 1 || got[0].name != "from-command" { + t.Errorf("got items %v, want the stubbed command's items", got) + } + }) + } +} + +// TestManifestMatchesNixProfileList checks that reading manifest.json gives +// the same items as `nix profile list --json` for the same manifest. nix only +// parses the store paths in the manifest, so they don't need to exist. +func TestManifestMatchesNixProfileList(t *testing.T) { + if _, err := exec.LookPath("nix"); err != nil { + t.Skip("nix not found") + } + if !nix.AtLeast(nix.Version2_20) { + t.Skip("version 3 manifests need nix >= 2.20") + } + + tests := map[string]string{ + "v3": manifestV3, + "v3 empty": `{"elements":{},"version":3}`, + "v3 no url": `{"elements":{"x":{"active":false,"attrPath":"a",` + + `"originalUrl":"flake:nixpkgs","url":""}},"version":3}`, + "v3 duplicate store paths": `{"elements":{"fd":{"active":true,` + + `"storePaths":["` + fdPath + `","` + fdPath + `"]}},"version":3}`, + "no manifest": "", + } + for name, manifest := range tests { + t.Run(name, func(t *testing.T) { + profile := t.TempDir() + if manifest != "" { + writeManifest(t, profile, "manifest.json", manifest) + } + + fromFile, ok := readManifestItems(profile) + if !ok { + t.Fatal("readManifestItems returned ok = false") + } + fromCommand, err := profileListItemsFromCommand(io.Discard, profile) + if err != nil { + t.Fatal(err) + } + assertItemsEqual(t, fromFile, fromCommand) + }) + } +} + +// stubProfileListCommand replaces the `nix profile list` fallback for the +// duration of the test. If allow is false, the test fails if it's called. +func stubProfileListCommand(t *testing.T, allow bool) { + t.Helper() + orig := profileListCommand + t.Cleanup(func() { profileListCommand = orig }) + profileListCommand = func(io.Writer, string) ([]*NixProfileListItem, error) { + if !allow { + t.Error("ProfileListItems ran nix profile list, want it to read manifest.json") + } + return []*NixProfileListItem{{name: "from-command"}}, nil + } +} + +func writeManifest(t *testing.T, dir, name, content string) { + t.Helper() + path := filepath.Join(dir, name) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } +} + +func assertItemsEqual(t *testing.T, got, want []*NixProfileListItem) { + t.Helper() + byName := func(a, b *NixProfileListItem) int { + return cmp.Compare(a.NameOrIndex(), b.NameOrIndex()) + } + got = slices.Clone(got) + want = slices.Clone(want) + slices.SortFunc(got, byName) + slices.SortFunc(want, byName) + if !reflect.DeepEqual(got, want) { + t.Errorf("got items:\n%v\nwant:\n%v", got, want) + } +} diff --git a/internal/nix/nixprofile/profile.go b/internal/nix/nixprofile/profile.go index b4cd2a7ae60..33654420182 100644 --- a/internal/nix/nixprofile/profile.go +++ b/internal/nix/nixprofile/profile.go @@ -7,6 +7,10 @@ import ( "bufio" "encoding/json" "io" + "io/fs" + "os" + "path/filepath" + "slices" "strconv" "strings" @@ -20,11 +24,113 @@ import ( ) // ProfileListItems returns a list of the installed packages. +// +// It reads the profile's manifest.json directly when it can, which avoids +// starting a nix process. Otherwise it runs `nix profile list`. func ProfileListItems( writer io.Writer, profileDir string, ) ([]*NixProfileListItem, error) { defer debug.FunctionTimer().End() + if items, ok := readManifestItems(profileDir); ok { + return items, nil + } + return profileListCommand(writer, profileDir) +} + +// profileListCommand is a variable so tests can check when ProfileListItems +// falls back to running nix. +var profileListCommand = profileListItemsFromCommand + +// profileListElement is an element of a profile's manifest.json, which is +// also what `nix profile list --json` prints. +type profileListElement struct { + // Active is a pointer so parseManifest can tell when it's missing. + Active *bool `json:"active"` + AttrPath string `json:"attrPath"` + OriginalURL string `json:"originalUrl"` + Priority int `json:"priority"` + StorePaths []string `json:"storePaths"` + URL string `json:"url"` +} + +func (e *profileListElement) item(name string, index int) *NixProfileListItem { + return &NixProfileListItem{ + index: index, + name: name, + unlockedReference: lo.Ternary(e.OriginalURL != "", e.OriginalURL+"#"+e.AttrPath, ""), + lockedReference: lo.Ternary(e.URL != "", e.URL+"#"+e.AttrPath, ""), + nixStorePaths: e.StorePaths, + } +} + +// readManifestItems reads the items in a profile from its manifest.json. It +// returns false if the caller should run `nix profile list` instead. +func readManifestItems(profileDir string) ([]*NixProfileListItem, bool) { + data, err := os.ReadFile(filepath.Join(profileDir, "manifest.json")) + if errors.Is(err, fs.ErrNotExist) { + // Like nix, treat a profile without a manifest as empty. This + // includes a profile that doesn't exist yet and a link to a + // generation that was garbage collected. A manifest.nix means the + // profile was made by nix-env, so let nix read that. + _, err := os.Stat(filepath.Join(profileDir, "manifest.nix")) + if errors.Is(err, fs.ErrNotExist) { + return []*NixProfileListItem{}, true + } + return nil, false + } + if err != nil { + return nil, false + } + return parseManifest(data) +} + +// parseManifest parses a version 3 manifest.json (nix >= 2.20) into the same +// items that parsing `nix profile list --json` would give. That command reads +// the manifest and prints it back out in the same format. +// +// It returns false for other versions and for anything nix would handle +// differently. Versions 1 and 2 store elements in an array, and nix >= 2.20 +// derives a name for each one when it reads them. Rather than copy that +// logic, those are left to nix. They're rewritten as version 3 the next time +// nix changes the profile. +func parseManifest(data []byte) ([]*NixProfileListItem, bool) { + var manifest struct { + Version int `json:"version"` + Elements map[string]profileListElement `json:"elements"` + } + if err := json.Unmarshal(data, &manifest); err != nil || manifest.Version != 3 { + return nil, false + } + items := make([]*NixProfileListItem, 0, len(manifest.Elements)) + for name, element := range manifest.Elements { + if element.Active == nil { + // nix fails to read an element without "active". + return nil, false + } + if element.URL == "" { + // nix ignores the flake source when there's no url, so the + // command doesn't print originalUrl or attrPath either. + element.OriginalURL = "" + element.AttrPath = "" + } + // nix keeps store paths in a sorted set and always prints the list. + if element.StorePaths == nil { + element.StorePaths = []string{} + } + slices.Sort(element.StorePaths) + element.StorePaths = slices.Compact(element.StorePaths) + items = append(items, element.item(name, 0)) + } + return items, true +} + +// profileListItemsFromCommand runs `nix profile list` to get the items in a +// profile. +func profileListItemsFromCommand( + writer io.Writer, + profileDir string, +) ([]*NixProfileListItem, error) { output, err := nix.ProfileList(writer, profileDir, true /*useJSON*/) if err != nil { // fallback to legacy profile list @@ -32,16 +138,8 @@ func ProfileListItems( return profileListLegacy(writer, profileDir) } - type ProfileListElement struct { - Active bool `json:"active"` - AttrPath string `json:"attrPath"` - OriginalURL string `json:"originalUrl"` - Priority int `json:"priority"` - StorePaths []string `json:"storePaths"` - URL string `json:"url"` - } type ProfileListOutput struct { - Elements map[string]ProfileListElement `json:"elements"` + Elements map[string]profileListElement `json:"elements"` Version int `json:"version"` } @@ -50,12 +148,7 @@ func ProfileListItems( if err := json.Unmarshal([]byte(output), &structOutput); err == nil { items := []*NixProfileListItem{} for name, element := range structOutput.Elements { - items = append(items, &NixProfileListItem{ - name: name, - unlockedReference: lo.Ternary(element.OriginalURL != "", element.OriginalURL+"#"+element.AttrPath, ""), - lockedReference: lo.Ternary(element.URL != "", element.URL+"#"+element.AttrPath, ""), - nixStorePaths: element.StorePaths, - }) + items = append(items, element.item(name, 0)) } return items, nil } @@ -64,7 +157,7 @@ func ProfileListItems( // ProfileListOutputJSONLegacy is for parsing `nix profile list --json` in nix < version 2.20 // that relied on index instead of name for each package installed. type ProfileListOutputJSONLegacy struct { - Elements []ProfileListElement `json:"elements"` + Elements []profileListElement `json:"elements"` Version int `json:"version"` } var structOutput2 ProfileListOutputJSONLegacy @@ -73,12 +166,7 @@ func ProfileListItems( } items := []*NixProfileListItem{} for index, element := range structOutput2.Elements { - items = append(items, &NixProfileListItem{ - index: index, - unlockedReference: lo.Ternary(element.OriginalURL != "", element.OriginalURL+"#"+element.AttrPath, ""), - lockedReference: lo.Ternary(element.URL != "", element.URL+"#"+element.AttrPath, ""), - nixStorePaths: element.StorePaths, - }) + items = append(items, element.item("", index)) } return items, nil From 199e93b2417ff93d0ebd64d27f7113546eb66e7f Mon Sep 17 00:00:00 2001 From: Mike Landau Date: Thu, 29 Oct 2026 14:09:59 -0700 Subject: [PATCH 2/2] docs(specs): mark perf item 10 done Co-Authored-By: Claude Opus 5.5 (1M context) --- specs/perf-improvements.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/perf-improvements.md b/specs/perf-improvements.md index b55a796d56c..726c10b3629 100644 --- a/specs/perf-improvements.md +++ b/specs/perf-improvements.md @@ -178,7 +178,7 @@ only lasts for the process. Fix: persist positive results in a `filecache` keyed by `uri/hash` with a long TTL (a narinfo for a given store hash is effectively immutable). Keep misses in memory only or with a short TTL. -### 10. Read the profile manifest instead of running `nix profile list` +### 10. Read the profile manifest instead of running `nix profile list` (done in #3030) **Win:** ~50–100ms per recompute. **Effort:** ~40–70 LOC.