test(pkgs): install and switch versions of every shipped package

.github/workflows/pkgdef-tests.yaml

@@ -0,0 +1,81 @@
+name: pkgdef-tests
+# Installs every package definition in pkgs/ from its vendor and switches
+# between two of its versions (see specs/package-definitions.md). Runs when
+# the definitions or their implementation change, and weekly to catch vendors
+# that change their downloads or metadata.
+
+concurrency:
+  group: pkgdef-tests-${{ github.ref }}
+  cancel-in-progress: true
+
+on:
+  pull_request:
+    paths:
+      - pkgs/**
+      - internal/pkgdef/**
+      - internal/devbox/pkgdef.go
+      - internal/lock/pkgdef.go
+      - internal/plugin/pkgdef.go
+      - testscripts/testrunner/pkgdefs.go
+      - .github/workflows/pkgdef-tests.yaml
+  push:
+    branches:
+      - main
+    paths:
+      - pkgs/**
+      - internal/pkgdef/**
+      - internal/devbox/pkgdef.go
+      - internal/lock/pkgdef.go
+      - internal/plugin/pkgdef.go
+      - testscripts/testrunner/pkgdefs.go
+      - .github/workflows/pkgdef-tests.yaml
+  workflow_dispatch:
+  schedule:
+    - cron: '0 9 * * 1' # Mondays at 9:00 UTC
+
+permissions:
+  contents: read
+
+defaults:
+  run:
+    shell: bash
+
+env:
+  # Version lookups and asset digests go through the GitHub API.
+  DEVBOX_GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  NIX_CONFIG: |
+    access-tokens = github.com=${{ secrets.GITHUB_TOKEN }}
+  DEVBOX_RUN_PKGDEF_TESTS: 1
+
+jobs:
+  pkgdef-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 60
+    steps:
+      - name: clear directories to reduce disk usage
+        if: runner.os == 'Linux'
+        # Every package installs two toolchains (several GB in total).
+        run: |
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+      - uses: actions/checkout@v7
+      - name: Mount golang cache
+        uses: actions/cache@v6
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg
+          key: go-devbox-tests-${{ runner.os }}-${{ hashFiles('go.sum') }}
+      - name: Install devbox
+        uses: jetify-com/devbox-install-action@v0.15.0
+        with:
+          enable-cache: true
+      - name: Resolve and lock the latest version of every package
+        run: devbox run go test -v -count=1 -run TestShippedDefinitionsResolve ./internal/pkgdef
+      - name: Install and switch versions of every package
+        run: devbox run go test -v -count=1 -timeout 50m -run TestPackageDefinitions ./testscripts

internal/pkgdef/pkgs_test.go

@@ -0,0 +1,97 @@
+// Copyright 2024 Jetify Inc. and contributors. All rights reserved.
+// Use of this source code is governed by the license in the LICENSE file.
+
+package pkgdef
+
+import (
+	"os"
+	"path/filepath"
+	"runtime"
+	"strconv"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// networkTestsEnv enables tests that resolve the shipped package
+// definitions against the real vendors.
+const networkTestsEnv = "DEVBOX_RUN_PKGDEF_TESTS"
+
+// shippedDefinitions returns the package definitions in pkgs/, keyed by
+// package directory name.
+func shippedDefinitions(t *testing.T) map[string]string {
+	t.Helper()
+	paths, err := filepath.Glob(filepath.Join("..", "..", "pkgs", "*", "*.pkg.json*"))
+	require.NoError(t, err)
+	require.NotEmpty(t, paths)
+	defs := map[string]string{}
+	for _, p := range paths {
+		defs[filepath.Base(filepath.Dir(p))] = p
+	}
+	return defs
+}
+
+func loadShipped(t *testing.T, path string) *Definition {
+	t.Helper()
+	loaded, err := loadLocal(Ref{Source: path}, "")
+	require.NoError(t, err)
+	return loaded.Definition
+}
+
+// TestShippedDefinitionsParse checks that every definition in pkgs/ is
+// valid, is named after its directory and file, and supports every
+// platform Devbox does.
+func TestShippedDefinitionsParse(t *testing.T) {
+	for dir, path := range shippedDefinitions(t) {
+		t.Run(dir, func(t *testing.T) {
+			def := loadShipped(t, path)
+			assert.Equal(t, dir, def.Name, "definition name matches its directory")
+			assert.Contains(t, []string{dir + ".pkg.json", dir + ".pkg.jsonc"}, filepath.Base(path))
+			assert.ElementsMatch(t, knownSystems, def.Platforms)
+			assert.NotEmpty(t, def.Binaries, "binaries lets install verify the result")
+			assert.NotEmpty(t, def.Homepage)
+			assert.NotEmpty(t, def.License)
+		})
+	}
+}
+
+// TestShippedDefinitionsResolve resolves the latest version of every
+// definition in pkgs/ against the real vendor and locks it for every
+// platform. It catches vendors that changed their URLs or metadata.
+func TestShippedDefinitionsResolve(t *testing.T) {
+	if on, _ := strconv.ParseBool(os.Getenv(networkTestsEnv)); !on {
+		t.Skipf("Skipping network test. To enable, set %s=1.", networkTestsEnv)
+	}
+	client := NewClient()
+	client.CacheDir = t.TempDir()
+	for dir, path := range shippedDefinitions(t) {
+		t.Run(dir, func(t *testing.T) {
+			def := loadShipped(t, path)
+			resolved, err := client.Resolve(t.Context(), def, Latest)
+			require.NoError(t, err)
+			for system, artifact := range resolved.Artifacts {
+				assert.NotEmpty(t, artifact.URL, system)
+				assert.True(t, artifact.HasDigest(), "%s: the latest version should have a vendor checksum", system)
+			}
+			// Vendors drop platforms (Temurin 27 has no Intel macOS build),
+			// so only the platform running the test is required.
+			assert.Contains(t, resolved.Artifacts, hostSystem(t))
+			var missing []string
+			for _, system := range knownSystems {
+				if _, ok := resolved.Artifacts[system]; !ok {
+					missing = append(missing, system)
+				}
+			}
+			t.Logf("%s@latest = %s (not published for: %v)", def.Name, resolved.Version, missing)
+		})
+	}
+}
+
+// hostSystem returns the nix system of the machine running the test.
+func hostSystem(t *testing.T) string {
+	t.Helper()
+	arch := map[string]string{"amd64": "x86_64", "arm64": "aarch64"}[runtime.GOARCH]
+	require.NotEmpty(t, arch, "unsupported GOARCH %s", runtime.GOARCH)
+	return arch + "-" + runtime.GOOS
+}

pkgs/bun/bun.test.txt

@@ -0,0 +1,33 @@
+# Bun from its GitHub releases.
+#
+# Installs bun@1.1 from devbox.json, switches to 1.2 with `devbox add`, and
+# switches back by editing devbox.json. Runs only when DEVBOX_RUN_PKGDEF_TESTS=1.
+
+cp $PKGS/bun/bun.pkg.jsonc bun.pkg.jsonc
+
+exec devbox install
+devboxlock.packages.contains devbox.lock pkg:./bun.pkg.jsonc@1.1
+exec devbox run -- bun --version
+stdout '^1\.1\.'
+exec devbox run -- bunx --version
+stdout '^1\.1\.'
+
+exec devbox add pkg:./bun.pkg.jsonc@1.2
+! devboxjson.packages.contains devbox.json pkg:./bun.pkg.jsonc@1.1
+! devboxlock.packages.contains devbox.lock pkg:./bun.pkg.jsonc@1.1
+exec devbox run -- bun --version
+stdout '^1\.2\.'
+
+# 1.1 is still installed, so switching back needs no download.
+cp devbox.1.1.json devbox.json
+exec devbox run -- bun --version
+stdout '^1\.1\.'
+
+-- devbox.json --
+{
+  "packages": ["pkg:./bun.pkg.jsonc@1.1"]
+}
+-- devbox.1.1.json --
+{
+  "packages": ["pkg:./bun.pkg.jsonc@1.1"]
+}

pkgs/deno/deno.test.txt

@@ -0,0 +1,33 @@
+# Deno from its GitHub releases. Deno 1.x published no checksums, so the
+# first install records one (trust on first use).
+#
+# Installs deno@1.46 from devbox.json, switches to 2 with `devbox add`, and
+# switches back by editing devbox.json. Runs only when DEVBOX_RUN_PKGDEF_TESTS=1.
+
+cp $PKGS/deno/deno.pkg.jsonc deno.pkg.jsonc
+
+exec devbox install
+devboxlock.packages.contains devbox.lock pkg:./deno.pkg.jsonc@1.46
+exec devbox run -- deno --version
+stdout '^deno 1\.46\.'
+grep '"sha256": "' devbox.lock
+
+exec devbox add pkg:./deno.pkg.jsonc@2
+! devboxjson.packages.contains devbox.json pkg:./deno.pkg.jsonc@1.46
+! devboxlock.packages.contains devbox.lock pkg:./deno.pkg.jsonc@1.46
+exec devbox run -- deno --version
+stdout '^deno 2\.'
+
+# 1.46 is still installed, so switching back needs no download.
+cp devbox.1.46.json devbox.json
+exec devbox run -- deno --version
+stdout '^deno 1\.46\.'
+
+-- devbox.json --
+{
+  "packages": ["pkg:./deno.pkg.jsonc@1.46"]
+}
+-- devbox.1.46.json --
+{
+  "packages": ["pkg:./deno.pkg.jsonc@1.46"]
+}

pkgs/go/go.test.txt

@@ -0,0 +1,36 @@
+# Go from go.dev.
+#
+# Installs go@1.22 from devbox.json, switches to 1.27 with `devbox add`, and
+# switches back by editing devbox.json. Runs only when DEVBOX_RUN_PKGDEF_TESTS=1.
+
+cp $PKGS/go/go.pkg.jsonc go.pkg.jsonc
+
+exec devbox install
+devboxlock.packages.contains devbox.lock pkg:./go.pkg.jsonc@1.22
+exec devbox run -- go version
+stdout 'go1\.22\.'
+# The version in devbox.json runs even if go.mod asks for a newer one.
+exec devbox run -- go env GOTOOLCHAIN
+stdout '^local$'
+
+exec devbox add pkg:./go.pkg.jsonc@1.27
+! devboxjson.packages.contains devbox.json pkg:./go.pkg.jsonc@1.22
+! devboxlock.packages.contains devbox.lock pkg:./go.pkg.jsonc@1.22
+exec devbox run -- go version
+stdout 'go1\.27\.'
+
+# 1.22 is still installed, so switching back needs no download.
+cp devbox.1.22.json devbox.json
+exec devbox run -- go version
+stdout 'go1\.22\.'
+exec devbox run -- go env GOTOOLCHAIN
+stdout '^local$'
+
+-- devbox.json --
+{
+  "packages": ["pkg:./go.pkg.jsonc@1.22"]
+}
+-- devbox.1.22.json --
+{
+  "packages": ["pkg:./go.pkg.jsonc@1.22"]
+}

pkgs/gradle/gradle.test.txt

@@ -0,0 +1,32 @@
+# Gradle from services.gradle.org, running on the Temurin JDK.
+#
+# Installs gradle@8 from devbox.json, switches to 9 with `devbox add`, and
+# switches back by editing devbox.json. Runs only when DEVBOX_RUN_PKGDEF_TESTS=1.
+
+cp $PKGS/java/java.pkg.jsonc java.pkg.jsonc
+cp $PKGS/gradle/gradle.pkg.jsonc gradle.pkg.jsonc
+
+exec devbox install
+devboxlock.packages.contains devbox.lock pkg:./gradle.pkg.jsonc@8
+exec devbox run -- gradle --version
+stdout '^Gradle 8\.'
+
+exec devbox add pkg:./gradle.pkg.jsonc@9
+! devboxjson.packages.contains devbox.json pkg:./gradle.pkg.jsonc@8
+! devboxlock.packages.contains devbox.lock pkg:./gradle.pkg.jsonc@8
+exec devbox run -- gradle --version
+stdout '^Gradle 9\.'
+
+# 8 is still installed, so switching back needs no download.
+cp devbox.8.json devbox.json
+exec devbox run -- gradle --version
+stdout '^Gradle 8\.'
+
+-- devbox.json --
+{
+  "packages": ["pkg:./java.pkg.jsonc@21", "pkg:./gradle.pkg.jsonc@8"]
+}
+-- devbox.8.json --
+{
+  "packages": ["pkg:./java.pkg.jsonc@21", "pkg:./gradle.pkg.jsonc@8"]
+}

pkgs/helm/helm.test.txt

@@ -0,0 +1,31 @@
+# Helm from get.helm.sh.
+#
+# Installs helm@3 from devbox.json, switches to 4 with `devbox add`, and
+# switches back by editing devbox.json. Runs only when DEVBOX_RUN_PKGDEF_TESTS=1.
+
+cp $PKGS/helm/helm.pkg.jsonc helm.pkg.jsonc
+
+exec devbox install
+devboxlock.packages.contains devbox.lock pkg:./helm.pkg.jsonc@3
+exec devbox run -- helm version --short
+stdout '^v3\.'
+
+exec devbox add pkg:./helm.pkg.jsonc@4
+! devboxjson.packages.contains devbox.json pkg:./helm.pkg.jsonc@3
+! devboxlock.packages.contains devbox.lock pkg:./helm.pkg.jsonc@3
+exec devbox run -- helm version --short
+stdout '^v4\.'
+
+# 3 is still installed, so switching back needs no download.
+cp devbox.3.json devbox.json
+exec devbox run -- helm version --short
+stdout '^v3\.'
+
+-- devbox.json --
+{
+  "packages": ["pkg:./helm.pkg.jsonc@3"]
+}
+-- devbox.3.json --
+{
+  "packages": ["pkg:./helm.pkg.jsonc@3"]
+}

pkgs/java/java.test.txt

@@ -0,0 +1,37 @@
+# Java: the Eclipse Temurin JDK from Adoptium.
+#
+# Installs java@17 from devbox.json, switches to 21 with `devbox add`, and
+# switches back by editing devbox.json. Runs only when DEVBOX_RUN_PKGDEF_TESTS=1.
+
+cp $PKGS/java/java.pkg.jsonc java.pkg.jsonc
+
+exec devbox install
+devboxlock.packages.contains devbox.lock pkg:./java.pkg.jsonc@17
+exec devbox run -- java -version
+stderr 'openjdk version "17\.'
+exec devbox run -- javac -version
+stdout '^javac 17\.'
+exec devbox run -- sh -c '"$JAVA_HOME/bin/java" -version'
+stderr 'openjdk version "17\.'
+
+exec devbox add pkg:./java.pkg.jsonc@21
+! devboxjson.packages.contains devbox.json pkg:./java.pkg.jsonc@17
+! devboxlock.packages.contains devbox.lock pkg:./java.pkg.jsonc@17
+exec devbox run -- java -version
+stderr 'openjdk version "21\.'
+
+# 17 is still installed, so switching back needs no download.
+cp devbox.17.json devbox.json
+exec devbox run -- java -version
+stderr 'openjdk version "17\.'
+exec devbox run -- sh -c 'echo $JAVA_HOME'
+stdout '/java/17\.'
+
+-- devbox.json --
+{
+  "packages": ["pkg:./java.pkg.jsonc@17"]
+}
+-- devbox.17.json --
+{
+  "packages": ["pkg:./java.pkg.jsonc@17"]
+}