From 5605e954d8f4c8720bc451c59f9a40395c7af530 Mon Sep 17 00:00:00 2001 From: Antonio Salinas <173939267+Tonisal-byte@users.noreply.github.com> Date: Sat, 31 Oct 2026 13:32:00 -0700 Subject: [PATCH 1/5] feat: add declarative Go vendor source preparation Preserve upstream packaging while converting supported Go application specs and generating vendor archives from prepared sources. Wire configuration, history, documentation, and regression coverage across preparation, builds, rendering, and source diffs. Validation: ./magefile.go check all passed, including Python lint and type checks. Build and unit phases passed. Full ./magefile.go all was attempted via Podman, but Docker-oriented scenario file-mount and network-isolation expectations fail in this environment; Docker WSL integration is unavailable. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/user/how-to/vendor-go-application.md | 92 +++++ .../cli/azldev_component_prepare-sources.md | 4 +- docs/user/reference/config/components.md | 46 +++ .../app/azldev/agentskill/agentskill_test.go | 11 + .../agentskill/content/comp-toml.md.tmpl | 17 + .../content/withoutlockfile/comp-toml.md.tmpl | 17 + internal/app/azldev/cmds/component/build.go | 2 +- .../app/azldev/cmds/component/diffsources.go | 2 +- .../cmds/component/history_customizations.go | 25 ++ .../cmds/component/history_internal_test.go | 6 + .../app/azldev/cmds/component/preparerctx.go | 26 ++ .../component/preparerctx_internal_test.go | 64 +++ .../azldev/cmds/component/preparesources.go | 14 +- internal/app/azldev/cmds/component/render.go | 6 +- .../azldev/cmds/component/sourcepreparer.go | 19 +- internal/app/azldev/core/sources/govendor.go | 373 ++++++++++++++++++ .../core/sources/govendor_internal_test.go | 244 ++++++++++++ .../app/azldev/core/sources/sourceprep.go | 38 +- internal/projectconfig/component.go | 23 ++ internal/projectconfig/configfile.go | 4 + internal/projectconfig/fingerprint_test.go | 1 + internal/projectconfig/govendor.go | 66 ++++ internal/projectconfig/govendor_test.go | 80 ++++ internal/rpm/spec/govendor.go | 322 +++++++++++++++ internal/rpm/spec/govendor_test.go | 197 +++++++++ ...shots_config_generate-schema_stdout_1.snap | 34 ++ schemas/azldev.schema.json | 34 ++ 27 files changed, 1750 insertions(+), 17 deletions(-) create mode 100644 docs/user/how-to/vendor-go-application.md create mode 100644 internal/app/azldev/cmds/component/preparerctx_internal_test.go create mode 100644 internal/app/azldev/core/sources/govendor.go create mode 100644 internal/app/azldev/core/sources/govendor_internal_test.go create mode 100644 internal/projectconfig/govendor.go create mode 100644 internal/projectconfig/govendor_test.go create mode 100644 internal/rpm/spec/govendor.go create mode 100644 internal/rpm/spec/govendor_test.go diff --git a/docs/user/how-to/vendor-go-application.md b/docs/user/how-to/vendor-go-application.md new file mode 100644 index 000000000..c0b430642 --- /dev/null +++ b/docs/user/how-to/vendor-go-application.md @@ -0,0 +1,92 @@ +# Vendor a Go application + +Use `go-vendor` to bundle a Go application's module dependencies instead of +building against separately packaged `golang(...)` libraries. This is a targeted +conversion of the selected upstream spec, not a `go2rpm` wrapper or a newly +generated replacement spec. + +## Enable and prepare + +Add this to the component definition: + +```toml +[components.example.go-vendor] +enabled = true +``` + +Then prepare the complete build inputs: + +```sh +azldev component prepare-sources -p example -o prepared-example --without-git +``` + +The project/build distro must have a working mock configuration and provide +`go-vendor-tools`, `askalono-cli`, `golang`, `go-rpm-macros`, `rpm-build`, and +`rpmdevtools`, `rpmautospec`, `git`, and `python3-click`. These are installed in a +fresh, network-enabled generation chroot; +neither `go2rpm` nor the Go vendor tools are required on the host. Configure +`prep-packages` for additional tools required by upstream prep scripts. + +The pipeline fetches the selected upstream sources, applies overlays, and copies +them into isolated staging. It processes automatic release/changelog metadata +there and runs the existing `%prep` section with `rpmbuild -bp --nodeps`. +This retains upstream patches and edits to module files. It then calls +`go_vendor_archive create` on the prepared module tree, extracts the result for +license detection, and runs `go_vendor_license` without interactive prompts. +Unrecognized licenses fail preparation; add reviewed, hash-qualified exceptions +to `tool-config` rather than suppressing detection. + +The output includes the converted spec, all its original sources and patches, +`-vendor.tar.bz2`, and `azldev-go-vendor.toml`. The archive is recorded +with SHA-512 in `sources` and excluded from synthetic git commits with `.gitignore`. +The upstream release/changelog is retained; ordinary azldev release handling +still applies afterwards. + +## What is converted + +The converter removes standalone `BuildRequires: golang(import/path)` tags, +including version constraints, and bare `%go_generate_buildrequires` invocations. +It changes a bare `%gocheck` to `%gotest ./...`, adds vendor extraction at the end +of `%prep`, and enables module/vendor mode for subsequent build scripts. +It adds the Go compiler/macros and vendor tooling build requirements, dynamic +license backend requirements, license installation/file-list handling, and a +license check in `%build` so disabling `%check` does not bypass license validation. +The main `License:` tag is updated to the detected cumulative SPDX expression. + +Unrelated runtime dependencies, C-library build dependencies, custom commands, +patches, metadata, and ordinary subpackages are retained. Go vendoring does not +eliminate dependencies on shared libraries used by cgo. + +The initial supported scope is application specs with one unconditional +main-package `%prep`, `%build`, `%install`, and `%files` section. The prep section +must finish in the module root, or use `module-dir` for a relative subdirectory. +Custom build/install commands must already address that same module. Conditional +commands within those sections are retained. + +Automatic Go development subpackages (`%gopkg`, `%goinstall`, Go file-list macros), +flagged `%gocheck` commands, compound/rich Go BuildRequires, existing vendor +macros, explicit conflicting module-mode settings, and ambiguous/conditional +target sections are rejected. Use narrowly scoped overlays to adapt these cases; +azldev does not silently drop subpackages or rewrite arbitrary shell scripts. + +## Review and upgrade + +Run `azldev component diff-sources -p example` to inspect the conversion, or +`azldev component render -p example` to refresh checked-in rendered output. +Rendering and source diffs also generate the archive: they need network access and +can be substantially slower for enabled components. + +Review the cumulative license expression and the retained build/install/check +commands before building. Automatic detection is not approval of a dependency's +license. The converter sets `GO111MODULE=on`, `-mod=vendor`, `GOPROXY=off`, +`GOSUMDB=off`, and `GOTOOLCHAIN=local` for build scripts; custom commands must not +override these settings to download dependencies. + +The same TOML applies after an upstream version update: no archive version, +import path, Source number, or spec line number is embedded in the declaration. +Generation reruns against the newly selected and patched inputs. Review changed +vendor hashes and licenses; an upstream packaging change may require updating +an overlay or resolving a newly unsupported construct. + +See [Go vendoring configuration](../reference/config/components.md#go-vendoring) +for module layout, dependency overrides, and license exceptions. diff --git a/docs/user/reference/cli/azldev_component_prepare-sources.md b/docs/user/reference/cli/azldev_component_prepare-sources.md index 45f9f8cb7..b133f34c5 100644 --- a/docs/user/reference/cli/azldev_component_prepare-sources.md +++ b/docs/user/reference/cli/azldev_component_prepare-sources.md @@ -41,8 +41,8 @@ azldev component prepare-sources [flags] -o, --output-dir string output directory --rpmdev-bumpspec Use rpmdev-bumpspec instead of the legacy static release calculation --skip-lock-validation skip lock file consistency checks - --skip-overlays skip applying overlays to prepared sources - --skip-sources skip downloading fetched sources when preparing the package (useful to extract dist-git metadata when source files are not needed) + --skip-overlays skip overlays and Go vendor conversion, preserving upstream packaging + --skip-sources skip downloading sources unless Go vendor conversion requires them -s, --spec-path stringArray Spec path --without-git Disable dist-git repository creation (enabled by default) ``` diff --git a/docs/user/reference/config/components.md b/docs/user/reference/config/components.md index ce06e6f60..33b7cbf8d 100644 --- a/docs/user/reference/config/components.md +++ b/docs/user/reference/config/components.md @@ -15,6 +15,7 @@ A component definition tells azldev where to find the spec file, how to customiz | Build config | `build` | [BuildConfig](#build-configuration) | No | Build-time options (macros, conditionals, check config) | | Render config | `render` | [RenderConfig](#render-configuration) | No | Options controlling spec rendering behavior | | Source files | `source-files` | array of [SourceFileReference](#source-file-references) | No | Additional source files to download for this component | +| Go vendoring | `go-vendor` | [GoVendorConfig](#go-vendoring) | No | Convert a supported Go application spec in place and generate bundled dependencies | | Package overrides | `packages` | map of string → [PackageConfig](package-groups.md#package-config) | No | Exact per-package configuration overrides; highest priority in the resolution order | | Tests | `tests` | [ComponentTests](#component-tests) | No | Test references that apply to this component (see [Tests and Test Groups](tests.md)) | @@ -165,6 +166,51 @@ skip-file-filter = true > **Note:** This should only be used for specs with macros that `spectool` cannot resolve. For most components, the default filtering behavior is correct and keeps the rendered output clean. +## Go Vendoring + +Go vendoring is opt-in and preserves the upstream spec rather than replacing it +with a generated spec. See [Vendor a Go application](../../how-to/vendor-go-application.md) +for the workflow and supported spec patterns. + +```toml +[components.example.go-vendor] +enabled = true +``` + +| TOML Key | Type | Default | Description | +|----------|------|---------|-------------| +| `enabled` | boolean | `false` | Enable conversion and vendor archive generation. An explicit `false` overrides inherited enablement. | +| `module-dir` | string | `""` | Relative directory containing `go.mod`, resolved from the working directory at the end of the upstream `%prep` section. Empty uses that directory itself. No absolute paths, traversal, or RPM macros. | +| `prep-packages` | array of string | `[]` | Additional RPM packages needed by upstream `%prep` commands or the selected license detector. Installed only in the isolated generation chroot. | +| `tool-config` | string | `""` | Inline [go-vendor-tools TOML](https://fedora.gitlab.io/sigs/go/go-vendor-tools/config/), for dependency overrides, archive options, and reviewed license exceptions. Use `module-dir`, not `general.go_mod_dir`. | + +The tool configuration is inline so its contents participate in component +fingerprinting without checkout-dependent paths. Configured command hooks execute +in the generation chroot; treat this configuration as trusted build code. + +```toml +[components.example.go-vendor] +enabled = true +prep-packages = ["git"] +tool-config = ''' +[archive.dependency_overrides] +"example.com/dependency" = "v1.2.3" +''' +``` + +Preparation emits `-vendor.tar.bz2` and `azldev-go-vendor.toml`, +allocating unused `Source` numbers automatically. These names are reserved for +generated output; do not also declare them in `source-files` or supply them through +overlays. The archive's SHA-512 digest is recorded in the prepared `sources` file. +Generation runs afresh on every preparation; there is no filename-based cache. +The recorded digest identifies that output, not a separately pinned vendor lock. + +All archive/spec/loose-file overlays run before conversion. Source preparation, +builds, rendering (including `--check-only`), and source diffs use the same +conversion. Enabled components require downloads even with `--skip-sources`, +because preparation and license detection need the effective upstream sources. +`--skip-overlays` also skips conversion, leaving the upstream packaging unchanged. + ## Build Configuration The `[components..build]` section controls build-time options for a component. diff --git a/internal/app/azldev/agentskill/agentskill_test.go b/internal/app/azldev/agentskill/agentskill_test.go index a263a100f..7ebe18bdf 100644 --- a/internal/app/azldev/agentskill/agentskill_test.go +++ b/internal/app/azldev/agentskill/agentskill_test.go @@ -160,6 +160,17 @@ func TestComponentTomlSkillDocumentsRPMDevBumpspecPreview(t *testing.T) { assert.Contains(t, doc, "no bumpspec runtime discovery occurs") } +func TestComponentTomlSkillDocumentsGoVendoring(t *testing.T) { + for _, withoutLockfile := range []bool{false, true} { + doc, err := agentskill.NewCatalog(withoutLockfile).SkillDocument("azldev-comp-toml", testParams()) + require.NoError(t, err) + assert.Contains(t, doc, "go-vendor.enabled") + assert.Contains(t, doc, "does\nnot run `go2rpm`") + assert.Contains(t, doc, "Rendering and source diffs also generate") + assert.Contains(t, doc, "reviewed license exceptions") + } +} + func TestImageSkillDocumentsSKUGroupGuidance(t *testing.T) { doc, err := agentskill.SkillDocument("azldev-image", testParams()) require.NoError(t, err) diff --git a/internal/app/azldev/agentskill/content/comp-toml.md.tmpl b/internal/app/azldev/agentskill/content/comp-toml.md.tmpl index 0def4bf59..1462c24c3 100644 --- a/internal/app/azldev/agentskill/content/comp-toml.md.tmpl +++ b/internal/app/azldev/agentskill/content/comp-toml.md.tmpl @@ -32,6 +32,23 @@ Add sub-tables only for what you change. The fields you will reach for most: | `build.defines` / `build.with` / `build.without` | RPM macro and bcond build tweaks | | `release.calculation` | how the `Release` tag is managed | | `render.skip-file-filter` | rendering edge-case escape hatch | +| `go-vendor.enabled` | opt-in Go application spec conversion and vendor archive generation | + +## Go application vendoring + +Set `[components..go-vendor]` with `enabled = true` to preserve the upstream +spec while converting supported application packaging to bundled modules. +Preparation runs the patched upstream `%prep` before `go_vendor_archive`; it does +not run `go2rpm`. Rendering and source diffs also generate the vendor archive and +require downloads. Review the generated cumulative `License:` expression. + +Use `module-dir` for a module below the final prep working directory, +`prep-packages` for additional chroot tools, and inline `tool-config` for +go-vendor-tools dependency overrides or reviewed license exceptions. +Generated `-vendor.tar.bz2` and `azldev-go-vendor.toml` must not also be +declared as `source-files`. Unsupported constructs (notably automatic Go devel +subpackages, flagged `%gocheck`, or ambiguous sections) require explicit overlays; +never replace the upstream spec wholesale to bypass a conversion error. ## Spec source diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl index 0e33a9e6d..c1f972f5d 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl @@ -32,6 +32,23 @@ Add sub-tables only for what you change. The fields you will reach for most: | `build.defines` / `build.with` / `build.without` | RPM macro and bcond build tweaks | | `release.calculation` | how the `Release` tag is managed | | `render.skip-file-filter` | rendering edge-case escape hatch | +| `go-vendor.enabled` | opt-in Go application spec conversion and vendor archive generation | + +## Go application vendoring + +Set `[components..go-vendor]` with `enabled = true` to preserve the upstream +spec while converting supported application packaging to bundled modules. +Preparation runs the patched upstream `%prep` before `go_vendor_archive`; it does +not run `go2rpm`. Rendering and source diffs also generate the vendor archive and +require downloads. Review the generated cumulative `License:` expression. + +Use `module-dir` for a module below the final prep working directory, +`prep-packages` for additional chroot tools, and inline `tool-config` for +go-vendor-tools dependency overrides or reviewed license exceptions. +Generated `-vendor.tar.bz2` and `azldev-go-vendor.toml` must not also be +declared as `source-files`. Unsupported constructs (notably automatic Go devel +subpackages, flagged `%gocheck`, or ambiguous sections) require explicit overlays; +never replace the upstream spec wholesale to bypass a conversion error. ## Spec source diff --git a/internal/app/azldev/cmds/component/build.go b/internal/app/azldev/cmds/component/build.go index 2c9f99171..f2d955277 100644 --- a/internal/app/azldev/cmds/component/build.go +++ b/internal/app/azldev/cmds/component/build.go @@ -288,7 +288,7 @@ func buildComponent( preparerOpts = append(preparerOpts, sources.WithMockProcessor(mockProcessor)) - sourcePreparer, err := newSourcePreparer(sourceManager, env.FS(), env, env, append( + sourcePreparer, err := newComponentSourcePreparer(env, component, sourceManager, options.MockConfigOpts, append( preparerOpts, sources.WithSpecEditor(options.specEditorMode()), )...) diff --git a/internal/app/azldev/cmds/component/diffsources.go b/internal/app/azldev/cmds/component/diffsources.go index 7fd728050..24a2dd370 100644 --- a/internal/app/azldev/cmds/component/diffsources.go +++ b/internal/app/azldev/cmds/component/diffsources.go @@ -101,7 +101,7 @@ func DiffComponentSources(env *azldev.Env, options *DiffSourcesOptions) (interfa return nil, fmt.Errorf("failed to create source manager:\n%w", err) } - preparer, err := newSourcePreparer(sourceManager, env.FS(), env, env, + preparer, err := newComponentSourcePreparer(env, component, sourceManager, nil, sources.WithUpstreamProvenance(sources.FedoraDistTag(distro.Ref.Name, distro.Version.ReleaseVer)), sources.WithSpecEditor(options.specEditorMode())) if err != nil { diff --git a/internal/app/azldev/cmds/component/history_customizations.go b/internal/app/azldev/cmds/component/history_customizations.go index 2e90e7eb3..d1950fc34 100644 --- a/internal/app/azldev/cmds/component/history_customizations.go +++ b/internal/app/azldev/cmds/component/history_customizations.go @@ -29,6 +29,31 @@ func collectCustomizations(name string, config *projectconfig.ComponentConfig) [ items = appendRenderItems(items, config.Render) items = appendPackageItems(items, config.Packages) items = appendSourceFileItems(items, config.SourceFiles) + items = appendGoVendorItems(items, config.GoVendor) + + return items +} + +func appendGoVendorItems(items []CustomizationItem, config *projectconfig.GoVendorConfig) []CustomizationItem { + if config == nil { + return items + } + + if config.Enabled != nil { + items = append(items, CustomizationItem{Kind: "go-vendor.enabled", Value: strconv.FormatBool(*config.Enabled)}) + } + + if config.ModuleDir != "" { + items = append(items, CustomizationItem{Kind: "go-vendor.module-dir", Value: config.ModuleDir}) + } + + if config.ToolConfig != "" { + items = append(items, CustomizationItem{Kind: "go-vendor.tool-config", Value: config.ToolConfig}) + } + + for _, name := range config.PrepPackages { + items = append(items, CustomizationItem{Kind: "go-vendor.prep-packages", Value: name}) + } return items } diff --git a/internal/app/azldev/cmds/component/history_internal_test.go b/internal/app/azldev/cmds/component/history_internal_test.go index 242094194..eb9ef98fd 100644 --- a/internal/app/azldev/cmds/component/history_internal_test.go +++ b/internal/app/azldev/cmds/component/history_internal_test.go @@ -88,6 +88,7 @@ func TestCustomizationCollectorsCoverEveryFingerprintableField(t *testing.T) { reflect.TypeFor[projectconfig.ComponentRenderConfig](), reflect.TypeFor[projectconfig.SourceFileReference](), reflect.TypeFor[projectconfig.Origin](), + reflect.TypeFor[projectconfig.GoVendorConfig](), } // Maps "StructName.FieldName" -> short note describing how the field @@ -103,6 +104,11 @@ func TestCustomizationCollectorsCoverEveryFingerprintableField(t *testing.T) { "ComponentConfig.Render": "appendRenderItems (per-field via ComponentRenderConfig walk)", "ComponentConfig.SourceFiles": "appendSourceFileItems (opaque unit per source file)", "ComponentConfig.Packages": "appendPackageItems (opaque unit per package override)", + "ComponentConfig.GoVendor": "appendGoVendorItems", + "GoVendorConfig.Enabled": "go-vendor.enabled", + "GoVendorConfig.ModuleDir": "go-vendor.module-dir", + "GoVendorConfig.ToolConfig": "go-vendor.tool-config", + "GoVendorConfig.PrepPackages": "go-vendor.prep-packages", // ComponentBuildConfig. "ComponentBuildConfig.With": "build.with", diff --git a/internal/app/azldev/cmds/component/preparerctx.go b/internal/app/azldev/cmds/component/preparerctx.go index f98f7b671..777ff4c73 100644 --- a/internal/app/azldev/cmds/component/preparerctx.go +++ b/internal/app/azldev/cmds/component/preparerctx.go @@ -4,11 +4,37 @@ package component import ( + "errors" + "fmt" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/sources" "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders" + "github.com/microsoft/azure-linux-dev-tools/internal/rpm/mock" ) +func goVendorPreparerOptions( + env *azldev.Env, component components.Component, mockConfigOpts map[string]string, +) ([]sources.PreparerOption, error) { + if !component.GetConfig().GoVendor.IsEnabled() || env.DryRun() { + return nil, nil + } + + _, distro, err := env.Distro() + if err != nil { + return nil, fmt.Errorf("resolving Go vendor build distro:\n%w", err) + } + + if distro.MockConfigPath == "" { + return nil, errors.New("go vendoring requires a project distro with a mock configuration") + } + + runner := mock.NewRunner(env, distro.MockConfigPath).WithConfigOpts(mockConfigOpts) + + return []sources.PreparerOption{sources.WithGoVendorRunner(runner)}, nil +} + // gitRepoPreparerOptions returns the [sources.PreparerOption] values that enable // synthetic dist-git history for the current mode. // diff --git a/internal/app/azldev/cmds/component/preparerctx_internal_test.go b/internal/app/azldev/cmds/component/preparerctx_internal_test.go new file mode 100644 index 000000000..7118b0017 --- /dev/null +++ b/internal/app/azldev/cmds/component/preparerctx_internal_test.go @@ -0,0 +1,64 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component + +import ( + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components/components_testutils" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/testutils" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/samber/lo" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +func TestGoVendorPreparerOptions(t *testing.T) { + const disabled = "disabled" + for _, mode := range []string{"enabled", disabled, "missing-mock", "missing-distro"} { + t.Run(mode, func(t *testing.T) { + env := testutils.NewTestEnv(t) + component := components_testutils.NewMockComponent(gomock.NewController(t)) + component.EXPECT().GetConfig().AnyTimes().Return(&projectconfig.ComponentConfig{ + GoVendor: &projectconfig.GoVendorConfig{Enabled: lo.ToPtr(mode != disabled)}, + }) + + switch mode { + case "missing-mock", disabled: + clearProjectMockConfig(env.Env.Config()) + case "missing-distro": + env.Env.Config().Project.DefaultDistro.Name = "absent" + } + + options, err := goVendorPreparerOptions(env.Env, component, nil) + + switch mode { + case "enabled": + require.NoError(t, err) + require.Len(t, options, 1) + case disabled: + require.NoError(t, err) + assert.Empty(t, options) + default: + require.Error(t, err) + assert.Empty(t, options) + } + }) + } +} + +func TestGoVendorHistoryCustomizations(t *testing.T) { + config := &projectconfig.GoVendorConfig{ + Enabled: lo.ToPtr(true), ModuleDir: "cmd/app", + ToolConfig: "[archive]\ntidy = false", PrepPackages: []string{"git"}, + } + assert.Equal(t, []CustomizationItem{ + {Kind: "go-vendor.enabled", Value: "true"}, + {Kind: "go-vendor.module-dir", Value: "cmd/app"}, + {Kind: "go-vendor.tool-config", Value: config.ToolConfig}, + {Kind: "go-vendor.prep-packages", Value: "git"}, + }, collectCustomizations("app", &projectconfig.ComponentConfig{GoVendor: config})) + assert.Empty(t, appendGoVendorItems(nil, &projectconfig.GoVendorConfig{})) +} diff --git a/internal/app/azldev/cmds/component/preparesources.go b/internal/app/azldev/cmds/component/preparesources.go index 317b7fb38..37b9668c8 100644 --- a/internal/app/azldev/cmds/component/preparesources.go +++ b/internal/app/azldev/cmds/component/preparesources.go @@ -73,7 +73,8 @@ Only one component may be selected at a time.`, _ = cmd.MarkFlagRequired("output-dir") _ = cmd.MarkFlagDirname("output-dir") - cmd.Flags().BoolVar(&options.SkipOverlays, "skip-overlays", false, "skip applying overlays to prepared sources") + cmd.Flags().BoolVar(&options.SkipOverlays, "skip-overlays", false, + "skip overlays and Go vendor conversion, preserving upstream packaging") cmd.Flags().BoolVar(&options.WithoutGitRepo, "without-git", false, "Disable dist-git repository creation (enabled by default)") addRPMDevBumpspecFlag(cmd, &options.RPMDevBumpspec) @@ -82,8 +83,7 @@ Only one component may be selected at a time.`, cmd.Flags().BoolVar(&options.AllowNoHashes, "allow-no-hashes", false, "compute missing hashes by downloading source files from their origin") cmd.Flags().BoolVar(&options.SkipSources, "skip-sources", false, - "skip downloading fetched sources when preparing the package (useful to extract "+ - "dist-git metadata when source files are not needed)") + "skip downloading sources unless Go vendor conversion requires them") return cmd } @@ -142,6 +142,14 @@ func PrepareComponentSources(env *azldev.Env, options *PrepareSourcesOptions) er } preparerOpts := buildPreparerOptions(env, distro, options) + if !options.SkipOverlays { + vendorOpts, err := goVendorPreparerOptions(env, component, nil) + if err != nil { + return err + } + + preparerOpts = append(preparerOpts, vendorOpts...) + } preparer, err := newSourcePreparer(sourceManager, env.FS(), env, env, append( preparerOpts, diff --git a/internal/app/azldev/cmds/component/render.go b/internal/app/azldev/cmds/component/render.go index ff99942bd..6f4277743 100644 --- a/internal/app/azldev/cmds/component/render.go +++ b/internal/app/azldev/cmds/component/render.go @@ -531,8 +531,8 @@ func prepareComponentSources( // Prepare sources with overlays, skipping lookaside downloads. // WithGitRepo preserves upstream .git and creates synthetic history so // rpmautospec can expand %autorelease and %autochangelog correctly. - // WithSkipLookaside avoids expensive tarball downloads — only spec + - // sidecar files are needed for rendering. + // WithSkipLookaside avoids tarball downloads unless Go vendoring requires + // the prepared sources to generate the archive and cumulative license. preparerOpts := append(gitRepoPreparerOptions(env, distro), sources.WithSkipLookaside(), sources.WithUpstreamProvenance(sources.FedoraDistTag(distro.Ref.Name, distro.Version.ReleaseVer)), @@ -542,7 +542,7 @@ func prepareComponentSources( preparerOpts = append(preparerOpts, sources.WithRPMDevBumpspec(env, env.WorkDir(), "")) } - preparer, err := newSourcePreparer(sourceManager, env.FS(), env, env, append( + preparer, err := newComponentSourcePreparer(env, comp, sourceManager, nil, append( preparerOpts, sources.WithSpecEditor(specEditor), )...) diff --git a/internal/app/azldev/cmds/component/sourcepreparer.go b/internal/app/azldev/cmds/component/sourcepreparer.go index a16e0a00c..a13890267 100644 --- a/internal/app/azldev/cmds/component/sourcepreparer.go +++ b/internal/app/azldev/cmds/component/sourcepreparer.go @@ -3,7 +3,24 @@ package component -import "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/sources" +import ( + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/sources" + "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders" +) //nolint:gochecknoglobals // Replaced by isolated command tests to observe constructed source preparers. var newSourcePreparer = sources.NewPreparer + +func newComponentSourcePreparer( + env *azldev.Env, component components.Component, manager sourceproviders.SourceManager, + mockConfigOpts map[string]string, options ...sources.PreparerOption, +) (sources.SourcePreparer, error) { + vendorOpts, err := goVendorPreparerOptions(env, component, mockConfigOpts) + if err != nil { + return nil, err + } + + return newSourcePreparer(manager, env.FS(), env, env, append(options, vendorOpts...)...) +} diff --git a/internal/app/azldev/core/sources/govendor.go b/internal/app/azldev/core/sources/govendor.go new file mode 100644 index 000000000..c6314e3a8 --- /dev/null +++ b/internal/app/azldev/core/sources/govendor.go @@ -0,0 +1,373 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package sources + +import ( + "bytes" + "context" + "errors" + "fmt" + "log/slog" + "path/filepath" + "strings" + "time" + + "github.com/kballard/go-shellquote" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders/fedorasource" + "github.com/microsoft/azure-linux-dev-tools/internal/rpm/mock" + "github.com/microsoft/azure-linux-dev-tools/internal/rpm/spec" + "github.com/microsoft/azure-linux-dev-tools/internal/rpm/spectool" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" +) + +const ( + goVendorMount = "/azldev-go-vendor" + goVendorTopDir = "/tmp/azldev-go-rpm" + goVendorBuild = goVendorTopDir + "/BUILD" + goVendorConfig = "azldev-go-vendor.toml" + goVendorScratch = "/azldev-go-scratch" + goVendorCleanupTimeout = 5 * time.Minute +) + +// WithGoVendorRunner provides a template runner. Every conversion uses a fresh, +// isolated root; only the component staging directory is exposed to it. +func WithGoVendorRunner(runner *mock.Runner) PreparerOption { + return func(p *sourcePreparerImpl) { p.goVendorRunner = runner } +} + +func (p *sourcePreparerImpl) prepareGoVendor( + ctx context.Context, component components.Component, outputDir string, +) (err error) { + config := component.GetConfig().GoVendor + if !config.IsEnabled() { + return nil + } + + if err := config.Validate(); err != nil { + return fmt.Errorf("invalid Go vendor configuration:\n%w", err) + } + + if p.dryRunnable.DryRun() { + slog.Info("Dry run; skipping Go vendor conversion and archive generation", "component", component.GetName()) + + return nil + } + + if p.goVendorRunner == nil { + return errors.New("go vendoring requires a mock runner with the project's build distro configuration") + } + + archiveName := component.GetName() + "-vendor.tar.bz2" + + specPath, err := p.resolveSpecPath(component, outputDir) + if err != nil { + return err + } + + data, err := fileutils.ReadFile(p.fs, specPath) + if err != nil { + return fmt.Errorf("reading Go spec:\n%w", err) + } + + converted, err := spec.ConvertGoVendor(data, spec.GoVendorOptions{ + ArchiveName: archiveName, ConfigName: goVendorConfig, ModuleDir: config.ModuleDir, + }) + if err != nil { + return fmt.Errorf("converting Go spec for %#q:\n%w", component.GetName(), err) + } + + if err := p.checkGoVendorSourceCollision(outputDir, archiveName); err != nil { + return err + } + + scratch, err := fileutils.MkdirTempInTempDir(p.fs, "azldev-go-vendor-*") + if err != nil { + return fmt.Errorf("creating Go vendor scratch directory:\n%w", err) + } + defer fileutils.RemoveAllAndUpdateErrorIfNil(p.fs, scratch, &err) + + staging := filepath.Join(scratch, "work") + if err := fileutils.MkdirAll(p.fs, staging); err != nil { + return fmt.Errorf("creating Go vendor staging directory:\n%w", err) + } + + sourceCopy := filepath.Join(scratch, "sources") + if err := p.stageGoVendorInputs(outputDir, sourceCopy, staging, filepath.Base(specPath), + converted.PrepSpec, config.ToolConfig); err != nil { + return err + } + + runner := p.goVendorRunner.Clone().WithBaseDir(filepath.Join(scratch, "mock")). + EnableNetwork().WithUnprivileged() + runner.AddBindMount(sourceCopy, goVendorMount) + runner.AddBindMount(staging, goVendorScratch) + + if err := p.runGoVendor(ctx, runner, archiveName, filepath.Base(specPath), config); err != nil { + return err + } + + if err := p.validateGoVendorInputs(staging, outputDir); err != nil { + return err + } + + return p.publishGoVendor(staging, outputDir, specPath, archiveName, converted.Spec, config.ToolConfig) +} + +func (p *sourcePreparerImpl) stageGoVendorInputs( + outputDir, sourceCopy, staging, specFilename string, prepSpec []byte, toolConfig string, +) error { + if err := fileutils.CopyDirRecursive(p.dryRunnable, p.fs, outputDir, sourceCopy, + fileutils.CopyDirOptions{CopyFileOptions: fileutils.CopyFileOptions{PreserveFileMode: true}}); err != nil { + return fmt.Errorf("copying Go sources for isolated prep:\n%w", err) + } + + if err := fileutils.WriteFile(p.fs, + filepath.Join(sourceCopy, specFilename), prepSpec, fileperms.PublicFile); err != nil { + return fmt.Errorf("staging Go prep spec:\n%w", err) + } + + if err := fileutils.WriteFile(p.fs, filepath.Join(staging, goVendorConfig), + []byte(toolConfig), fileperms.PublicFile); err != nil { + return fmt.Errorf("staging Go vendor tool configuration:\n%w", err) + } + + return nil +} + +func (p *sourcePreparerImpl) publishGoVendor( + staging, outputDir, specPath, archiveName string, data []byte, toolConfig string, +) error { + archiveInfo, err := p.fs.Stat(filepath.Join(staging, archiveName)) + if err != nil { + return fmt.Errorf("missing generated Go vendor archive:\n%w", err) + } + + if !archiveInfo.Mode().IsRegular() || archiveInfo.Size() == 0 { + return errors.New("generated Go vendor archive must be a nonempty regular file") + } + + license, err := fileutils.ReadFile(p.fs, filepath.Join(staging, "license")) + if err != nil { + return fmt.Errorf("reading Go vendor license report:\n%w", err) + } + + expression := strings.TrimSpace(string(license)) + if expression == "" || strings.ContainsAny(expression, "\r\n%") { + return fmt.Errorf("invalid license expression returned by 'go_vendor_license': %#q", expression) + } + + editor, err := spec.OpenSpec(bytes.NewReader(data), spec.WithEditor(spec.EditorStructural)) + if err != nil { + return fmt.Errorf("reading converted Go spec:\n%w", err) + } + + if err := editor.UpdateExistingTag("", "License", expression); err != nil { + return fmt.Errorf("updating bundled Go licenses:\n%w", err) + } + + var finalSpec bytes.Buffer + if err := editor.Serialize(&finalSpec); err != nil { + return fmt.Errorf("serializing converted Go spec:\n%w", err) + } + // Publish only after generation and license detection succeed. + if err := fileutils.CopyFile(p.dryRunnable, p.fs, + filepath.Join(staging, archiveName), filepath.Join(outputDir, archiveName), fileutils.CopyFileOptions{}); err != nil { + return fmt.Errorf("publishing Go vendor archive:\n%w", err) + } + + for name, contents := range map[string][]byte{ + filepath.Base(specPath): finalSpec.Bytes(), + goVendorConfig: []byte(toolConfig), + } { + if err := fileutils.WriteFile(p.fs, filepath.Join(outputDir, name), contents, fileperms.PublicFile); err != nil { + return fmt.Errorf("publishing Go vendor file %#q:\n%w", name, err) + } + } + + return p.recordGoVendorArchive(outputDir, archiveName) +} + +func (p *sourcePreparerImpl) runGoVendor( + ctx context.Context, runner *mock.Runner, archiveName, specFilename string, config *projectconfig.GoVendorConfig, +) (err error) { + defer func() { + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), goVendorCleanupTimeout) + defer cancel() + + if scrubErr := runner.ScrubRoot(cleanupCtx); scrubErr != nil { + slog.Error("Failed to clean Go vendor mock root", "error", scrubErr) + + if err == nil { + err = scrubErr + } + } + }() + + if err := runner.InitRoot(ctx); err != nil { + return fmt.Errorf("initializing Go vendor mock root:\n%w", err) + } + + packages := append([]string{ + "go-vendor-tools", "askalono-cli", "golang", "go-rpm-macros", "rpm-build", "rpmdevtools", + "rpmautospec", "git", "python3-click", + }, config.PrepPackages...) + if err := runner.InstallPackages(ctx, packages); err != nil { + return fmt.Errorf("installing Go vendor prerequisites:\n%w", err) + } + + cmd, err := runner.CmdInChroot(ctx, []string{ + "bash", "-euo", "pipefail", "-c", + goVendorScript(archiveName, config.ModuleDir, runner.ConfigOpts()["target_arch"], specFilename), + }, false) + if err != nil { + return fmt.Errorf("constructing Go vendor command:\n%w", err) + } + + output, err := cmd.RunAndGetOutput(ctx) + if err != nil { + return fmt.Errorf("go vendor generation or license detection failed; use 'prep-packages' for missing prep tools "+ + "and 'tool-config' for reviewed license exceptions:\n%s\n%w", output, err) + } + + return nil +} + +func goVendorScript(archiveName, moduleDir, targetArch, specFilename string) string { + if moduleDir == "" { + moduleDir = "." + } + + quote := shellquote.Join + + prepArgs := []string{ + "rpmbuild", "-bp", "--nodeps", "--define", "_topdir /tmp/azldev-go-rpm", + "--define", "_builddir " + goVendorBuild, "--define", "_sourcedir " + goVendorMount, + } + if targetArch != "" { + prepArgs = append(prepArgs, "--target", targetArch) + } + + prepArgs = append(prepArgs, goVendorMount+"/"+specFilename) + + return strings.Join([]string{ + "cd " + goVendorMount, + "mkdir -p " + goVendorBuild, + quote("git", "config", "--global", "--add", "safe.directory", goVendorMount), + quote("rpmautospec", "process-distgit", specFilename, specFilename), + quote(prepArgs...), + `IFS= read -r module_root < ` + goVendorTopDir + "/azldev-go-module-root", + `cd "$module_root"`, + "cd " + quote(moduleDir), + "test -f go.mod", + "export GOTOOLCHAIN=local", + quote("go_vendor_archive", "create", "--config", goVendorScratch+"/"+goVendorConfig, + "--output", goVendorScratch+"/"+archiveName, "."), + quote("tar", "-xf", goVendorScratch+"/"+archiveName), + quote("go_vendor_license", "--config", goVendorScratch+"/"+goVendorConfig, "--path", ".", + "report", "--no-prompt", "--autofill", "off", "expression") + " > " + goVendorScratch + "/license", + "cd " + goVendorMount, + quote("spectool", "--define", "_sourcedir "+goVendorMount, "-l", "-a", + specFilename) + " > " + goVendorScratch + "/sources-list", + }, "\n") +} + +func (p *sourcePreparerImpl) checkGoVendorSourceCollision(outputDir, name string) error { + for _, filename := range []string{name, goVendorConfig} { + exists, err := fileutils.Exists(p.fs, filepath.Join(outputDir, filename)) + if err != nil { + return fmt.Errorf("checking generated Go source path:\n%w", err) + } + + if exists { + return fmt.Errorf("generated Go vendor file %#q conflicts with an existing source", filename) + } + } + + content, err := p.readSourcesFileIfExists(filepath.Join(outputDir, fedorasource.SourcesFileName)) + if err != nil { + return err + } + + lines, err := fedorasource.ReadSourcesFile(content) + if err != nil { + return fmt.Errorf("reading Go vendor 'sources' entries:\n%w", err) + } + + for _, line := range lines { + if line.Entry != nil && line.Entry.Filename == name { + return fmt.Errorf("generated Go vendor archive %#q conflicts with an upstream 'sources' entry", name) + } + } + + return nil +} + +func (p *sourcePreparerImpl) recordGoVendorArchive(outputDir, name string) error { + hash, err := fileutils.ComputeFileHash(p.fs, fileutils.HashTypeSHA512, filepath.Join(outputDir, name)) + if err != nil { + return fmt.Errorf("hashing generated Go vendor archive:\n%w", err) + } + + content, err := p.readSourcesFileIfExists(filepath.Join(outputDir, fedorasource.SourcesFileName)) + if err != nil { + return err + } + + content = strings.TrimRight(content, "\n") + if content != "" { + content += "\n" + } + + content += fedorasource.FormatSourcesEntry(name, fileutils.HashTypeSHA512, hash) + "\n" + if err := fileutils.WriteFile(p.fs, filepath.Join(outputDir, fedorasource.SourcesFileName), + []byte(content), fileperms.PublicFile); err != nil { + return fmt.Errorf("recording Go vendor digest:\n%w", err) + } + // Generated payloads belong in lookaside storage, not synthetic git commits. + ignorePath := filepath.Join(outputDir, ".gitignore") + + ignore, err := p.readSourcesFileIfExists(ignorePath) + if err != nil { + return err + } + + if err := fileutils.WriteFile(p.fs, ignorePath, + []byte(strings.TrimRight(ignore, "\n")+"\n/"+name+"\n"), fileperms.PublicFile); err != nil { + return fmt.Errorf("ignoring generated Go archive in git:\n%w", err) + } + + return nil +} + +func (p *sourcePreparerImpl) validateGoVendorInputs(scratch, outputDir string) error { + data, err := fileutils.ReadFile(p.fs, filepath.Join(scratch, "sources-list")) + if err != nil { + return fmt.Errorf("reading prepared Go source list:\n%w", err) + } + + for _, line := range strings.Split(strings.TrimSpace(string(data)), "\n") { + if line == "" { + continue + } + + files := spectool.ParseSpectoolOutput(line) + if len(files) != 1 || strings.Contains(files[0], "%") { + return fmt.Errorf("cannot resolve Go source/patch reference %#q", line) + } + + info, err := p.fs.Stat(filepath.Join(outputDir, files[0])) + if err != nil { + return fmt.Errorf("missing prepared Go source/patch %#q:\n%w", files[0], err) + } + + if !info.Mode().IsRegular() { + return fmt.Errorf("go source/patch %#q is not a regular file", files[0]) + } + } + + return nil +} diff --git a/internal/app/azldev/core/sources/govendor_internal_test.go b/internal/app/azldev/core/sources/govendor_internal_test.go new file mode 100644 index 000000000..d79bbf4a5 --- /dev/null +++ b/internal/app/azldev/core/sources/govendor_internal_test.go @@ -0,0 +1,244 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package sources + +import ( + "context" + "errors" + "os/exec" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components/components_testutils" + "github.com/microsoft/azure-linux-dev-tools/internal/global/testctx" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders" + "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders/fedorasource" + "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders/sourceproviders_test" + "github.com/microsoft/azure-linux-dev-tools/internal/rpm/mock" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/samber/lo" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +func goVendorTestSpec() string { + return strings.Join([]string{ + "Name: app", "Version: 1", "Release: 1", "Summary: app", "License: MIT", + "Source0: app.tar.gz", "Patch0: fix.patch", "BuildRequires: golang(example.com/dep)", + "%description", "app", "%prep", "%goprep -p1", "%build", "%gobuild -o app .", + "%install", "install -D app %{buildroot}%{_bindir}/app", "%files", "%{_bindir}/app", "", + }, "\n") +} + +func TestPrepareSourcesGoVendor(t *testing.T) { + for _, skipLookaside := range []bool{false, true} { + t.Run(map[bool]string{false: "prepare", true: "render"}[skipLookaside], func(t *testing.T) { + ctx := newMockProcessorTestCtx() + ctrl := gomock.NewController(t) + component := components_testutils.NewMockComponent(ctrl) + manager := sourceproviders_test.NewMockSourceManager(ctrl) + config := &projectconfig.ComponentConfig{ + GoVendor: &projectconfig.GoVendorConfig{Enabled: lo.ToPtr(true), PrepPackages: []string{"git"}}, + Overlays: []projectconfig.ComponentOverlay{{ + Type: "spec-set-tag", Tag: "Summary", Value: "overlaid", + }}, + } + + component.EXPECT().GetName().AnyTimes().Return("app") + component.EXPECT().GetConfig().AnyTimes().Return(config) + manager.EXPECT().FetchComponent(gomock.Any(), component, "/output", gomock.Any()).DoAndReturn( + func(_ context.Context, _ any, _ string, _ ...sourceproviders.FetchComponentOption) error { + for name, content := range map[string]string{ + "app.spec": goVendorTestSpec(), "app.tar.gz": "upstream", "fix.patch": "patch", + "sources": "# upstream\n", ".gitignore": "/app.tar.gz\n", + } { + require.NoError(t, fileutils.WriteFile(ctx.FS(), "/output/"+name, []byte(content), fileperms.PublicFile)) + } + + return nil + }) + manager.EXPECT().FetchFiles(gomock.Any(), component, "/output").Return(nil) + + var calls []string + + ctx.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + calls = append(calls, strings.Join(cmd.Args, " ")) + + return nil + } + ctx.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + assert.Contains(t, cmd.Args, "--unpriv") + assert.Contains(t, cmd.Args, "--enable-network") + assert.Contains(t, strings.Join(cmd.Args, " "), "go_vendor_archive") + staging := goVendorTestStaging(t, ctx) + prep, err := fileutils.ReadFile(ctx.FS(), filepath.Join(filepath.Dir(staging), "sources", "app.spec")) + require.NoError(t, err) + assert.Contains(t, string(prep), "Summary: overlaid", "overlays must run before generation") + + for name, content := range map[string]string{ + "app-vendor.tar.bz2": "generated archive", + "license": "Apache-2.0 AND MIT\n", + "sources-list": "Source0: app.tar.gz\nPatch0: fix.patch\n", + } { + require.NoError(t, fileutils.WriteFile(ctx.FS(), + filepath.Join(staging, name), []byte(content), fileperms.PublicFile)) + } + + return "", nil + } + + options := []PreparerOption{WithGoVendorRunner(mock.NewRunner(ctx, "/mock.cfg"))} + if skipLookaside { + options = append(options, WithSkipLookaside()) + } + + preparer, err := NewPreparer(manager, ctx.FS(), ctx, ctx, options...) + require.NoError(t, err) + require.NoError(t, preparer.PrepareSources(ctx, component, "/output", true)) + output, err := fileutils.ReadFile(ctx.FS(), "/output/app.spec") + require.NoError(t, err) + assert.Contains(t, string(output), "License: Apache-2.0 AND MIT") + assert.Contains(t, string(output), "tar -xf %{SOURCE1}") + assert.NotContains(t, string(output), "BuildRequires: golang(") + + digest, err := fileutils.ComputeFileHash(ctx.FS(), fileutils.HashTypeSHA512, "/output/app-vendor.tar.bz2") + require.NoError(t, err) + sources, err := fileutils.ReadFile(ctx.FS(), "/output/sources") + require.NoError(t, err) + assert.Contains(t, string(sources), "# upstream\n") + assert.Contains(t, string(sources), + fedorasource.FormatSourcesEntry("app-vendor.tar.bz2", fileutils.HashTypeSHA512, digest)) + ignored, err := fileutils.ReadFile(ctx.FS(), "/output/.gitignore") + require.NoError(t, err) + assert.Equal(t, "/app.tar.gz\n/app-vendor.tar.bz2\n", string(ignored)) + assert.Contains(t, strings.Join(calls, "\n"), "--install go-vendor-tools askalono-cli") + assert.Contains(t, strings.Join(calls, "\n"), " git") + assert.Contains(t, strings.Join(calls, "\n"), "--scrub all") + assert.NotContains(t, strings.Join(calls, "\n"), "go2rpm") + + matches, err := fileutils.Glob(ctx.FS(), "/tmp/azldev-go-vendor-*") + require.NoError(t, err) + assert.Empty(t, matches) + }) + } +} + +func goVendorTestStaging(t *testing.T, ctx *testctx.TestCtx) string { + t.Helper() + + matches, err := fileutils.Glob(ctx.FS(), "/tmp/azldev-go-vendor-*/work/azldev-go-vendor.toml") + require.NoError(t, err) + require.Len(t, matches, 1) + + return filepath.Dir(matches[0]) +} + +func TestGoVendorFailuresDoNotPublishSpec(t *testing.T) { + failure := errors.New("tool failed") + + for _, stage := range []string{ + "init", "install", "generate", "missing-archive", "empty-archive", + "license", "missing-source", "source-list", "scrub", + } { + t.Run(stage, func(t *testing.T) { + ctx := newMockProcessorTestCtx() + ctrl := gomock.NewController(t) + component := components_testutils.NewMockComponent(ctrl) + component.EXPECT().GetName().AnyTimes().Return("app") + component.EXPECT().GetConfig().AnyTimes().Return(&projectconfig.ComponentConfig{ + GoVendor: &projectconfig.GoVendorConfig{Enabled: lo.ToPtr(true)}, + }) + require.NoError(t, fileutils.WriteFile(ctx.FS(), + "/output/app.spec", []byte(goVendorTestSpec()), fileperms.PublicFile)) + require.NoError(t, fileutils.WriteFile(ctx.FS(), "/output/app.tar.gz", []byte("source"), fileperms.PublicFile)) + ctx.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + if slices.Contains(cmd.Args, "--"+stage) { + return failure + } + + return nil + } + ctx.CmdFactory.RunAndGetOutputHandler = func(_ *exec.Cmd) (string, error) { + if stage == "generate" { + return "license failure details", failure + } + + staging := goVendorTestStaging(t, ctx) + files := map[string]string{ + "app-vendor.tar.bz2": "archive", "license": "MIT\n", "sources-list": "Source0: app.tar.gz\n", + } + + switch stage { + case "missing-archive": + delete(files, "app-vendor.tar.bz2") + case "empty-archive": + files["app-vendor.tar.bz2"] = "" + case "license": + files["license"] = "bad\nLicense" + case "missing-source": + files["sources-list"] += "Patch0: absent.patch\n" + case "source-list": + files["sources-list"] = "Source0: /etc/passwd\n" + } + + for name, data := range files { + require.NoError(t, fileutils.WriteFile(ctx.FS(), filepath.Join(staging, name), []byte(data), fileperms.PublicFile)) + } + + return "", nil + } + p := &sourcePreparerImpl{fs: ctx.FS(), dryRunnable: ctx, goVendorRunner: mock.NewRunner(ctx, "/mock.cfg")} + err := p.prepareGoVendor(ctx, component, "/output") + require.Error(t, err) + + unchanged, readErr := fileutils.ReadFile(ctx.FS(), "/output/app.spec") + require.NoError(t, readErr) + assert.Equal(t, goVendorTestSpec(), string(unchanged)) + + exists, readErr := fileutils.Exists(ctx.FS(), "/output/app-vendor.tar.bz2") + require.NoError(t, readErr) + assert.False(t, exists) + }) + } +} + +func TestGoVendorScriptOrdering(t *testing.T) { + script := goVendorScript("app-vendor.tar.bz2", "cmd/app", "aarch64", "app.spec") + assert.Less(t, strings.Index(script, "rpmautospec"), strings.Index(script, "rpmbuild")) + assert.Less(t, strings.Index(script, "rpmbuild"), strings.Index(script, "go_vendor_archive")) + assert.Less(t, strings.Index(script, "go_vendor_archive"), strings.Index(script, "go_vendor_license")) + assert.Contains(t, script, "cd cmd/app") + assert.Contains(t, script, "--target aarch64") + assert.Contains(t, script, "--no-prompt") + assert.Contains(t, script, "--autofill off") + assert.NotContains(t, script, "--idempotent") +} + +func TestGoVendorSourceCollisions(t *testing.T) { + for _, name := range []string{"app-vendor.tar.bz2", goVendorConfig, "sources", "invalid-sources"} { + t.Run(name, func(t *testing.T) { + ctx := testctx.NewCtx() + preparer := &sourcePreparerImpl{fs: ctx.FS()} + content := "existing" + + filename := name + switch name { + case "sources": + content = fedorasource.FormatSourcesEntry( + "app-vendor.tar.bz2", fileutils.HashTypeSHA512, strings.Repeat("a", 128)) + "\n" + case "invalid-sources": + filename = "sources" + content = "SHA512 (broken" + } + + require.NoError(t, fileutils.WriteFile(ctx.FS(), "/output/"+filename, []byte(content), fileperms.PublicFile)) + require.Error(t, preparer.checkGoVendorSourceCollision("/output", "app-vendor.tar.bz2")) + }) + } +} diff --git a/internal/app/azldev/core/sources/sourceprep.go b/internal/app/azldev/core/sources/sourceprep.go index 4edeb30b6..8759787d8 100644 --- a/internal/app/azldev/core/sources/sourceprep.go +++ b/internal/app/azldev/core/sources/sourceprep.go @@ -24,6 +24,7 @@ import ( "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders" "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders/fedorasource" + "github.com/microsoft/azure-linux-dev-tools/internal/rpm/mock" "github.com/microsoft/azure-linux-dev-tools/internal/rpm/spec" "github.com/microsoft/azure-linux-dev-tools/internal/utils/dirdiff" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" @@ -133,7 +134,8 @@ func WithSpecEditor(mode spec.EditorMode) PreparerOption { // component fetching ([FedoraSourceDownloader.ExtractSourcesFromRepo]). // Git-tracked files (spec, patches, scripts, configs) are still fetched from // the upstream clone. This is useful for rendering, where only the spec and -// sidecar files are needed and downloading large source tarballs is unnecessary. +// sidecar files are normally needed. Enabled Go vendor conversion overrides this +// optimization because its archive and license expression require prepared sources. func WithSkipLookaside() PreparerOption { return func(p *sourcePreparerImpl) { p.skipLookaside = true @@ -184,11 +186,12 @@ func WithAllowNoHashes() PreparerOption { // Standard implementation of the [SourcePreparer] interface. type sourcePreparerImpl struct { - sourceManager sourceproviders.SourceManager - specEditor spec.EditorMode - fs opctx.FS - eventListener opctx.EventListener - dryRunnable opctx.DryRunnable + sourceManager sourceproviders.SourceManager + specEditor spec.EditorMode + fs opctx.FS + eventListener opctx.EventListener + dryRunnable opctx.DryRunnable + goVendorRunner *mock.Runner // withGitRepo, when true, enables dist-git creation by preserving the // upstream .git directory and generating synthetic commit history. @@ -297,6 +300,8 @@ func NewPreparer( func (p *sourcePreparerImpl) PrepareSources( ctx context.Context, component components.Component, outputDir string, applyOverlays bool, ) error { + p = p.withRequiredDownloads(component, applyOverlays) + if applyOverlays { if err := p.validateArchiveOverlayConfig(component); err != nil { return err @@ -360,6 +365,21 @@ func (p *sourcePreparerImpl) PrepareSources( return nil } +func (p *sourcePreparerImpl) withRequiredDownloads( + component components.Component, applyConversion bool, +) *sourcePreparerImpl { + if applyConversion && p.skipLookaside && component.GetConfig().GoVendor.IsEnabled() { + effective := *p + effective.skipLookaside = false + + slog.Info("Go vendoring requires source downloads, including during rendering", "component", component.GetName()) + + return &effective + } + + return p +} + func (p *sourcePreparerImpl) validateArchiveOverlayConfig(component components.Component) error { config := component.GetConfig() if err := config.ValidateArchiveOverlays(p.allowNoHashes); err != nil { @@ -394,6 +414,10 @@ func (p *sourcePreparerImpl) applyOverlaysToSources( component.GetName(), err) } + if err := p.prepareGoVendor(ctx, component, outputDir); err != nil { + return nil, fmt.Errorf("preparing Go vendor sources for component %#q:\n%w", component.GetName(), err) + } + return repackedArchives, nil } @@ -740,6 +764,8 @@ func removeSubmoduleEntries(fs opctx.FS, repo *gogit.Repository, repoDir string) func (p *sourcePreparerImpl) DiffSources( ctx context.Context, component components.Component, baseDir string, ) (result *dirdiff.DiffResult, err error) { + p = p.withRequiredDownloads(component, true) + event := p.eventListener.StartEvent("Computing overlay diff", "component", component.GetName()) defer event.End() diff --git a/internal/projectconfig/component.go b/internal/projectconfig/component.go index 6b56f9d8e..cf753e065 100644 --- a/internal/projectconfig/component.go +++ b/internal/projectconfig/component.go @@ -374,6 +374,8 @@ type ComponentLockData struct { } // Defines a component. +// +//nolint:recvcheck // HashInclude must also support configs passed to hashstructure by value. type ComponentConfig struct { // The component's name; not actually present in serialized files. Name string `toml:"-" json:"name" table:",sortkey" fingerprint:"-"` @@ -432,6 +434,9 @@ type ComponentConfig struct { // Source file references for this component. SourceFiles []SourceFileReference `toml:"source-files,omitempty" json:"sourceFiles,omitempty" table:"-" jsonschema:"title=Source files,description=Source files to download for this component"` + // GoVendor converts supported Go application specs and generates their vendored sources. + GoVendor *GoVendorConfig `toml:"go-vendor,omitempty" json:"goVendor,omitempty" table:"-" jsonschema:"title=Go vendoring,description=Convert a Go application spec and generate its vendor archive during source preparation"` + // Per-package configuration overrides, keyed by exact binary package name. // Takes precedence over package-group defaults. Packages map[string]PackageConfig `toml:"packages,omitempty" json:"packages,omitempty" table:"-" validate:"dive" jsonschema:"title=Package overrides,description=Per-package configuration overrides keyed by exact binary package name"` @@ -450,6 +455,11 @@ type ComponentConfig struct { Tests *ComponentTestsConfig `toml:"tests,omitempty" json:"tests,omitempty" table:"-" jsonschema:"title=Tests,description=Per-component test or test-group references" fingerprint:"-"` } +// HashInclude preserves existing fingerprints when Go vendoring is not configured. +func (c ComponentConfig) HashInclude(field string, _ any) (bool, error) { + return field != "GoVendor" || c.GoVendor != nil, nil +} + // AllowedSourceFilesHashTypes defines the set of hash types that are supported // for use in [SourceFileReference] entries in component configs. // MD5 is excluded by design. @@ -480,9 +490,20 @@ func (c *ComponentConfig) MergeUpdatesFrom(other *ComponentConfig) error { c.OverlayFiles = otherOverlayFiles } + c.overrideGoVendorEnabled(other) + return nil } +func (c *ComponentConfig) overrideGoVendorEnabled(other *ComponentConfig) { + if other.GoVendor != nil && other.GoVendor.Enabled != nil { + config := *c.GoVendor + enabled := *other.GoVendor.Enabled + config.Enabled = &enabled + c.GoVendor = &config + } +} + func (c *ComponentConfig) resolveLocalCustomScriptPaths() { if c.Spec.SourceType != SpecSourceTypeLocal || c.Spec.Path == "" { return @@ -549,6 +570,8 @@ func (c *ComponentConfig) MergeOverridesFrom(other *ComponentConfig) error { c.OverlayFiles = otherOverlayFiles } + c.overrideGoVendorEnabled(other) + return nil } diff --git a/internal/projectconfig/configfile.go b/internal/projectconfig/configfile.go index ee6f07c73..aee4be162 100644 --- a/internal/projectconfig/configfile.go +++ b/internal/projectconfig/configfile.go @@ -171,6 +171,10 @@ func validateComponentConfigs( } // Validate build configuration. + if err := component.GoVendor.Validate(); err != nil { + return fmt.Errorf("invalid 'go-vendor' config for component %#q:\n%w", componentName, err) + } + err := component.Build.Validate() if err != nil { return fmt.Errorf("invalid build config for component %#q:\n%w", componentName, err) diff --git a/internal/projectconfig/fingerprint_test.go b/internal/projectconfig/fingerprint_test.go index 836eee6f3..dcf89f830 100644 --- a/internal/projectconfig/fingerprint_test.go +++ b/internal/projectconfig/fingerprint_test.go @@ -36,6 +36,7 @@ func TestAllFingerprintedFieldsHaveDecision(t *testing.T) { reflect.TypeFor[projectconfig.Origin](), reflect.TypeFor[projectconfig.ReleaseConfig](), reflect.TypeFor[projectconfig.ComponentRenderConfig](), + reflect.TypeFor[projectconfig.GoVendorConfig](), } // Maps "StructName.FieldName" for every field that should carry a diff --git a/internal/projectconfig/govendor.go b/internal/projectconfig/govendor.go new file mode 100644 index 000000000..cf9fc4eae --- /dev/null +++ b/internal/projectconfig/govendor.go @@ -0,0 +1,66 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package projectconfig + +import ( + "errors" + "fmt" + "strings" + + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/pelletier/go-toml/v2" +) + +// GoVendorConfig describes a repeatable conversion, not a replacement spec. +type GoVendorConfig struct { + // Enabled is a pointer so an explicit false overrides inherited settings. + Enabled *bool `toml:"enabled,omitempty" json:"enabled,omitempty" jsonschema:"title=Enabled,description=Enable Go application vendoring"` + // ModuleDir is relative to the directory left by the spec's prep section. + ModuleDir string `toml:"module-dir,omitempty" json:"moduleDir,omitempty" jsonschema:"title=Module directory,description=Relative module directory within the prepared source tree; defaults to its root"` + // ToolConfig is inline go-vendor-tools TOML, including reviewed license exceptions. + ToolConfig string `toml:"tool-config,omitempty" json:"toolConfig,omitempty" jsonschema:"title=Tool configuration,description=Inline go-vendor-tools TOML for dependency overrides and licensing settings"` + // PrepPackages supplies additional tools used by upstream prep commands. + PrepPackages []string `toml:"prep-packages,omitempty" json:"prepPackages,omitempty" jsonschema:"title=Prep packages,description=Additional RPM packages needed to execute the upstream prep section in mock"` +} + +// IsEnabled reports whether vendoring was explicitly enabled. +func (c *GoVendorConfig) IsEnabled() bool { + return c != nil && c.Enabled != nil && *c.Enabled +} + +// Validate checks paths and tool configuration before any source processing. +func (c *GoVendorConfig) Validate() error { + if c == nil { + return nil + } + + if c.ModuleDir != "" { + if err := fileutils.ValidateRelPath(c.ModuleDir); err != nil { + return fmt.Errorf("invalid 'module-dir':\n%w", err) + } + + if strings.ContainsAny(c.ModuleDir, "%\n\r") { + return errors.New("'module-dir' must not contain RPM macros or newlines") + } + } + + var toolConfig map[string]any + if err := toml.Unmarshal([]byte(c.ToolConfig), &toolConfig); err != nil { + return fmt.Errorf("invalid 'tool-config' TOML:\n%w", err) + } + + if general, ok := toolConfig["general"].(map[string]any); ok { + if _, exists := general["go_mod_dir"]; exists { + return errors.New("use 'module-dir' instead of 'general.go_mod_dir' in 'tool-config'") + } + } + + for _, name := range c.PrepPackages { + if name == "" || strings.HasPrefix(name, "-") || strings.ContainsAny(name, "\r\n") { + return fmt.Errorf("invalid 'prep-packages' entry %#q", name) + } + } + + return nil +} diff --git a/internal/projectconfig/govendor_test.go b/internal/projectconfig/govendor_test.go new file mode 100644 index 000000000..1c363001a --- /dev/null +++ b/internal/projectconfig/govendor_test.go @@ -0,0 +1,80 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package projectconfig_test + +import ( + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/mitchellh/hashstructure/v2" + "github.com/pelletier/go-toml/v2" + "github.com/samber/lo" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGoVendorConfig(t *testing.T) { + var config projectconfig.ComponentConfig + require.NoError(t, toml.Unmarshal([]byte(` +[go-vendor] +enabled = true +module-dir = "cmd/app" +prep-packages = ["git"] +tool-config = ''' +[archive.dependency_overrides] +"example.com/module" = "v1.2.3" +''' +`), &config)) + require.True(t, config.GoVendor.IsEnabled()) + require.NoError(t, config.GoVendor.Validate()) + require.NoError(t, config.MergeUpdatesFrom(&projectconfig.ComponentConfig{ + GoVendor: &projectconfig.GoVendorConfig{Enabled: lo.ToPtr(false)}, + })) + assert.False(t, config.GoVendor.IsEnabled(), "explicit false must override an inherited true") +} + +func TestGoVendorConfigValidation(t *testing.T) { + for _, config := range []*projectconfig.GoVendorConfig{ + {ModuleDir: "../outside"}, + {ModuleDir: "/absolute"}, + {ModuleDir: "%{macro}"}, + {ToolConfig: "["}, + {ToolConfig: "[general]\ngo_mod_dir = 'subdir'"}, + {PrepPackages: []string{"--bad"}}, + {PrepPackages: []string{""}}, + } { + require.Error(t, config.Validate()) + } + + var absent *projectconfig.GoVendorConfig + assert.False(t, absent.IsEnabled()) + require.NoError(t, absent.Validate()) +} + +func TestGoVendorFingerprint(t *testing.T) { + empty := projectconfig.ComponentConfig{} + included, err := empty.HashInclude("GoVendor", nil) + require.NoError(t, err) + assert.False(t, included, "absent vendoring must not invalidate existing component fingerprints") + included, err = empty.HashInclude("Build", nil) + require.NoError(t, err) + assert.True(t, included) + + base := projectconfig.ComponentConfig{GoVendor: &projectconfig.GoVendorConfig{Enabled: lo.ToPtr(true)}} + original, err := hashstructure.Hash(base, hashstructure.FormatV2, &hashstructure.HashOptions{TagName: "fingerprint"}) + require.NoError(t, err) + + for _, config := range []*projectconfig.GoVendorConfig{ + {Enabled: lo.ToPtr(false)}, + {Enabled: lo.ToPtr(true), ModuleDir: "sub"}, + {Enabled: lo.ToPtr(true), ToolConfig: "[archive]\ntidy = false"}, + {Enabled: lo.ToPtr(true), PrepPackages: []string{"git"}}, + } { + changed := base + changed.GoVendor = config + hash, err := hashstructure.Hash(changed, hashstructure.FormatV2, &hashstructure.HashOptions{TagName: "fingerprint"}) + require.NoError(t, err) + assert.NotEqual(t, original, hash) + } +} diff --git a/internal/rpm/spec/govendor.go b/internal/rpm/spec/govendor.go new file mode 100644 index 000000000..29791e3c1 --- /dev/null +++ b/internal/rpm/spec/govendor.go @@ -0,0 +1,322 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package spec + +import ( + "errors" + "fmt" + "regexp" + "slices" + "strconv" + "strings" + + "github.com/kballard/go-shellquote" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" +) + +// GoVendorOptions names generated artifacts. All paths except ModuleDir are basenames. +type GoVendorOptions struct { + ArchiveName string + ConfigName string + ModuleDir string +} + +// GoVendorSpec contains the converted spec and the otherwise identical prep-only +// spec used to obtain the patched module tree before a vendor archive exists. +type GoVendorSpec struct { + Spec []byte + PrepSpec []byte +} + +const goVendorEnvironment = `export GO111MODULE=on GOPROXY=off GOSUMDB=off GOTOOLCHAIN=local +export GOFLAGS="${GOFLAGS:+$GOFLAGS }-mod=vendor"` + +var goBuildRequirement = regexp.MustCompile(`^golang\([^)]+\)(\s*(>=|<=|=|>|<)\s*[^\s,]+)?$`) + +var ( + goVendorUnsupportedMacro = regexp.MustCompile( + `%(?:\{[!?]*)?(?:gopkg\w*|goinstall|gofiles?\w*|go_vendor_\w*|include)\b`) + goVendorRequiresMacro = regexp.MustCompile(`%(?:\{[!?]*)?go_generate_buildrequires\b`) + goVendorCheckMacro = regexp.MustCompile(`%(?:\{[!?]*)?gocheck\b`) +) + +// ConvertGoVendor changes only recognized Go application packaging constructs. +// It always uses the lossless tree, independently of the selected overlay editor. +// Unsupported Go macros and ambiguous section ownership fail before returning output. +func ConvertGoVendor(data []byte, options GoVendorOptions) (*GoVendorSpec, error) { + if err := options.validate(); err != nil { + return nil, err + } + + root, err := parseTree(strings.Split(strings.TrimSuffix(string(data), "\n"), "\n")) + if err != nil { + return nil, err + } + + tree := &specTree{root: root} + if err := validateGoVendorTree(tree); err != nil { + return nil, err + } + + if err := convertGoVendorLines(tree); err != nil { + return nil, err + } + + // Do not replace upstream prep or build scripts: capture the directory after + // their existing prep commands, including patches and module-file edits. + prep := tree.Section("%prep", "") + prep.PrependLines([]string{"export GOTOOLCHAIN=local"}) + prep.AppendLines([]string{"", `pwd -P > "%{_topdir}/azldev-go-module-root"`}) + + prepSpec := serializeGoVendorTree(tree) + + sourceNumber, err := freeGoVendorSources(tree, options) + if err != nil { + return nil, err + } + + configRef := fmt.Sprintf("%%{SOURCE%d}", sourceNumber+1) + + extract := fmt.Sprintf("tar -xf %%{SOURCE%d}", sourceNumber) + if options.ModuleDir != "" { + extract += " -C " + shellquote.Join(options.ModuleDir) + } + + prep.AppendLines([]string{"", extract}) + tree.Section("", "").AppendLines([]string{ + fmt.Sprintf("Source%d: %s", sourceNumber, options.ArchiveName), + fmt.Sprintf("Source%d: %s", sourceNumber+1, options.ConfigName), + "BuildRequires: go-vendor-tools", + "BuildRequires: go-rpm-macros", + "BuildRequires: golang", + "%global gomodulesmode GO111MODULE=on", + "%global go_vendor_license_filelist() %{_topdir}/azldev-go-vendor-licenses.list", + "", + }) + + addGoVendorBuildSections(tree, options, configRef) + + return &GoVendorSpec{Spec: serializeGoVendorTree(tree), PrepSpec: prepSpec}, nil +} + +func addGoVendorBuildSections(tree *specTree, options GoVendorOptions, configRef string) { + requires := tree.Section("%generate_buildrequires", "") + if requires == nil { + blk := &block{Kind: sectionBlock, Name: "%generate_buildrequires", Header: "%generate_buildrequires"} + buildIndex := slices.Index(tree.root.Children, tree.Section("%build", "").block) + tree.root.Children = slices.Insert(tree.root.Children, buildIndex, blk) + requires = §ionHandle{block: blk, tree: tree} + } + + requires.PrependLines([]string{"%go_vendor_license_buildrequires -c " + configRef}) + + for _, name := range []string{"%build", "%install", "%check", "%generate_buildrequires"} { + if section := tree.Section(name, ""); section != nil { + section.PrependLines([]string{goVendorEnvironment}) + } + } + + tree.Section("%install", "").AppendLines([]string{ + "", goVendorInModule(options.ModuleDir, "%go_vendor_license_install -c "+configRef), + }) + // Licensing is a build invariant, even when the component disables %check. + tree.Section("%build", "").PrependLines([]string{ + goVendorInModule(options.ModuleDir, "%go_vendor_license_check -c "+configRef), + }) + files := tree.Section("%files", "") + files.block.Header += " -f %{go_vendor_license_filelist}" +} + +func (o GoVendorOptions) validate() error { + for _, name := range []string{o.ArchiveName, o.ConfigName} { + if err := fileutils.ValidateFilename(name); err != nil { + return fmt.Errorf("invalid Go vendor artifact name:\n%w", err) + } + + if strings.ContainsAny(name, "%\r\n") { + return errors.New("go vendor artifact name must not contain macros or newlines") + } + } + + if o.ModuleDir != "" { + if err := fileutils.ValidateRelPath(o.ModuleDir); err != nil { + return fmt.Errorf("invalid Go module directory:\n%w", err) + } + + if strings.ContainsAny(o.ModuleDir, "%\r\n") { + return errors.New("go module directory must not contain macros or newlines") + } + } + + return nil +} + +func serializeGoVendorTree(tree *specTree) []byte { + return []byte(strings.Join(serializeTree(tree.root), "\n") + "\n") +} + +func goVendorInModule(moduleDir, command string) string { + lines := []string{`( cd "$(cat "%{_topdir}/azldev-go-module-root")"`} + if moduleDir != "" { + lines = append(lines, "cd "+shellquote.Join(moduleDir)) + } + + lines = append(lines, command, ")") + + return strings.Join(lines, "\n") +} + +func validateGoVendorTree(tree *specTree) error { + if tree.HasSection("%sourcelist") { + return errors.New("go vendoring requires explicit Source tags instead of '%sourcelist'") + } + + for _, name := range []string{"", "%prep", "%build", "%install", "%files"} { + if err := validateGoVendorSection(tree, name, true); err != nil { + return err + } + } + + for _, name := range []string{"%check", "%generate_buildrequires"} { + if err := validateGoVendorSection(tree, name, false); err != nil { + return err + } + } + + var invalid error + + walkBlocks(tree.root, func(blk *block) bool { + if invalid != nil { + return false + } + + invalid = validateGoVendorBlock(blk) + + return true + }) + + return invalid +} + +func validateGoVendorSection(tree *specTree, name string, required bool) error { + sections := tree.Sections(name, "") + if len(sections) == 0 && !required { + return nil + } + + if len(sections) != 1 { + return fmt.Errorf("go vendoring requires one main-package %#q section; use overlays to disambiguate", name) + } + + if !slices.Contains(tree.root.Children, sections[0].block) { + return fmt.Errorf("go vendoring does not support conditional %#q sections; use an overlay", name) + } + + return nil +} + +func validateGoVendorBlock(blk *block) error { + if blk.Kind == macroDefBlock && (blk.Name == "gomodulesmode" || strings.HasPrefix(blk.Name, "go_vendor_")) { + return fmt.Errorf("explicit %#q conflicts with Go vendoring; remove it with an overlay", blk.Name) + } + + for _, line := range append([]string{blk.Header}, blk.Lines...) { + if strings.HasPrefix(strings.TrimSpace(line), "#") { + continue + } + + if macro := goVendorUnsupportedMacro.FindString(line); macro != "" { + return fmt.Errorf("unsupported Go vendoring macro %#q in %#q; adapt it with an overlay", macro, line) + } + + if blk.Kind == macroDefBlock && + (goVendorRequiresMacro.MatchString(line) || goVendorCheckMacro.MatchString(line) || + strings.Contains(line, "golang(")) { + return fmt.Errorf("go dependency or check logic inside a macro definition requires an explicit overlay: %#q", line) + } + + for _, macro := range []string{"-mod=mod", "GO111MODULE=off"} { + if strings.Contains(line, macro) { + return fmt.Errorf("unsupported Go vendoring construct %#q in %#q; adapt it with an overlay", macro, line) + } + } + } + + return nil +} + +func convertGoVendorLines(tree *specTree) error { + return tree.VisitAllLines(func(section, _ string, line *lineHandle) error { + text := strings.TrimSpace(line.Text) + if strings.HasPrefix(text, "#") { + return nil + } + + if tag, value, ok := parseTagLine(text); ok && isTagBearingSection(section) && + strings.EqualFold(tag, "BuildRequires") && strings.Contains(value, "golang(") { + if !goBuildRequirement.MatchString(value) { + return fmt.Errorf("ambiguous Go BuildRequires %#q; split it into individual tags with an overlay", line.Text) + } + + line.Remove() + } + + switch { + case goVendorRequiresMacro.MatchString(text): + if text != "%go_generate_buildrequires" || section != "%generate_buildrequires" { + return fmt.Errorf("unsupported Go dependency generator %#q; adapt it with an overlay", text) + } + + line.Remove() + case goVendorCheckMacro.MatchString(text): + if text != "%gocheck" || section != "%check" { + return fmt.Errorf("unsupported Go check command %#q; use a module-aware check overlay", text) + } + + line.Replace("%gotest ./...") + } + + return nil + }) +} + +func freeGoVendorSources(tree *specTree, options GoVendorOptions) (int, error) { + used := make(map[int]bool) + + err := tree.VisitAllLines(func(section, _ string, line *lineHandle) error { + tag, value, ok := parseTagLine(line.Text) + if !ok || !isTagBearingSection(section) || !strings.HasPrefix(strings.ToLower(tag), "source") { + return nil + } + + suffix := tag[len("source"):] + number := 0 + + if suffix != "" { + var err error + + number, err = strconv.Atoi(suffix) + if err != nil || number < 0 { + return fmt.Errorf("cannot allocate vendor Source tags alongside %#q", tag) + } + } + + used[number] = true + + if value == options.ArchiveName || value == options.ConfigName { + return fmt.Errorf("generated Go vendor source collides with existing Source tag %#q", line.Text) + } + + return nil + }) + if err != nil { + return 0, err + } + + for number := 0; ; number++ { + if !used[number] && !used[number+1] { + return number, nil + } + } +} diff --git a/internal/rpm/spec/govendor_test.go b/internal/rpm/spec/govendor_test.go new file mode 100644 index 000000000..cfed507aa --- /dev/null +++ b/internal/rpm/spec/govendor_test.go @@ -0,0 +1,197 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package spec_test + +import ( + "strings" + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/rpm/spec" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func vendorInput() string { + return strings.Join([]string{ + "# Upstream customization", + "%global goipath example.com/app", + "Name: app", + "Version: 1.2", + "Release: %autorelease", + "Summary: App", + "License: MIT", + "Source0: app-%{version}.tar.gz", + "Source1: app.service", + "Source9999: app.macros", + "Patch0: fix.patch", + "BuildRequires: golang(example.com/dep) >= 1.0", + "BuildRequires: gcc", + "Requires: systemd", + "", + "%description", + "App description", + "%package tools", + "Summary: Tools", + "Requires: app = %{version}-%{release}", + "%description tools", + "Tool description", + "%prep", + "%goprep -p1", + "sed -i 's/old/new/' go.mod", + "%generate_buildrequires", + "%go_generate_buildrequires", + "echo custom-build-dependency", + "%build", + "%gobuild -o app %{goipath}", + "%install", + "install -D app %{buildroot}%{_bindir}/app", + "%check", + "%if %{with check}", + "%gocheck", + "./custom-test", + "%endif", + "%files", + "%{_bindir}/app", + "%files tools", + "%{_bindir}/tool", + "%changelog", + "%autochangelog", + "", + }, "\n") +} + +func TestConvertGoVendorPreservesUpstream(t *testing.T) { + for _, version := range []string{"1.2", "2.7"} { + t.Run(version, func(t *testing.T) { + input := strings.ReplaceAll(vendorInput(), "Version: 1.2", "Version: "+version) + result, err := spec.ConvertGoVendor([]byte(input), spec.GoVendorOptions{ + ArchiveName: "app-vendor.tar.bz2", ConfigName: "vendor.toml", + }) + require.NoError(t, err) + + output := string(result.Spec) + for _, line := range []string{ + "# Upstream customization", "Version: " + version, "Release: %autorelease", + "Source0: app-%{version}.tar.gz", "Source1: app.service", "Source9999: app.macros", + "Patch0: fix.patch", "BuildRequires: gcc", "Requires: systemd", + "%package tools", "Requires: app = %{version}-%{release}", + "%goprep -p1", "sed -i 's/old/new/' go.mod", "echo custom-build-dependency", + "%gobuild -o app %{goipath}", "install -D app %{buildroot}%{_bindir}/app", + "%if %{with check}\n%gotest ./...\n./custom-test\n%endif", + "%files tools\n%{_bindir}/tool", "%autochangelog", + "Source2: app-vendor.tar.bz2", "Source3: vendor.toml", "tar -xf %{SOURCE2}", + "%go_vendor_license_buildrequires -c %{SOURCE3}", + "%go_vendor_license_check -c %{SOURCE3}", + "%go_vendor_license_install -c %{SOURCE3}", + "%files -f %{go_vendor_license_filelist}", + "GOPROXY=off", "-mod=vendor", "%global gomodulesmode GO111MODULE=on", + } { + assert.Contains(t, output, line) + } + + assert.NotContains(t, output, "BuildRequires: golang(") + assert.NotContains(t, output, "%go_generate_buildrequires") + assert.Contains(t, output, `pwd -P > "%{_topdir}/azldev-go-module-root"`) + assert.Contains(t, output, `%global go_vendor_license_filelist() %{_topdir}/azldev-go-vendor-licenses.list`) + assert.Contains(t, string(result.PrepSpec), "Release: %autorelease") + assert.Contains(t, string(result.PrepSpec), `pwd -P > "%{_topdir}/azldev-go-module-root"`) + assert.Contains(t, string(result.PrepSpec), "sed -i 's/old/new/' go.mod") + assert.NotContains(t, string(result.PrepSpec), "app-vendor.tar.bz2") + assert.Contains(t, string(result.PrepSpec), "%autochangelog") + + _, err = spec.OpenSpec(strings.NewReader(output), spec.WithEditor(spec.EditorStructural)) + require.NoError(t, err) + }) + } +} + +func TestConvertGoVendorRejectsAmbiguity(t *testing.T) { + for name, input := range map[string]string{ + "development subpackages": strings.ReplaceAll(vendorInput(), "%package tools", "%gopkg\n%package tools"), + "mixed requirements": strings.ReplaceAll(vendorInput(), + "golang(example.com/dep) >= 1.0", "gcc, golang(example.com/dep)"), + "check flags": strings.ReplaceAll(vendorInput(), "%gocheck", "%gocheck -z foo"), + "generator shell": strings.ReplaceAll(vendorInput(), + "%go_generate_buildrequires", "%go_generate_buildrequires | cat"), + "conditional prep": strings.ReplaceAll(vendorInput(), + "%prep\n%goprep -p1", "%if 1\n%prep\n%goprep -p1\n%endif"), + "module mode conflict": "%global gomodulesmode GO111MODULE=off\n" + vendorInput(), + "module mode override": "%global gomodulesmode GO111MODULE=on\n" + vendorInput(), + "license check override": "%global go_vendor_license_check_disable 1\n" + vendorInput(), + "existing vendor config": strings.ReplaceAll(vendorInput(), "app.service", "vendor.toml"), + "missing install": strings.ReplaceAll(vendorInput(), "%install", "# install"), + "macro source index": strings.ReplaceAll(vendorInput(), "Source1:", "Source%{number}:"), + "already vendored": strings.ReplaceAll(vendorInput(), "%gocheck", "%go_vendor_license_check"), + "braced generator": strings.ReplaceAll(vendorInput(), + "%go_generate_buildrequires", "%{go_generate_buildrequires}"), + "braced devel macro": strings.ReplaceAll(vendorInput(), "%package tools", "%{gopkg}\n%package tools"), + "hidden generator": "%global dependencies %{go_generate_buildrequires}\n" + vendorInput(), + "go filelist": strings.ReplaceAll(vendorInput(), "%files\n", "%files -f %{gofilelist}\n"), + "unsupported after comment": "# unrelated comment\n%gopkg\n" + vendorInput(), + } { + t.Run(name, func(t *testing.T) { + result, err := spec.ConvertGoVendor([]byte(input), spec.GoVendorOptions{ + ArchiveName: "app-vendor.tar.bz2", ConfigName: "vendor.toml", + }) + require.Error(t, err) + assert.Nil(t, result) + }) + } +} + +func TestConvertGoVendorModuleDirAndStaticRequires(t *testing.T) { + input := strings.ReplaceAll(vendorInput(), + "%generate_buildrequires\n%go_generate_buildrequires\necho custom-build-dependency\n", "") + input = strings.ReplaceAll(input, "%changelog\n%autochangelog\n", "") + result, err := spec.ConvertGoVendor([]byte(input), spec.GoVendorOptions{ + ArchiveName: "app-vendor.tar.bz2", ConfigName: "vendor.toml", ModuleDir: "cmd/app", + }) + require.NoError(t, err) + assert.Contains(t, string(result.Spec), "tar -xf %{SOURCE2} -C cmd/app") + assert.Contains(t, string(result.Spec), "cd cmd/app\n%go_vendor_license_install") + assert.Contains(t, string(result.Spec), `( cd "$(cat "%{_topdir}/azldev-go-module-root")"`) + assert.Contains(t, string(result.Spec), "%generate_buildrequires\n") +} + +func TestConvertGoVendorRejectsUnsafeOptions(t *testing.T) { + for _, options := range []spec.GoVendorOptions{ + {ArchiveName: "../vendor.tar.bz2", ConfigName: "config"}, + {ArchiveName: "vendor", ConfigName: "%{name}"}, + {ArchiveName: "vendor", ConfigName: "config", ModuleDir: "../escape"}, + {ArchiveName: "vendor", ConfigName: "config", ModuleDir: "%{bad}"}, + } { + _, err := spec.ConvertGoVendor([]byte(vendorInput()), options) + require.Error(t, err) + } +} + +func TestConvertGoVendorPreservesStaticReleaseAndCustomDirectories(t *testing.T) { + input := strings.ReplaceAll(vendorInput(), "Release: %autorelease", "Release: 42%{?dist}") + input = strings.ReplaceAll(input, "%check", "cd assets\n%check") + result, err := spec.ConvertGoVendor([]byte(input), spec.GoVendorOptions{ + ArchiveName: "app-vendor.tar.bz2", ConfigName: "vendor.toml", + }) + require.NoError(t, err) + assert.Contains(t, string(result.PrepSpec), "Release: 42%{?dist}") + assert.Contains(t, string(result.Spec), "cd assets") + assert.Contains(t, string(result.Spec), + `( cd "$(cat "%{_topdir}/azldev-go-module-root")"`+"\n%go_vendor_license_install") + assert.Contains(t, string(result.Spec), + "%global go_vendor_license_filelist() %{_topdir}/azldev-go-vendor-licenses.list") +} + +func TestConvertGoVendorPreservesNonTagsAndComments(t *testing.T) { + script := "cat <<'EOF' > metadata\nBuildRequires: golang(not-a-dependency)\nSource2: not-a-source\nEOF\n" + input := strings.ReplaceAll(vendorInput(), "%gobuild", script+"%gobuild") + input = "# %gopkg is intentionally not invoked\n" + input + input = strings.ReplaceAll(input, "%files\n", "%files\n# app files\n") + result, err := spec.ConvertGoVendor([]byte(input), spec.GoVendorOptions{ + ArchiveName: "app-vendor.tar.bz2", ConfigName: "vendor.toml", + }) + require.NoError(t, err) + assert.Contains(t, string(result.Spec), script) + assert.Contains(t, string(result.Spec), "Source2: app-vendor.tar.bz2") + assert.Contains(t, string(result.Spec), "# %gopkg is intentionally not invoked") + assert.Contains(t, string(result.Spec), "%files -f %{go_vendor_license_filelist}\n# app files") +} diff --git a/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap b/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap index 9cf47bd35..65c59d83a 100755 --- a/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap @@ -150,6 +150,11 @@ "title": "Source files", "description": "Source files to download for this component" }, + "go-vendor": { + "$ref": "#/$defs/GoVendorConfig", + "title": "Go vendoring", + "description": "Convert a Go application spec and generate its vendor archive during source preparation" + }, "packages": { "additionalProperties": { "$ref": "#/$defs/PackageConfig" @@ -682,6 +687,35 @@ "additionalProperties": false, "type": "object" }, + "GoVendorConfig": { + "properties": { + "enabled": { + "type": "boolean", + "title": "Enabled", + "description": "Enable Go application vendoring" + }, + "module-dir": { + "type": "string", + "title": "Module directory", + "description": "Relative module directory within the prepared source tree; defaults to its root" + }, + "tool-config": { + "type": "string", + "title": "Tool configuration", + "description": "Inline go-vendor-tools TOML for dependency overrides and licensing settings" + }, + "prep-packages": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Prep packages", + "description": "Additional RPM packages needed to execute the upstream prep section in mock" + } + }, + "additionalProperties": false, + "type": "object" + }, "ImageCapabilities": { "properties": { "machine-bootable": { diff --git a/schemas/azldev.schema.json b/schemas/azldev.schema.json index 9cf47bd35..65c59d83a 100644 --- a/schemas/azldev.schema.json +++ b/schemas/azldev.schema.json @@ -150,6 +150,11 @@ "title": "Source files", "description": "Source files to download for this component" }, + "go-vendor": { + "$ref": "#/$defs/GoVendorConfig", + "title": "Go vendoring", + "description": "Convert a Go application spec and generate its vendor archive during source preparation" + }, "packages": { "additionalProperties": { "$ref": "#/$defs/PackageConfig" @@ -682,6 +687,35 @@ "additionalProperties": false, "type": "object" }, + "GoVendorConfig": { + "properties": { + "enabled": { + "type": "boolean", + "title": "Enabled", + "description": "Enable Go application vendoring" + }, + "module-dir": { + "type": "string", + "title": "Module directory", + "description": "Relative module directory within the prepared source tree; defaults to its root" + }, + "tool-config": { + "type": "string", + "title": "Tool configuration", + "description": "Inline go-vendor-tools TOML for dependency overrides and licensing settings" + }, + "prep-packages": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Prep packages", + "description": "Additional RPM packages needed to execute the upstream prep section in mock" + } + }, + "additionalProperties": false, + "type": "object" + }, "ImageCapabilities": { "properties": { "machine-bootable": { From c7e4bc2764db6da3a94f3ab71740b7db80fc26a9 Mon Sep 17 00:00:00 2001 From: Antonio Salinas <173939267+Tonisal-byte@users.noreply.github.com> Date: Sun, 1 Nov 2026 11:47:14 -0800 Subject: [PATCH 2/5] feat: regenerate existing Go vendor sources Preserve upstream Source entries, tool configuration, and reviewed licenses when regenerating already-vendored packages. Use a common spec plan and reusable Go-only generation backend. Keep command construction in Go, automatically clone component configuration while retaining shared back-references, and exclude payloads from rendered specs. Add regression coverage and document both workflows. Validation: unit tests, build, and all quality checks passed. All ten prepared containerd files are byte-for-byte identical to the pre-refactor baseline. Full Mage scenarios remain blocked by Docker provider initialization in this WSL environment. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/instructions/go.instructions.md | 3 + docs/user/how-to/vendor-go-application.md | 30 +- .../reference/cli/azldev_component_render.md | 6 +- docs/user/reference/config/components.md | 9 +- .../agentskill/content/comp-toml.md.tmpl | 5 +- .../content/withoutlockfile/comp-toml.md.tmpl | 5 +- internal/app/azldev/cmds/component/render.go | 12 +- .../azldev/cmds/component/render_govendor.go | 53 ++++ .../render_govendor_internal_test.go | 51 +++ internal/app/azldev/core/sources/govendor.go | 300 +++++------------- .../govendor_existing_internal_test.go | 97 ++++++ .../core/sources/govendor_internal_test.go | 25 +- internal/govendor/generate.go | 253 +++++++++++++++ internal/govendor/generate_internal_test.go | 88 +++++ internal/projectconfig/component.go | 37 +-- .../component_clone_internal_test.go | 36 +++ internal/projectconfig/govendor_test.go | 1 + internal/rpm/spec/govendor.go | 74 ++++- internal/rpm/spec/govendor_existing.go | 212 +++++++++++++ internal/rpm/spec/govendor_existing_test.go | 87 +++++ internal/rpm/spec/govendor_test.go | 5 +- 21 files changed, 1104 insertions(+), 285 deletions(-) create mode 100644 internal/app/azldev/cmds/component/render_govendor.go create mode 100644 internal/app/azldev/cmds/component/render_govendor_internal_test.go create mode 100644 internal/app/azldev/core/sources/govendor_existing_internal_test.go create mode 100644 internal/govendor/generate.go create mode 100644 internal/govendor/generate_internal_test.go create mode 100644 internal/projectconfig/component_clone_internal_test.go create mode 100644 internal/rpm/spec/govendor_existing.go create mode 100644 internal/rpm/spec/govendor_existing_test.go diff --git a/.github/instructions/go.instructions.md b/.github/instructions/go.instructions.md index 439a7642a..7c0f3cb4f 100644 --- a/.github/instructions/go.instructions.md +++ b/.github/instructions/go.instructions.md @@ -105,6 +105,9 @@ description: "Instructions for working on the azldev Go codebase. IMPORTANT: Alw ## Component Fingerprinting — `fingerprint:"-"` Tags +New component fields must survive `ComponentConfig.WithAbsolutePaths` without aliasing +mutable data. Its automatic clone deliberately shares only config-file back-references. + Structs in `internal/projectconfig/` are hashed by `hashstructure.Hash()` to detect component changes. Fields **included by default** (safe: false positive > false negative). When adding a new field to a fingerprinted struct, ask: **"Does changing this field change the build output?"** diff --git a/docs/user/how-to/vendor-go-application.md b/docs/user/how-to/vendor-go-application.md index c0b430642..068673213 100644 --- a/docs/user/how-to/vendor-go-application.md +++ b/docs/user/how-to/vendor-go-application.md @@ -3,7 +3,8 @@ Use `go-vendor` to bundle a Go application's module dependencies instead of building against separately packaged `golang(...)` libraries. This is a targeted conversion of the selected upstream spec, not a `go2rpm` wrapper or a newly -generated replacement spec. +generated replacement spec. The same setting regenerates sources for supported +specs that already use go-vendor-tools. ## Enable and prepare @@ -64,17 +65,40 @@ Custom build/install commands must already address that same module. Conditional commands within those sections are retained. Automatic Go development subpackages (`%gopkg`, `%goinstall`, Go file-list macros), -flagged `%gocheck` commands, compound/rich Go BuildRequires, existing vendor -macros, explicit conflicting module-mode settings, and ambiguous/conditional +flagged `%gocheck` commands, compound/rich Go BuildRequires, +explicit conflicting module-mode settings, and ambiguous/conditional target sections are rejected. Use narrowly scoped overlays to adapt these cases; azldev does not silently drop subpackages or rewrite arbitrary shell scripts. +## Regenerate an existing vendor archive + +For an already-vendored spec, azldev recognizes a single +`tar -xf %{S:N}` (or `%{SOURCEN}`) prep command and the configuration source +referenced by `%go_vendor_license_* -c %{S:M}` commands. The Source numbers need +not be 1 and 2. The configuration must name a plain sidecar file. + +The original Source declarations, extraction command, license/file-list macros, +module-mode settings, and custom packaging are retained. The temporary prep spec +omits the vendor Source declaration and its extraction command, so generating +the replacement does not consume the old vendor payload. RPM expands the +upstream archive filename; the regenerated archive replaces that file and its +`sources` checksum rather than adding a second vendor archive. + +When `tool-config` is omitted, the upstream go-vendor-tools configuration is used, +including its reviewed license exceptions. An explicit `tool-config` replaces +that configuration. The detected licenses are verified against the existing +main-package `License:` expression; mismatches fail instead of rewriting it. +The source-fetch phase currently still downloads the upstream vendor archive +alongside the other upstream sources, before regenerating its contents. + ## Review and upgrade Run `azldev component diff-sources -p example` to inspect the conversion, or `azldev component render -p example` to refresh checked-in rendered output. Rendering and source diffs also generate the archive: they need network access and can be substantially slower for enabled components. +Rendered output retains the `sources` checksums but excludes payloads listed in +that manifest; archives remain preparation/build artifacts rather than checked-in files. Review the cumulative license expression and the retained build/install/check commands before building. Automatic detection is not approval of a dependency's diff --git a/docs/user/reference/cli/azldev_component_render.md b/docs/user/reference/cli/azldev_component_render.md index 88a9d5157..239018bd4 100644 --- a/docs/user/reference/cli/azldev_component_render.md +++ b/docs/user/reference/cli/azldev_component_render.md @@ -16,9 +16,9 @@ Within the output directory, components are organized into letter-prefixed subdirectories based on the first character of their name (e.g., specs/c/curl, specs/v/vim). -Unlike prepare-sources, render skips downloading source tarballs from the -lookaside cache — only spec files, patches, scripts, and other git-tracked -sidecar files are included. Multiple components can be rendered at once. +Render normally skips source downloads. Go vendoring downloads and prepares sources for archive generation +and license checks, but excludes payloads from checked-in output. Only specs, patches, scripts, checksums, +and sidecar files are included. Multiple components can be rendered at once. When rendering all components (-a), the --clean-stale flag prunes orphan rendered-spec directories (per-component dirs that no longer correspond to diff --git a/docs/user/reference/config/components.md b/docs/user/reference/config/components.md index 33b7cbf8d..d16fffc7b 100644 --- a/docs/user/reference/config/components.md +++ b/docs/user/reference/config/components.md @@ -182,7 +182,7 @@ enabled = true | `enabled` | boolean | `false` | Enable conversion and vendor archive generation. An explicit `false` overrides inherited enablement. | | `module-dir` | string | `""` | Relative directory containing `go.mod`, resolved from the working directory at the end of the upstream `%prep` section. Empty uses that directory itself. No absolute paths, traversal, or RPM macros. | | `prep-packages` | array of string | `[]` | Additional RPM packages needed by upstream `%prep` commands or the selected license detector. Installed only in the isolated generation chroot. | -| `tool-config` | string | `""` | Inline [go-vendor-tools TOML](https://fedora.gitlab.io/sigs/go/go-vendor-tools/config/), for dependency overrides, archive options, and reviewed license exceptions. Use `module-dir`, not `general.go_mod_dir`. | +| `tool-config` | string | `""` | Inline [go-vendor-tools TOML](https://fedora.gitlab.io/sigs/go/go-vendor-tools/config/), for dependency overrides, archive options, and reviewed license exceptions. When omitted for an already-vendored spec, reuse its upstream tool configuration. An explicit value replaces it. Use `module-dir`, not `general.go_mod_dir`. | The tool configuration is inline so its contents participate in component fingerprinting without checkout-dependent paths. Configured command hooks execute @@ -198,13 +198,18 @@ tool-config = ''' ''' ``` -Preparation emits `-vendor.tar.bz2` and `azldev-go-vendor.toml`, +For previously unbundled specs, preparation emits `-vendor.tar.bz2` and `azldev-go-vendor.toml`, allocating unused `Source` numbers automatically. These names are reserved for generated output; do not also declare them in `source-files` or supply them through overlays. The archive's SHA-512 digest is recorded in the prepared `sources` file. Generation runs afresh on every preparation; there is no filename-based cache. The recorded digest identifies that output, not a separately pinned vendor lock. +For supported specs already using go-vendor-tools, preparation instead regenerates +the existing vendor source, retaining its Source number, RPM-expanded filename, +and configuration sidecar. It updates that archive's `sources` entry in place and +verifies the existing `License:` expression without rewriting it. + All archive/spec/loose-file overlays run before conversion. Source preparation, builds, rendering (including `--check-only`), and source diffs use the same conversion. Enabled components require downloads even with `--skip-sources`, diff --git a/internal/app/azldev/agentskill/content/comp-toml.md.tmpl b/internal/app/azldev/agentskill/content/comp-toml.md.tmpl index 1462c24c3..a6dbf477e 100644 --- a/internal/app/azldev/agentskill/content/comp-toml.md.tmpl +++ b/internal/app/azldev/agentskill/content/comp-toml.md.tmpl @@ -46,7 +46,10 @@ Use `module-dir` for a module below the final prep working directory, `prep-packages` for additional chroot tools, and inline `tool-config` for go-vendor-tools dependency overrides or reviewed license exceptions. Generated `-vendor.tar.bz2` and `azldev-go-vendor.toml` must not also be -declared as `source-files`. Unsupported constructs (notably automatic Go devel +declared as `source-files`. For supported already-vendored specs, regenerate the +existing vendor Source instead, reuse upstream tool configuration unless overridden, +and verify rather than replace the reviewed `License:` expression. +Unsupported constructs (notably automatic Go devel subpackages, flagged `%gocheck`, or ambiguous sections) require explicit overlays; never replace the upstream spec wholesale to bypass a conversion error. diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl index c1f972f5d..96690b936 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl @@ -46,7 +46,10 @@ Use `module-dir` for a module below the final prep working directory, `prep-packages` for additional chroot tools, and inline `tool-config` for go-vendor-tools dependency overrides or reviewed license exceptions. Generated `-vendor.tar.bz2` and `azldev-go-vendor.toml` must not also be -declared as `source-files`. Unsupported constructs (notably automatic Go devel +declared as `source-files`. For supported already-vendored specs, regenerate the +existing vendor Source instead, reuse upstream tool configuration unless overridden, +and verify rather than replace the reviewed `License:` expression. +Unsupported constructs (notably automatic Go devel subpackages, flagged `%gocheck`, or ambiguous sections) require explicit overlays; never replace the upstream spec wholesale to bypass a conversion error. diff --git a/internal/app/azldev/cmds/component/render.go b/internal/app/azldev/cmds/component/render.go index 6f4277743..e9df2be13 100644 --- a/internal/app/azldev/cmds/component/render.go +++ b/internal/app/azldev/cmds/component/render.go @@ -64,9 +64,9 @@ Within the output directory, components are organized into letter-prefixed subdirectories based on the first character of their name (e.g., specs/c/curl, specs/v/vim). -Unlike prepare-sources, render skips downloading source tarballs from the -lookaside cache — only spec files, patches, scripts, and other git-tracked -sidecar files are included. Multiple components can be rendered at once. +Render normally skips source downloads. Go vendoring downloads and prepares sources for archive generation +and license checks, but excludes payloads from checked-in output. Only specs, patches, scripts, checksums, +and sidecar files are included. Multiple components can be rendered at once. When rendering all components (-a), the --clean-stale flag prunes orphan rendered-spec directories (per-component dirs that no longer correspond to @@ -765,6 +765,12 @@ func finishComponentRender( return false, fmt.Errorf("filtering unreferenced files for %#q:\n%w", componentName, filterErr) } + if err := removeGoVendorRenderPayloads( + env.FS(), componentDir, prep.comp.GetConfig().GoVendor.IsEnabled(), + ); err != nil { + return false, fmt.Errorf("removing generated render payloads for %#q:\n%w", componentName, err) + } + // Remove .git directory — must not appear in rendered output. // rpmautospec already read it during the batch mock phase. gitDir := filepath.Join(componentDir, ".git") diff --git a/internal/app/azldev/cmds/component/render_govendor.go b/internal/app/azldev/cmds/component/render_govendor.go new file mode 100644 index 000000000..bcfae2005 --- /dev/null +++ b/internal/app/azldev/cmds/component/render_govendor.go @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component + +import ( + "errors" + "fmt" + "os" + "path/filepath" + + "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx" + "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders/fedorasource" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" +) + +// Vendoring needs real payloads during preparation, but rendered dist-git output +// must contain their manifest rather than copying source archives into the repository. +func removeGoVendorRenderPayloads(fs opctx.FS, directory string, enabled bool) error { + if !enabled { + return nil + } + + contents, err := fileutils.ReadFile(fs, filepath.Join(directory, fedorasource.SourcesFileName)) + if errors.Is(err, os.ErrNotExist) { + return nil + } + + if err != nil { + return fmt.Errorf("reading render source manifest:\n%w", err) + } + + lines, err := fedorasource.ReadSourcesFile(string(contents)) + if err != nil { + return fmt.Errorf("parsing render source manifest:\n%w", err) + } + + for _, line := range lines { + if line.Entry == nil { + continue + } + + if err := fileutils.ValidateFilename(line.Entry.Filename); err != nil { + return fmt.Errorf("invalid render source filename:\n%w", err) + } + + if err := fs.Remove(filepath.Join(directory, line.Entry.Filename)); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("removing render source payload %#q:\n%w", line.Entry.Filename, err) + } + } + + return nil +} diff --git a/internal/app/azldev/cmds/component/render_govendor_internal_test.go b/internal/app/azldev/cmds/component/render_govendor_internal_test.go new file mode 100644 index 000000000..e6b336335 --- /dev/null +++ b/internal/app/azldev/cmds/component/render_govendor_internal_test.go @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component + +import ( + "strings" + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/global/testctx" + "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders/fedorasource" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRemoveGoVendorRenderPayloads(t *testing.T) { + for _, enabled := range []bool{false, true} { + ctx := testctx.NewCtx() + + manifest := "# keep manifest\n" + + fedorasource.FormatSourcesEntry("upstream.tar.gz", fileutils.HashTypeSHA512, strings.Repeat("a", 128)) + "\n" + + fedorasource.FormatSourcesEntry("vendor.tar.bz2", fileutils.HashTypeSHA512, strings.Repeat("b", 128)) + "\n" + for name, content := range map[string]string{ + "app.spec": "spec", "config.toml": "[archive]", "sources": manifest, + "upstream.tar.gz": "source", "vendor.tar.bz2": "vendor", + } { + require.NoError(t, fileutils.WriteFile(ctx.FS(), "/render/"+name, []byte(content), fileperms.PublicFile)) + } + + require.NoError(t, removeGoVendorRenderPayloads(ctx.FS(), "/render", enabled)) + + for _, name := range []string{"upstream.tar.gz", "vendor.tar.bz2"} { + exists, err := fileutils.Exists(ctx.FS(), "/render/"+name) + require.NoError(t, err) + assert.Equal(t, !enabled, exists) + } + + for _, name := range []string{"app.spec", "config.toml", "sources"} { + exists, err := fileutils.Exists(ctx.FS(), "/render/"+name) + require.NoError(t, err) + assert.True(t, exists) + } + + data, err := fileutils.ReadFile(ctx.FS(), "/render/sources") + require.NoError(t, err) + assert.Equal(t, manifest, string(data)) + require.NoError(t, removeGoVendorRenderPayloads(ctx.FS(), "/render", enabled)) + } +} diff --git a/internal/app/azldev/core/sources/govendor.go b/internal/app/azldev/core/sources/govendor.go index c6314e3a8..4d2e42b54 100644 --- a/internal/app/azldev/core/sources/govendor.go +++ b/internal/app/azldev/core/sources/govendor.go @@ -4,37 +4,24 @@ package sources import ( - "bytes" "context" "errors" "fmt" "log/slog" "path/filepath" + "slices" "strings" - "time" - "github.com/kballard/go-shellquote" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" - "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/govendor" "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders/fedorasource" "github.com/microsoft/azure-linux-dev-tools/internal/rpm/mock" "github.com/microsoft/azure-linux-dev-tools/internal/rpm/spec" - "github.com/microsoft/azure-linux-dev-tools/internal/rpm/spectool" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" ) -const ( - goVendorMount = "/azldev-go-vendor" - goVendorTopDir = "/tmp/azldev-go-rpm" - goVendorBuild = goVendorTopDir + "/BUILD" - goVendorConfig = "azldev-go-vendor.toml" - goVendorScratch = "/azldev-go-scratch" - goVendorCleanupTimeout = 5 * time.Minute -) - -// WithGoVendorRunner provides a template runner. Every conversion uses a fresh, -// isolated root; only the component staging directory is exposed to it. +// WithGoVendorRunner provides a template for an isolated generation root. func WithGoVendorRunner(runner *mock.Runner) PreparerOption { return func(p *sourcePreparerImpl) { p.goVendorRunner = runner } } @@ -52,7 +39,7 @@ func (p *sourcePreparerImpl) prepareGoVendor( } if p.dryRunnable.DryRun() { - slog.Info("Dry run; skipping Go vendor conversion and archive generation", "component", component.GetName()) + slog.Info("Dry run; skipping Go vendor generation", "component", component.GetName()) return nil } @@ -61,8 +48,6 @@ func (p *sourcePreparerImpl) prepareGoVendor( return errors.New("go vendoring requires a mock runner with the project's build distro configuration") } - archiveName := component.GetName() + "-vendor.tar.bz2" - specPath, err := p.resolveSpecPath(component, outputDir) if err != nil { return err @@ -73,14 +58,16 @@ func (p *sourcePreparerImpl) prepareGoVendor( return fmt.Errorf("reading Go spec:\n%w", err) } - converted, err := spec.ConvertGoVendor(data, spec.GoVendorOptions{ - ArchiveName: archiveName, ConfigName: goVendorConfig, ModuleDir: config.ModuleDir, + plan, err := spec.ConvertGoVendor(data, spec.GoVendorOptions{ + ArchiveName: component.GetName() + "-vendor.tar.bz2", + ConfigName: govendor.ConfigFilename, ModuleDir: config.ModuleDir, }) if err != nil { - return fmt.Errorf("converting Go spec for %#q:\n%w", component.GetName(), err) + return fmt.Errorf("planning Go vendoring for %#q:\n%w", component.GetName(), err) } - if err := p.checkGoVendorSourceCollision(outputDir, archiveName); err != nil { + toolConfig, err := p.resolveGoVendorConfig(outputDir, plan, config.ToolConfig) + if err != nil { return err } @@ -90,200 +77,76 @@ func (p *sourcePreparerImpl) prepareGoVendor( } defer fileutils.RemoveAllAndUpdateErrorIfNil(p.fs, scratch, &err) - staging := filepath.Join(scratch, "work") - if err := fileutils.MkdirAll(p.fs, staging); err != nil { - return fmt.Errorf("creating Go vendor staging directory:\n%w", err) - } - - sourceCopy := filepath.Join(scratch, "sources") - if err := p.stageGoVendorInputs(outputDir, sourceCopy, staging, filepath.Base(specPath), - converted.PrepSpec, config.ToolConfig); err != nil { - return err - } + slog.Info("Generating Go vendor archive", "spec", filepath.Base(specPath), "regenerating", plan.AllowReplacement) - runner := p.goVendorRunner.Clone().WithBaseDir(filepath.Join(scratch, "mock")). - EnableNetwork().WithUnprivileged() - runner.AddBindMount(sourceCopy, goVendorMount) - runner.AddBindMount(staging, goVendorScratch) - - if err := p.runGoVendor(ctx, runner, archiveName, filepath.Base(specPath), config); err != nil { - return err - } - - if err := p.validateGoVendorInputs(staging, outputDir); err != nil { - return err + artifact, err := govendor.Generate(ctx, p.fs, p.goVendorRunner, govendor.Request{ + SourceDir: outputDir, WorkDir: scratch, SpecName: filepath.Base(specPath), PrepSpec: plan.PrepSpec, + ArchiveFile: plan.ArchiveFile, ArchiveSuffix: plan.ArchiveSuffix, ModuleDir: config.ModuleDir, + ToolConfig: toolConfig, Packages: config.PrepPackages, ExpectedLicense: plan.ExpectedLicense, + }) + if err != nil { + return fmt.Errorf("generating Go vendor sources:\n%w", err) } - return p.publishGoVendor(staging, outputDir, specPath, archiveName, converted.Spec, config.ToolConfig) + return p.publishGoVendor(outputDir, filepath.Base(specPath), toolConfig, plan, artifact) } -func (p *sourcePreparerImpl) stageGoVendorInputs( - outputDir, sourceCopy, staging, specFilename string, prepSpec []byte, toolConfig string, -) error { - if err := fileutils.CopyDirRecursive(p.dryRunnable, p.fs, outputDir, sourceCopy, - fileutils.CopyDirOptions{CopyFileOptions: fileutils.CopyFileOptions{PreserveFileMode: true}}); err != nil { - return fmt.Errorf("copying Go sources for isolated prep:\n%w", err) +func (p *sourcePreparerImpl) resolveGoVendorConfig( + outputDir string, plan *spec.GoVendorSpec, configured string, +) (string, error) { + if !plan.AllowReplacement { + if err := p.checkGoVendorSourceCollision(outputDir, plan.ArchiveFile, plan.ConfigName); err != nil { + return "", err + } } - if err := fileutils.WriteFile(p.fs, - filepath.Join(sourceCopy, specFilename), prepSpec, fileperms.PublicFile); err != nil { - return fmt.Errorf("staging Go prep spec:\n%w", err) + if configured != "" || !plan.AllowReplacement { + return configured, nil } - if err := fileutils.WriteFile(p.fs, filepath.Join(staging, goVendorConfig), - []byte(toolConfig), fileperms.PublicFile); err != nil { - return fmt.Errorf("staging Go vendor tool configuration:\n%w", err) + data, err := fileutils.ReadFile(p.fs, filepath.Join(outputDir, plan.ConfigName)) + if err != nil { + return "", fmt.Errorf("reading upstream Go vendor configuration:\n%w", err) } - return nil + return string(data), nil } func (p *sourcePreparerImpl) publishGoVendor( - staging, outputDir, specPath, archiveName string, data []byte, toolConfig string, + outputDir, specName, toolConfig string, plan *spec.GoVendorSpec, artifact govendor.Artifact, ) error { - archiveInfo, err := p.fs.Stat(filepath.Join(staging, archiveName)) - if err != nil { - return fmt.Errorf("missing generated Go vendor archive:\n%w", err) - } - - if !archiveInfo.Mode().IsRegular() || archiveInfo.Size() == 0 { - return errors.New("generated Go vendor archive must be a nonempty regular file") - } - - license, err := fileutils.ReadFile(p.fs, filepath.Join(staging, "license")) - if err != nil { - return fmt.Errorf("reading Go vendor license report:\n%w", err) - } - - expression := strings.TrimSpace(string(license)) - if expression == "" || strings.ContainsAny(expression, "\r\n%") { - return fmt.Errorf("invalid license expression returned by 'go_vendor_license': %#q", expression) + if !plan.AllowReplacement && artifact.Name != plan.ArchiveFile { + return fmt.Errorf("generated Go archive %#q does not match planned name %#q", artifact.Name, plan.ArchiveFile) } - editor, err := spec.OpenSpec(bytes.NewReader(data), spec.WithEditor(spec.EditorStructural)) + finalSpec, err := plan.FinalizeLicense(artifact.License) if err != nil { - return fmt.Errorf("reading converted Go spec:\n%w", err) - } - - if err := editor.UpdateExistingTag("", "License", expression); err != nil { - return fmt.Errorf("updating bundled Go licenses:\n%w", err) + return fmt.Errorf("finalizing Go vendor spec:\n%w", err) } - var finalSpec bytes.Buffer - if err := editor.Serialize(&finalSpec); err != nil { - return fmt.Errorf("serializing converted Go spec:\n%w", err) - } - // Publish only after generation and license detection succeed. - if err := fileutils.CopyFile(p.dryRunnable, p.fs, - filepath.Join(staging, archiveName), filepath.Join(outputDir, archiveName), fileutils.CopyFileOptions{}); err != nil { + if err := fileutils.CopyFile(p.dryRunnable, p.fs, artifact.Path, + filepath.Join(outputDir, artifact.Name), fileutils.CopyFileOptions{}); err != nil { return fmt.Errorf("publishing Go vendor archive:\n%w", err) } - for name, contents := range map[string][]byte{ - filepath.Base(specPath): finalSpec.Bytes(), - goVendorConfig: []byte(toolConfig), - } { + for name, contents := range map[string][]byte{specName: finalSpec, plan.ConfigName: []byte(toolConfig)} { if err := fileutils.WriteFile(p.fs, filepath.Join(outputDir, name), contents, fileperms.PublicFile); err != nil { return fmt.Errorf("publishing Go vendor file %#q:\n%w", name, err) } } - return p.recordGoVendorArchive(outputDir, archiveName) -} - -func (p *sourcePreparerImpl) runGoVendor( - ctx context.Context, runner *mock.Runner, archiveName, specFilename string, config *projectconfig.GoVendorConfig, -) (err error) { - defer func() { - cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), goVendorCleanupTimeout) - defer cancel() - - if scrubErr := runner.ScrubRoot(cleanupCtx); scrubErr != nil { - slog.Error("Failed to clean Go vendor mock root", "error", scrubErr) - - if err == nil { - err = scrubErr - } - } - }() - - if err := runner.InitRoot(ctx); err != nil { - return fmt.Errorf("initializing Go vendor mock root:\n%w", err) - } - - packages := append([]string{ - "go-vendor-tools", "askalono-cli", "golang", "go-rpm-macros", "rpm-build", "rpmdevtools", - "rpmautospec", "git", "python3-click", - }, config.PrepPackages...) - if err := runner.InstallPackages(ctx, packages); err != nil { - return fmt.Errorf("installing Go vendor prerequisites:\n%w", err) - } - - cmd, err := runner.CmdInChroot(ctx, []string{ - "bash", "-euo", "pipefail", "-c", - goVendorScript(archiveName, config.ModuleDir, runner.ConfigOpts()["target_arch"], specFilename), - }, false) - if err != nil { - return fmt.Errorf("constructing Go vendor command:\n%w", err) - } - - output, err := cmd.RunAndGetOutput(ctx) - if err != nil { - return fmt.Errorf("go vendor generation or license detection failed; use 'prep-packages' for missing prep tools "+ - "and 'tool-config' for reviewed license exceptions:\n%s\n%w", output, err) - } - - return nil -} - -func goVendorScript(archiveName, moduleDir, targetArch, specFilename string) string { - if moduleDir == "" { - moduleDir = "." - } - - quote := shellquote.Join - - prepArgs := []string{ - "rpmbuild", "-bp", "--nodeps", "--define", "_topdir /tmp/azldev-go-rpm", - "--define", "_builddir " + goVendorBuild, "--define", "_sourcedir " + goVendorMount, - } - if targetArch != "" { - prepArgs = append(prepArgs, "--target", targetArch) - } - - prepArgs = append(prepArgs, goVendorMount+"/"+specFilename) - - return strings.Join([]string{ - "cd " + goVendorMount, - "mkdir -p " + goVendorBuild, - quote("git", "config", "--global", "--add", "safe.directory", goVendorMount), - quote("rpmautospec", "process-distgit", specFilename, specFilename), - quote(prepArgs...), - `IFS= read -r module_root < ` + goVendorTopDir + "/azldev-go-module-root", - `cd "$module_root"`, - "cd " + quote(moduleDir), - "test -f go.mod", - "export GOTOOLCHAIN=local", - quote("go_vendor_archive", "create", "--config", goVendorScratch+"/"+goVendorConfig, - "--output", goVendorScratch+"/"+archiveName, "."), - quote("tar", "-xf", goVendorScratch+"/"+archiveName), - quote("go_vendor_license", "--config", goVendorScratch+"/"+goVendorConfig, "--path", ".", - "report", "--no-prompt", "--autofill", "off", "expression") + " > " + goVendorScratch + "/license", - "cd " + goVendorMount, - quote("spectool", "--define", "_sourcedir "+goVendorMount, "-l", "-a", - specFilename) + " > " + goVendorScratch + "/sources-list", - }, "\n") + return p.recordGoVendorArchive(outputDir, artifact, plan.AllowReplacement) } -func (p *sourcePreparerImpl) checkGoVendorSourceCollision(outputDir, name string) error { - for _, filename := range []string{name, goVendorConfig} { - exists, err := fileutils.Exists(p.fs, filepath.Join(outputDir, filename)) +func (p *sourcePreparerImpl) checkGoVendorSourceCollision(outputDir string, names ...string) error { + for _, name := range names { + exists, err := fileutils.Exists(p.fs, filepath.Join(outputDir, name)) if err != nil { return fmt.Errorf("checking generated Go source path:\n%w", err) } if exists { - return fmt.Errorf("generated Go vendor file %#q conflicts with an existing source", filename) + return fmt.Errorf("generated Go vendor file %#q conflicts with an existing source", name) } } @@ -298,36 +161,34 @@ func (p *sourcePreparerImpl) checkGoVendorSourceCollision(outputDir, name string } for _, line := range lines { - if line.Entry != nil && line.Entry.Filename == name { - return fmt.Errorf("generated Go vendor archive %#q conflicts with an upstream 'sources' entry", name) + if line.Entry != nil && slices.Contains(names, line.Entry.Filename) { + return fmt.Errorf("generated Go vendor file %#q conflicts with an upstream 'sources' entry", line.Entry.Filename) } } return nil } -func (p *sourcePreparerImpl) recordGoVendorArchive(outputDir, name string) error { - hash, err := fileutils.ComputeFileHash(p.fs, fileutils.HashTypeSHA512, filepath.Join(outputDir, name)) - if err != nil { - return fmt.Errorf("hashing generated Go vendor archive:\n%w", err) - } +func (p *sourcePreparerImpl) recordGoVendorArchive(outputDir string, artifact govendor.Artifact, replace bool) error { + path := filepath.Join(outputDir, fedorasource.SourcesFileName) - content, err := p.readSourcesFileIfExists(filepath.Join(outputDir, fedorasource.SourcesFileName)) + content, err := p.readSourcesFileIfExists(path) if err != nil { return err } - content = strings.TrimRight(content, "\n") - if content != "" { - content += "\n" + content, err = upsertGoVendorEntry(content, artifact, replace) + if err != nil { + return err } - content += fedorasource.FormatSourcesEntry(name, fileutils.HashTypeSHA512, hash) + "\n" - if err := fileutils.WriteFile(p.fs, filepath.Join(outputDir, fedorasource.SourcesFileName), - []byte(content), fileperms.PublicFile); err != nil { + if err := fileutils.WriteFile(p.fs, path, []byte(content), fileperms.PublicFile); err != nil { return fmt.Errorf("recording Go vendor digest:\n%w", err) } - // Generated payloads belong in lookaside storage, not synthetic git commits. + + slog.Info("Recorded generated Go vendor archive", "filename", artifact.Name, + "hashType", fileutils.HashTypeSHA512, "hash", artifact.Digest) + ignorePath := filepath.Join(outputDir, ".gitignore") ignore, err := p.readSourcesFileIfExists(ignorePath) @@ -335,39 +196,46 @@ func (p *sourcePreparerImpl) recordGoVendorArchive(outputDir, name string) error return err } + for _, line := range strings.Split(ignore, "\n") { + if strings.TrimPrefix(line, "/") == artifact.Name { + return nil + } + } + if err := fileutils.WriteFile(p.fs, ignorePath, - []byte(strings.TrimRight(ignore, "\n")+"\n/"+name+"\n"), fileperms.PublicFile); err != nil { + []byte(strings.TrimRight(ignore, "\n")+"\n/"+artifact.Name+"\n"), fileperms.PublicFile); err != nil { return fmt.Errorf("ignoring generated Go archive in git:\n%w", err) } return nil } -func (p *sourcePreparerImpl) validateGoVendorInputs(scratch, outputDir string) error { - data, err := fileutils.ReadFile(p.fs, filepath.Join(scratch, "sources-list")) +func upsertGoVendorEntry(content string, artifact govendor.Artifact, replace bool) (string, error) { + lines, err := fedorasource.ReadSourcesFile(strings.TrimRight(content, "\n")) if err != nil { - return fmt.Errorf("reading prepared Go source list:\n%w", err) + return "", fmt.Errorf("parsing Go vendor source manifest:\n%w", err) } - for _, line := range strings.Split(strings.TrimSpace(string(data)), "\n") { - if line == "" { - continue - } + entry := fedorasource.FormatSourcesEntry(artifact.Name, fileutils.HashTypeSHA512, artifact.Digest) + raw := make([]string, 0, len(lines)+1) + found := false - files := spectool.ParseSpectoolOutput(line) - if len(files) != 1 || strings.Contains(files[0], "%") { - return fmt.Errorf("cannot resolve Go source/patch reference %#q", line) - } + for _, line := range lines { + value := line.Raw + if line.Entry != nil && line.Entry.Filename == artifact.Name { + if !replace { + return "", fmt.Errorf("go vendor source %#q already exists", artifact.Name) + } - info, err := p.fs.Stat(filepath.Join(outputDir, files[0])) - if err != nil { - return fmt.Errorf("missing prepared Go source/patch %#q:\n%w", files[0], err) + value, found = entry, true } - if !info.Mode().IsRegular() { - return fmt.Errorf("go source/patch %#q is not a regular file", files[0]) - } + raw = append(raw, value) } - return nil + if !found { + raw = append(raw, entry) + } + + return strings.Join(raw, "\n") + "\n", nil } diff --git a/internal/app/azldev/core/sources/govendor_existing_internal_test.go b/internal/app/azldev/core/sources/govendor_existing_internal_test.go new file mode 100644 index 000000000..b9d47d92b --- /dev/null +++ b/internal/app/azldev/core/sources/govendor_existing_internal_test.go @@ -0,0 +1,97 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package sources + +import ( + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components/components_testutils" + "github.com/microsoft/azure-linux-dev-tools/internal/govendor" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders/fedorasource" + "github.com/microsoft/azure-linux-dev-tools/internal/rpm/mock" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/samber/lo" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +func TestPrepareExistingGoVendorReplacesPayload(t *testing.T) { + ctx := newMockProcessorTestCtx() + component := components_testutils.NewMockComponent(gomock.NewController(t)) + component.EXPECT().GetName().AnyTimes().Return("app") + component.EXPECT().GetConfig().AnyTimes().Return(&projectconfig.ComponentConfig{ + GoVendor: &projectconfig.GoVendorConfig{Enabled: lo.ToPtr(true)}, + }) + + input := strings.Join([]string{ + "Name: app", "Version: 1.2", "Release: 1", "License: MIT AND Apache-2.0", + "Source0: app.tar.gz", "Source1: app-%{version}-vendor.tar.bz2", "Source2: go-vendor-tools.toml", + "%description", "app", "%prep", "%goprep", "tar -xf %{S:1}", + "%build", "%global gomodulesmode GO111MODULE=on", "%gobuild -o app .", + "%install", "%go_vendor_license_install -c %{S:2}", + "%files -f %{go_vendor_license_filelist}", "/usr/bin/app", "", + }, "\n") + + const ( + archiveName = "app-1.2-vendor.tar.bz2" + upstreamConfig = "[licensing]\ndetector = 'askalono'\n" + ) + + oldEntry := fedorasource.FormatSourcesEntry(archiveName, fileutils.HashTypeSHA512, strings.Repeat("a", 128)) + for name, data := range map[string]string{ + "app.spec": input, "app.tar.gz": "upstream source", archiveName: "old vendor", + "go-vendor-tools.toml": upstreamConfig, "sources": oldEntry + "\n", ".gitignore": "/" + archiveName + "\n", + } { + require.NoError(t, fileutils.WriteFile(ctx.FS(), "/output/"+name, []byte(data), fileperms.PublicFile)) + } + + ctx.CmdFactory.RunAndGetOutputHandler = func(command *exec.Cmd) (string, error) { + assert.Contains(t, strings.Join(command.Args, " "), "--verify") + assert.Contains(t, strings.Join(command.Args, " "), "MIT AND Apache-2.0") + staging := goVendorTestStaging(t, ctx) + data, err := fileutils.ReadFile(ctx.FS(), filepath.Join(staging, govendor.ConfigFilename)) + require.NoError(t, err) + assert.Equal(t, upstreamConfig, string(data)) + + prep, err := fileutils.ReadFile(ctx.FS(), filepath.Join(filepath.Dir(staging), "sources", "app.spec")) + require.NoError(t, err) + assert.NotContains(t, string(prep), "Source1:") + assert.NotContains(t, string(prep), "tar -xf %{S:1}") + + for name, data := range map[string]string{ + "azldev-generated-vendor.tar.bz2": "new vendor", "archive-name": archiveName + "\n", + "license": "Apache-2.0 AND MIT\n", "sources-list": "Source0: app.tar.gz\nSource2: go-vendor-tools.toml\n", + } { + require.NoError(t, fileutils.WriteFile(ctx.FS(), filepath.Join(staging, name), []byte(data), fileperms.PublicFile)) + } + + return "", nil + } + preparer := &sourcePreparerImpl{fs: ctx.FS(), dryRunnable: ctx, goVendorRunner: mock.NewRunner(ctx, "/mock.cfg")} + require.NoError(t, preparer.prepareGoVendor(ctx, component, "/output")) + payload, err := fileutils.ReadFile(ctx.FS(), "/output/"+archiveName) + require.NoError(t, err) + assert.Equal(t, "new vendor", string(payload)) + + finalSpec, err := fileutils.ReadFile(ctx.FS(), "/output/app.spec") + require.NoError(t, err) + assert.Contains(t, string(finalSpec), "Source1: app-%{version}-vendor.tar.bz2") + assert.Contains(t, string(finalSpec), "License: MIT AND Apache-2.0") + assert.NotContains(t, string(finalSpec), "azldev-go-vendor.toml") + + sources, err := fileutils.ReadFile(ctx.FS(), "/output/sources") + require.NoError(t, err) + digest, err := fileutils.ComputeFileHash(ctx.FS(), fileutils.HashTypeSHA512, "/output/"+archiveName) + require.NoError(t, err) + assert.Equal(t, fedorasource.FormatSourcesEntry(archiveName, fileutils.HashTypeSHA512, digest)+"\n", string(sources)) + ignore, err := fileutils.ReadFile(ctx.FS(), "/output/.gitignore") + require.NoError(t, err) + assert.Equal(t, "/"+archiveName+"\n", string(ignore)) +} diff --git a/internal/app/azldev/core/sources/govendor_internal_test.go b/internal/app/azldev/core/sources/govendor_internal_test.go index d79bbf4a5..6509507b2 100644 --- a/internal/app/azldev/core/sources/govendor_internal_test.go +++ b/internal/app/azldev/core/sources/govendor_internal_test.go @@ -14,6 +14,7 @@ import ( "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components/components_testutils" "github.com/microsoft/azure-linux-dev-tools/internal/global/testctx" + "github.com/microsoft/azure-linux-dev-tools/internal/govendor" "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders" "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders/fedorasource" @@ -83,6 +84,7 @@ func TestPrepareSourcesGoVendor(t *testing.T) { for name, content := range map[string]string{ "app-vendor.tar.bz2": "generated archive", + "archive-name": "app-vendor.tar.bz2\n", "license": "Apache-2.0 AND MIT\n", "sources-list": "Source0: app.tar.gz\nPatch0: fix.patch\n", } { @@ -144,7 +146,7 @@ func TestGoVendorFailuresDoNotPublishSpec(t *testing.T) { for _, stage := range []string{ "init", "install", "generate", "missing-archive", "empty-archive", - "license", "missing-source", "source-list", "scrub", + "license", "missing-source", "source-list", "scrub", "archive-name", "unexpected-name", } { t.Run(stage, func(t *testing.T) { ctx := newMockProcessorTestCtx() @@ -172,6 +174,7 @@ func TestGoVendorFailuresDoNotPublishSpec(t *testing.T) { staging := goVendorTestStaging(t, ctx) files := map[string]string{ "app-vendor.tar.bz2": "archive", "license": "MIT\n", "sources-list": "Source0: app.tar.gz\n", + "archive-name": "app-vendor.tar.bz2\n", } switch stage { @@ -185,6 +188,10 @@ func TestGoVendorFailuresDoNotPublishSpec(t *testing.T) { files["sources-list"] += "Patch0: absent.patch\n" case "source-list": files["sources-list"] = "Source0: /etc/passwd\n" + case "archive-name": + files["archive-name"] = "../outside.tar.bz2\n" + case "unexpected-name": + files["archive-name"] = "other.tar.bz2\n" } for name, data := range files { @@ -208,20 +215,8 @@ func TestGoVendorFailuresDoNotPublishSpec(t *testing.T) { } } -func TestGoVendorScriptOrdering(t *testing.T) { - script := goVendorScript("app-vendor.tar.bz2", "cmd/app", "aarch64", "app.spec") - assert.Less(t, strings.Index(script, "rpmautospec"), strings.Index(script, "rpmbuild")) - assert.Less(t, strings.Index(script, "rpmbuild"), strings.Index(script, "go_vendor_archive")) - assert.Less(t, strings.Index(script, "go_vendor_archive"), strings.Index(script, "go_vendor_license")) - assert.Contains(t, script, "cd cmd/app") - assert.Contains(t, script, "--target aarch64") - assert.Contains(t, script, "--no-prompt") - assert.Contains(t, script, "--autofill off") - assert.NotContains(t, script, "--idempotent") -} - func TestGoVendorSourceCollisions(t *testing.T) { - for _, name := range []string{"app-vendor.tar.bz2", goVendorConfig, "sources", "invalid-sources"} { + for _, name := range []string{"app-vendor.tar.bz2", govendor.ConfigFilename, "sources", "invalid-sources"} { t.Run(name, func(t *testing.T) { ctx := testctx.NewCtx() preparer := &sourcePreparerImpl{fs: ctx.FS()} @@ -238,7 +233,7 @@ func TestGoVendorSourceCollisions(t *testing.T) { } require.NoError(t, fileutils.WriteFile(ctx.FS(), "/output/"+filename, []byte(content), fileperms.PublicFile)) - require.Error(t, preparer.checkGoVendorSourceCollision("/output", "app-vendor.tar.bz2")) + require.Error(t, preparer.checkGoVendorSourceCollision("/output", "app-vendor.tar.bz2", govendor.ConfigFilename)) }) } } diff --git a/internal/govendor/generate.go b/internal/govendor/generate.go new file mode 100644 index 000000000..0f888ddfe --- /dev/null +++ b/internal/govendor/generate.go @@ -0,0 +1,253 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// Package govendor generates vendor artifacts without resolving components or editing specs. +package govendor + +import ( + "cmp" + "context" + "errors" + "fmt" + "log/slog" + "path/filepath" + "strings" + "time" + + "github.com/kballard/go-shellquote" + "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx" + "github.com/microsoft/azure-linux-dev-tools/internal/rpm/mock" + "github.com/microsoft/azure-linux-dev-tools/internal/rpm/spectool" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" +) + +const ( + // ConfigFilename is the tool configuration filename inside generation staging. + ConfigFilename = "azldev-go-vendor.toml" + sourceMount = "/azldev-go-vendor" + outputMount = "/azldev-go-scratch" + topDir = "/tmp/azldev-go-rpm" + cleanupTimeout = 5 * time.Minute +) + +// Request supplies prepared spec bytes and input files. WorkDir must be a private +// scratch directory outside SourceDir; the caller owns its cleanup and dry-run policy. +// PrepSpec must record 'azldev-go-module-root' and 'azldev-go-archive-name' in RPM's topdir. +type Request struct { + SourceDir, WorkDir, SpecName string + ArchiveFile, ArchiveSuffix string + ModuleDir, ToolConfig string + ExpectedLicense string + PrepSpec []byte + Packages []string +} + +// Artifact is validated generation output, ready for publication by the caller. +type Artifact struct { + Path, Name, License, Digest string +} + +// Generate prepares inputs and generates an archive in one isolated mock root. +// It never changes SourceDir or publishes a spec or source manifest. +func Generate(ctx context.Context, fs opctx.FS, base *mock.Runner, request Request) (Artifact, error) { + staging := filepath.Join(request.WorkDir, "work") + + sourceCopy := filepath.Join(request.WorkDir, "sources") + if err := stage(fs, request, sourceCopy, staging); err != nil { + return Artifact{}, err + } + + runner := base.Clone().WithBaseDir(filepath.Join(request.WorkDir, "mock")).EnableNetwork().WithUnprivileged() + runner.AddBindMount(sourceCopy, sourceMount).AddBindMount(staging, outputMount) + + if err := execute(ctx, runner, request); err != nil { + return Artifact{}, err + } + + if err := validateInputs(fs, staging, request.SourceDir); err != nil { + return Artifact{}, err + } + + return readArtifact(fs, staging, request) +} + +func stage(fs opctx.FS, request Request, sourceCopy, staging string) error { + for _, name := range []string{request.SpecName, request.ArchiveFile} { + if err := fileutils.ValidateFilename(name); err != nil { + return fmt.Errorf("invalid Go vendor staging filename:\n%w", err) + } + } + + if err := fileutils.MkdirAll(fs, staging); err != nil { + return fmt.Errorf("creating Go vendor staging directory:\n%w", err) + } + + if err := fileutils.CopyDirRecursiveCrossFS(fs, request.SourceDir, fs, sourceCopy, + fileutils.CopyDirOptions{CopyFileOptions: fileutils.CopyFileOptions{PreserveFileMode: true}}); err != nil { + return fmt.Errorf("copying Go sources for isolated prep:\n%w", err) + } + + for path, data := range map[string][]byte{ + filepath.Join(sourceCopy, request.SpecName): request.PrepSpec, + filepath.Join(staging, ConfigFilename): []byte(request.ToolConfig), + } { + if err := fileutils.WriteFile(fs, path, data, fileperms.PublicFile); err != nil { + return fmt.Errorf("staging Go vendor input %#q:\n%w", path, err) + } + } + + return nil +} + +func execute(ctx context.Context, runner *mock.Runner, request Request) (err error) { + defer func() { + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), cleanupTimeout) + defer cancel() + + if cleanupErr := runner.ScrubRoot(cleanupCtx); cleanupErr != nil { + slog.Error("Failed to clean Go vendor mock root", "error", cleanupErr) + + if err == nil { + err = cleanupErr + } + } + }() + + if err := runner.InitRoot(ctx); err != nil { + return fmt.Errorf("initializing Go vendor mock root:\n%w", err) + } + + packages := append([]string{ + "go-vendor-tools", "askalono-cli", "golang", "go-rpm-macros", "rpm-build", "rpmdevtools", + "rpmautospec", "git", "python3-click", + }, request.Packages...) + if err := runner.InstallPackages(ctx, packages); err != nil { + return fmt.Errorf("installing Go vendor prerequisites:\n%w", err) + } + + cmd, err := runner.CmdInChroot(ctx, generationCommand(request, runner.ConfigOpts()["target_arch"]), false) + if err != nil { + return fmt.Errorf("constructing Go vendor command:\n%w", err) + } + + output, err := cmd.RunAndGetOutput(ctx) + if err != nil { + return fmt.Errorf("go vendor generation or license detection failed; use 'prep-packages' for missing prep tools "+ + "and 'tool-config' for reviewed license exceptions:\n%s\n%w", output, err) + } + + return nil +} + +func generationCommand(request Request, targetArch string) []string { + quote := shellquote.Join + + prepArgs := []string{ + "rpmbuild", "-bp", "--nodeps", "--define", "_topdir " + topDir, + "--define", "_builddir " + topDir + "/BUILD", "--define", "_sourcedir " + sourceMount, + } + if targetArch != "" { + prepArgs = append(prepArgs, "--target", targetArch) + } + + licenseArgs := []string{ + "go_vendor_license", "--config", outputMount + "/" + ConfigFilename, "--path", ".", + "report", "--no-prompt", "--autofill", "off", "expression", + } + if request.ExpectedLicense != "" { + licenseArgs = append(licenseArgs, "--verify", request.ExpectedLicense) + } + + commands := []string{ + "cd " + sourceMount, + "mkdir -p " + topDir + "/BUILD", + quote("git", "config", "--global", "--add", "safe.directory", sourceMount), + quote("rpmautospec", "process-distgit", request.SpecName, request.SpecName), + quote(append(prepArgs, sourceMount+"/"+request.SpecName)...), + quote("cp", topDir+"/azldev-go-archive-name", outputMount+"/archive-name"), + `IFS= read -r module_root < ` + topDir + "/azldev-go-module-root", + `cd "$module_root"`, + "cd " + quote(cmp.Or(request.ModuleDir, ".")), + "test -f go.mod", + "export GOTOOLCHAIN=local", + quote("go_vendor_archive", "create", "--config", outputMount+"/"+ConfigFilename, + "--output", outputMount+"/"+request.ArchiveFile, "."), + quote("tar", "-xf", outputMount+"/"+request.ArchiveFile), + quote(licenseArgs...) + " > " + outputMount + "/license", + "cd " + sourceMount, + quote("spectool", "--define", "_sourcedir "+sourceMount, "-l", "-a", request.SpecName) + + " > " + outputMount + "/sources-list", + } + + return []string{"bash", "-euo", "pipefail", "-c", strings.Join(commands, "\n")} +} + +func readArtifact(fs opctx.FS, staging string, request Request) (Artifact, error) { + result := Artifact{Path: filepath.Join(staging, request.ArchiveFile)} + + info, err := fs.Stat(result.Path) + if err != nil { + return result, fmt.Errorf("missing generated Go vendor archive:\n%w", err) + } + + if !info.Mode().IsRegular() || info.Size() == 0 { + return result, errors.New("generated Go vendor archive must be a nonempty regular file") + } + + for name, destination := range map[string]*string{"archive-name": &result.Name, "license": &result.License} { + data, err := fileutils.ReadFile(fs, filepath.Join(staging, name)) + if err != nil { + return result, fmt.Errorf("reading Go vendor output %#q:\n%w", name, err) + } + + *destination = strings.TrimSpace(string(data)) + if *destination == "" || strings.ContainsAny(*destination, "%\r\n") { + return result, fmt.Errorf("invalid Go vendor output %#q: %#q", name, *destination) + } + } + + if err := fileutils.ValidateFilename(result.Name); err != nil { + return result, fmt.Errorf("invalid expanded Go vendor archive name:\n%w", err) + } + + if !strings.HasSuffix(result.Name, request.ArchiveSuffix) { + return result, fmt.Errorf("unexpected expanded Go vendor archive name %#q", result.Name) + } + + result.Digest, err = fileutils.ComputeFileHash(fs, fileutils.HashTypeSHA512, result.Path) + if err != nil { + return result, fmt.Errorf("hashing generated Go vendor archive:\n%w", err) + } + + return result, nil +} + +func validateInputs(fs opctx.FS, staging, sourceDir string) error { + data, err := fileutils.ReadFile(fs, filepath.Join(staging, "sources-list")) + if err != nil { + return fmt.Errorf("reading prepared Go source list:\n%w", err) + } + + for _, line := range strings.Split(strings.TrimSpace(string(data)), "\n") { + if line == "" { + continue + } + + files := spectool.ParseSpectoolOutput(line) + if len(files) != 1 || strings.Contains(files[0], "%") { + return fmt.Errorf("cannot resolve Go source/patch reference %#q", line) + } + + info, err := fs.Stat(filepath.Join(sourceDir, files[0])) + if err != nil { + return fmt.Errorf("missing prepared Go source/patch %#q:\n%w", files[0], err) + } + + if !info.Mode().IsRegular() { + return fmt.Errorf("go source/patch %#q is not a regular file", files[0]) + } + } + + return nil +} diff --git a/internal/govendor/generate_internal_test.go b/internal/govendor/generate_internal_test.go new file mode 100644 index 000000000..68df88413 --- /dev/null +++ b/internal/govendor/generate_internal_test.go @@ -0,0 +1,88 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package govendor + +import ( + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/global/testctx" + "github.com/microsoft/azure-linux-dev-tools/internal/rpm/mock" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGenerateArtifact(t *testing.T) { + for _, invalid := range []string{"", "archive-name", "license", "sources-list"} { + t.Run("invalid-"+invalid, func(t *testing.T) { + ctx := testctx.NewCtx() + ctx.CmdFactory.RegisterCommandInSearchPath(mock.MockBinary) + require.NoError(t, fileutils.WriteFile(ctx.FS(), "/input/app.spec", []byte("original"), fileperms.PublicFile)) + request := Request{ + SourceDir: "/input", WorkDir: "/scratch", SpecName: "app.spec", + ArchiveFile: "vendor.tar.bz2", ArchiveSuffix: ".tar.bz2", PrepSpec: []byte("prepared"), + } + ctx.CmdFactory.RunAndGetOutputHandler = func(command *exec.Cmd) (string, error) { + assert.Contains(t, command.Args, "--unpriv") + assert.Contains(t, command.Args, "--enable-network") + + for name, content := range map[string]string{ + request.ArchiveFile: "archive", "archive-name": "app-vendor.tar.bz2\n", + "license": "MIT\n", "sources-list": "Source0: app.spec\n", + } { + if name == invalid { + content = "%{unexpanded}" + } + + require.NoError(t, fileutils.WriteFile(ctx.FS(), + filepath.Join("/scratch/work", name), []byte(content), fileperms.PublicFile)) + } + + return "", nil + } + + artifact, err := Generate(ctx, ctx.FS(), mock.NewRunner(ctx, "/mock.cfg"), request) + if invalid != "" { + require.Error(t, err) + + return + } + + require.NoError(t, err) + assert.Equal(t, "app-vendor.tar.bz2", artifact.Name) + assert.Equal(t, "MIT", artifact.License) + digest, err := fileutils.ComputeFileHash(ctx.FS(), fileutils.HashTypeSHA512, artifact.Path) + require.NoError(t, err) + assert.Equal(t, digest, artifact.Digest) + + original, err := fileutils.ReadFile(ctx.FS(), "/input/app.spec") + require.NoError(t, err) + assert.Equal(t, "original", string(original)) + }) + } +} + +func TestGenerationCommand(t *testing.T) { + for _, expected := range []string{"", "MIT AND Apache-2.0"} { + command := generationCommand(Request{ + ArchiveFile: "app-vendor.tar.bz2", ModuleDir: "cmd/app", SpecName: "app.spec", ExpectedLicense: expected, + }, "aarch64") + assert.Equal(t, []string{"bash", "-euo", "pipefail", "-c"}, command[:4]) + script := command[4] + assert.Less(t, strings.Index(script, "rpmautospec"), strings.Index(script, "rpmbuild")) + assert.Less(t, strings.Index(script, "rpmbuild"), strings.Index(script, "go_vendor_archive")) + assert.Less(t, strings.Index(script, "go_vendor_archive"), strings.Index(script, "go_vendor_license")) + assert.Contains(t, script, "cd cmd/app") + assert.Contains(t, script, "--target aarch64") + assert.Contains(t, script, "--no-prompt") + assert.Contains(t, script, "--autofill off") + assert.NotContains(t, script, "--idempotent") + assert.Contains(t, script, "cp "+topDir+"/azldev-go-archive-name "+outputMount+"/archive-name") + assert.Equal(t, expected != "", strings.Contains(script, "--verify")) + } +} diff --git a/internal/projectconfig/component.go b/internal/projectconfig/component.go index cf753e065..d3b2404aa 100644 --- a/internal/projectconfig/component.go +++ b/internal/projectconfig/component.go @@ -644,28 +644,12 @@ func ResolveComponentConfig( // Returns a copy of the component config with relative file paths converted to absolute // file paths (relative to referenceDir, not the current working directory). func (c *ComponentConfig) WithAbsolutePaths(referenceDir string) *ComponentConfig { - // Deep copy the input to avoid unexpected sharing. Make sure *not* to deep-copy - // the SourceConfigFile, as we *do* want to alias that pointer, sharing it across - // all configs that came from that source config file. - result := &ComponentConfig{ - Name: c.Name, - SourceConfigFile: c.SourceConfigFile, - upstreamCommitConfigFile: c.upstreamCommitConfigFile, - RenderedSpecDir: c.RenderedSpecDir, - Locked: deep.MustCopy(c.Locked), - Release: c.Release, - Spec: deep.MustCopy(c.Spec), - Build: deep.MustCopy(c.Build), - Render: c.Render, - SourceFiles: deep.MustCopy(c.SourceFiles), - Packages: deep.MustCopy(c.Packages), - Publish: deep.MustCopy(c.Publish), - // OverlayFiles is consumed after component config resolution; preserve it verbatim - // here so inherited patterns can be interpreted relative to the concrete component - // config file. - OverlayFiles: slices.Clone(c.OverlayFiles), - Tests: deep.MustCopy(c.Tests), - } + // Copy all configuration fields, excluding shared back-references which can + // point back into the component map and must retain their identity. + input := *c + input.SourceConfigFile, input.upstreamCommitConfigFile = nil, nil + result := deep.MustCopy(&input) + result.SourceConfigFile, result.upstreamCommitConfigFile = c.SourceConfigFile, c.upstreamCommitConfigFile // Fix up paths. result.Spec.Path = makeAbsolute(referenceDir, result.Spec.Path) @@ -677,13 +661,8 @@ func (c *ComponentConfig) WithAbsolutePaths(referenceDir string) *ComponentConfi } } - // Copy and fix up overlays. - if c.Overlays != nil { - result.Overlays = make([]ComponentOverlay, len(c.Overlays)) - - for i := range result.Overlays { - result.Overlays[i] = *c.Overlays[i].WithAbsolutePaths(referenceDir) - } + for i := range result.Overlays { + result.Overlays[i] = *result.Overlays[i].WithAbsolutePaths(referenceDir) } return result diff --git a/internal/projectconfig/component_clone_internal_test.go b/internal/projectconfig/component_clone_internal_test.go new file mode 100644 index 000000000..2f6e43388 --- /dev/null +++ b/internal/projectconfig/component_clone_internal_test.go @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package projectconfig + +import ( + "testing" + + "github.com/samber/lo" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestComponentClonePreservesBackReferences(t *testing.T) { + source, pinSource := &ConfigFile{}, &ConfigFile{} + config := ComponentConfig{ + SourceConfigFile: source, upstreamCommitConfigFile: pinSource, + GoVendor: &GoVendorConfig{Enabled: lo.ToPtr(true), PrepPackages: []string{"git"}}, + Build: ComponentBuildConfig{Defines: map[string]string{"feature": "1"}}, + } + source.Components = map[string]ComponentConfig{"app": config} + pinSource.Components = source.Components + + resolved := config.WithAbsolutePaths("/project") + require.Same(t, source, resolved.SourceConfigFile) + require.Same(t, pinSource, resolved.upstreamCommitConfigFile) + require.NotSame(t, config.GoVendor, resolved.GoVendor) + require.Equal(t, config.GoVendor, resolved.GoVendor) + resolved.Build.Defines["feature"] = "2" + *resolved.GoVendor.Enabled = false + resolved.GoVendor.PrepPackages[0] = "changed" + assert.True(t, config.GoVendor.IsEnabled()) + + assert.Equal(t, "1", config.Build.Defines["feature"]) + assert.Equal(t, []string{"git"}, config.GoVendor.PrepPackages) +} diff --git a/internal/projectconfig/govendor_test.go b/internal/projectconfig/govendor_test.go index 1c363001a..e94ead8c6 100644 --- a/internal/projectconfig/govendor_test.go +++ b/internal/projectconfig/govendor_test.go @@ -28,6 +28,7 @@ tool-config = ''' `), &config)) require.True(t, config.GoVendor.IsEnabled()) require.NoError(t, config.GoVendor.Validate()) + require.Equal(t, config.GoVendor, config.WithAbsolutePaths("/project").GoVendor) require.NoError(t, config.MergeUpdatesFrom(&projectconfig.ComponentConfig{ GoVendor: &projectconfig.GoVendorConfig{Enabled: lo.ToPtr(false)}, })) diff --git a/internal/rpm/spec/govendor.go b/internal/rpm/spec/govendor.go index 29791e3c1..492faec23 100644 --- a/internal/rpm/spec/govendor.go +++ b/internal/rpm/spec/govendor.go @@ -22,11 +22,15 @@ type GoVendorOptions struct { ModuleDir string } -// GoVendorSpec contains the converted spec and the otherwise identical prep-only -// spec used to obtain the patched module tree before a vendor archive exists. +// GoVendorSpec is the common plan for converting or refreshing Go vendoring. type GoVendorSpec struct { - Spec []byte - PrepSpec []byte + Spec []byte + PrepSpec []byte + ArchiveFile string // Filename used inside generation staging. + ArchiveSuffix string // Required suffix of the RPM-expanded destination name, if any. + ConfigName string + ExpectedLicense string // Empty: detect and update. Otherwise: verify and preserve. + AllowReplacement bool // Permit replacing the existing vendor payload and configuration. } const goVendorEnvironment = `export GO111MODULE=on GOPROXY=off GOSUMDB=off GOTOOLCHAIN=local @@ -55,6 +59,16 @@ func ConvertGoVendor(data []byte, options GoVendorOptions) (*GoVendorSpec, error } tree := &specTree{root: root} + + configSource, err := existingVendorConfigSource(tree) + if err != nil { + return nil, err + } + + if configSource != "" { + return prepareExistingGoVendor(tree, configSource) + } + if err := validateGoVendorTree(tree); err != nil { return nil, err } @@ -69,7 +83,7 @@ func ConvertGoVendor(data []byte, options GoVendorOptions) (*GoVendorSpec, error prep.PrependLines([]string{"export GOTOOLCHAIN=local"}) prep.AppendLines([]string{"", `pwd -P > "%{_topdir}/azldev-go-module-root"`}) - prepSpec := serializeGoVendorTree(tree) + prepSpec := goVendorPrepSpec(tree, options.ArchiveName) sourceNumber, err := freeGoVendorSources(tree, options) if err != nil { @@ -97,7 +111,10 @@ func ConvertGoVendor(data []byte, options GoVendorOptions) (*GoVendorSpec, error addGoVendorBuildSections(tree, options, configRef) - return &GoVendorSpec{Spec: serializeGoVendorTree(tree), PrepSpec: prepSpec}, nil + return &GoVendorSpec{ + Spec: serializeGoVendorTree(tree), PrepSpec: prepSpec, + ArchiveFile: options.ArchiveName, ConfigName: options.ConfigName, + }, nil } func addGoVendorBuildSections(tree *specTree, options GoVendorOptions, configRef string) { @@ -111,11 +128,7 @@ func addGoVendorBuildSections(tree *specTree, options GoVendorOptions, configRef requires.PrependLines([]string{"%go_vendor_license_buildrequires -c " + configRef}) - for _, name := range []string{"%build", "%install", "%check", "%generate_buildrequires"} { - if section := tree.Section(name, ""); section != nil { - section.PrependLines([]string{goVendorEnvironment}) - } - } + addGoVendorEnvironment(tree) tree.Section("%install", "").AppendLines([]string{ "", goVendorInModule(options.ModuleDir, "%go_vendor_license_install -c "+configRef), @@ -156,6 +169,45 @@ func serializeGoVendorTree(tree *specTree) []byte { return []byte(strings.Join(serializeTree(tree.root), "\n") + "\n") } +func addGoVendorEnvironment(tree *specTree) { + for _, name := range []string{"%build", "%install", "%check", "%generate_buildrequires"} { + if section := tree.Section(name, ""); section != nil { + section.PrependLines([]string{goVendorEnvironment}) + } + } +} + +func goVendorPrepSpec(tree *specTree, archiveValue string) []byte { + prep := tree.Section("%prep", "") + prep.AppendLines([]string{ + "printf '%%s\\n' %{shescape:" + archiveValue + `} > "%{_topdir}/azldev-go-archive-name"`, + }) + + result := serializeGoVendorTree(tree) + prep.block.Children = prep.block.Children[:len(prep.block.Children)-1] + + return result +} + +// FinalizeLicense preserves a verified upstream expression or writes the detected +// expression for a newly converted spec. It does not mutate the plan. +func (p *GoVendorSpec) FinalizeLicense(expression string) ([]byte, error) { + if p.ExpectedLicense != "" { + return p.Spec, nil + } + + editor, err := openStructuralSpec(strings.NewReader(string(p.Spec))) + if err != nil { + return nil, err + } + + if err := editor.UpdateExistingTag("", "License", expression); err != nil { + return nil, err + } + + return []byte(strings.Join(editor.rawLines, "\n") + "\n"), nil +} + func goVendorInModule(moduleDir, command string) string { lines := []string{`( cd "$(cat "%{_topdir}/azldev-go-module-root")"`} if moduleDir != "" { diff --git a/internal/rpm/spec/govendor_existing.go b/internal/rpm/spec/govendor_existing.go new file mode 100644 index 000000000..b7a237c0a --- /dev/null +++ b/internal/rpm/spec/govendor_existing.go @@ -0,0 +1,212 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package spec + +import ( + "errors" + "fmt" + "path/filepath" + "regexp" + "slices" + "strings" + + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" +) + +var ( + existingVendorLicense = regexp.MustCompile( + `^%go_vendor_license_(?:buildrequires|install|check)\s+-c\s+%\{(?:S:|SOURCE)([0-9]+)\}\s*$`) + existingVendorExtract = regexp.MustCompile(`^tar\s+-xf\s+%\{(?:S:|SOURCE)([0-9]+)\}\s*$`) +) + +func prepareExistingGoVendor(tree *specTree, configSource string) (*GoVendorSpec, error) { + if err := validateGoVendorSection(tree, "%prep", true); err != nil { + return nil, err + } + + configName, err := existingVendorTagValue(tree, "Source"+configSource) + if err != nil { + return nil, err + } + + if err := fileutils.ValidateFilename(configName); err != nil || strings.Contains(configName, "%") { + return nil, fmt.Errorf("existing Go vendor config Source%s must name a plain file, got %#q", configSource, configName) + } + + extraction, sourceNumber, err := existingVendorExtraction(tree) + if err != nil { + return nil, err + } + + if sourceNumber == configSource { + return nil, errors.New("go vendor archive and tool configuration cannot use the same Source tag") + } + + archiveValue, err := existingVendorTagValue(tree, "Source"+sourceNumber) + if err != nil { + return nil, err + } + + suffix, err := existingVendorArchiveSuffix(archiveValue) + if err != nil { + return nil, err + } + + license, err := existingVendorLicenseExpression(tree) + if err != nil { + return nil, err + } + + // Keep upstream's Source numbers, macro settings, file lists, and custom commands. + for _, sectionName := range []string{"%build", "%install", "%check", "%generate_buildrequires"} { + if err := validateGoVendorSection(tree, sectionName, false); err != nil { + return nil, err + } + } + + addGoVendorEnvironment(tree) + finalSpec := serializeGoVendorTree(tree) + + if err := removeExistingVendorInput(tree, sourceNumber, extraction); err != nil { + return nil, err + } + + tree.Section("%prep", "").AppendLines([]string{ + `pwd -P > "%{_topdir}/azldev-go-module-root"`, + }) + + return &GoVendorSpec{ + Spec: finalSpec, PrepSpec: goVendorPrepSpec(tree, archiveValue), + ArchiveFile: "azldev-generated-vendor" + suffix, ArchiveSuffix: suffix, ConfigName: configName, + ExpectedLicense: license, AllowReplacement: true, + }, nil +} + +func existingVendorLicenseExpression(tree *specTree) (string, error) { + license, err := existingVendorTagValue(tree, "License") + if err != nil { + return "", err + } + + if strings.ContainsAny(license, "%\r\n") || license == "" { + return "", errors.New("regenerating an existing Go vendor archive requires an explicit main-package 'License' tag") + } + + return license, nil +} + +// An empty result means the spec has no existing vendor license commands. +func existingVendorConfigSource(tree *specTree) (string, error) { + source := "" + + err := tree.VisitAllLines(func(_, _ string, line *lineHandle) error { + text := strings.TrimSpace(line.Text) + if !strings.HasPrefix(text, "%go_vendor_license_") { + return nil + } + + matches := existingVendorLicense.FindStringSubmatch(text) + if len(matches) == 0 { + return fmt.Errorf("unsupported existing Go vendor license command %#q", text) + } + + if source != "" && source != matches[1] { + return errors.New("existing Go vendor license commands reference different configuration sources") + } + + source = matches[1] + + return nil + }) + if err != nil { + return "", err + } + + return source, nil +} + +func existingVendorExtraction(tree *specTree) (line, source string, err error) { + count := 0 + + err = tree.Section("%prep", "").VisitLines(func(candidate *lineHandle) error { + matches := existingVendorExtract.FindStringSubmatch(strings.TrimSpace(candidate.Text)) + if len(matches) != 0 { + if !slices.Contains(tree.Section("%prep", "").block.Children, candidate.block) { + return errors.New("conditional Go vendor extraction requires an explicit preparation strategy") + } + + count++ + line = strings.TrimSpace(candidate.Text) + source = matches[1] + } + + return nil + }) + if err != nil { + return "", "", err + } + + if count != 1 { + return "", "", errors.New("existing Go vendoring requires one unambiguous 'tar -xf %{S:N}' prep command") + } + + return line, source, nil +} + +func existingVendorTagValue(tree *specTree, name string) (string, error) { + var values []string + + err := tree.VisitAllLines(func(section, pkg string, line *lineHandle) error { + tag, value, ok := parseTagLine(line.Text) + if ok && section == "" && pkg == "" && strings.EqualFold(tag, name) { + if !slices.Contains(tree.Section("", "").block.Children, line.block) { + return fmt.Errorf("existing Go vendoring requires an unconditional %#q tag", name) + } + + values = append(values, value) + } + + return nil + }) + if err != nil { + return "", err + } + + if len(values) != 1 { + return "", fmt.Errorf("expected exactly one %#q declaration for existing Go vendoring", name) + } + + return values[0], nil +} + +func existingVendorArchiveSuffix(value string) (string, error) { + if strings.Contains(value, "://") { + return "", errors.New("existing Go vendor archive must use a local source filename") + } + + extension := filepath.Ext(value) + switch extension { + case ".bz2", ".gz", ".xz", ".zst", ".zstd": + return ".tar" + extension, nil + case ".tar": + return ".tar", nil + default: + return "", fmt.Errorf("unsupported existing Go vendor archive format %#q", value) + } +} + +func removeExistingVendorInput(tree *specTree, sourceNumber, extraction string) error { + return tree.VisitAllLines(func(section, pkg string, line *lineHandle) error { + tag, _, ok := parseTagLine(line.Text) + if ok && section == "" && pkg == "" && strings.EqualFold(tag, "Source"+sourceNumber) { + line.Remove() + } + + if section == "%prep" && strings.TrimSpace(line.Text) == extraction { + line.Remove() + } + + return nil + }) +} diff --git a/internal/rpm/spec/govendor_existing_test.go b/internal/rpm/spec/govendor_existing_test.go new file mode 100644 index 000000000..3b1927f95 --- /dev/null +++ b/internal/rpm/spec/govendor_existing_test.go @@ -0,0 +1,87 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package spec_test + +import ( + "strings" + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/rpm/spec" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func existingVendorSpec() string { + return strings.Join([]string{ + "Name: app", "Version: 1.2", "Release: %autorelease", "Summary: Application", + "License: MIT AND Apache-2.0", + "Source0: app-%{version}.tar.gz", + "Source5: app-%{version}-vendor.tar.bz2", + "Source8: go-vendor-tools.toml", + "Source9: app.conf", + "BuildRequires: go-vendor-tools", "Requires: runtime-helper", + "%description", "Application", "%prep", "%goprep -p1", + "tar -xf %{S:5}", "sed -i 's/old/new/' go.mod", "cp %{S:9} .", + "%generate_buildrequires", "%go_vendor_license_buildrequires -c %{S:8}", + "%build", "%global gomodulesmode GO111MODULE=on", "%gobuild -o app .", + "%install", "%go_vendor_license_install -c %{S:8}", + "install -D app %{buildroot}%{_bindir}/app", + "%check", "%go_vendor_license_check -c %{S:8}", "%gocheck2 -t integration", + "%files -f %{go_vendor_license_filelist}", "%{_bindir}/app", "%changelog", "%autochangelog", "", + }, "\n") +} + +func TestExistingGoVendoringPreservesPackaging(t *testing.T) { + for _, version := range []string{"1.2", "2.4"} { + input := strings.ReplaceAll(existingVendorSpec(), "Version: 1.2", "Version: "+version) + result, err := spec.ConvertGoVendor([]byte(input), spec.GoVendorOptions{ + ArchiveName: "app-vendor.tar.bz2", ConfigName: "azldev-go-vendor.toml", + }) + require.NoError(t, err) + assert.True(t, result.AllowReplacement) + assert.Equal(t, "go-vendor-tools.toml", result.ConfigName) + assert.Equal(t, ".tar.bz2", result.ArchiveSuffix) + assert.Equal(t, "MIT AND Apache-2.0", result.ExpectedLicense) + + for _, line := range strings.Split(input, "\n") { + assert.Contains(t, string(result.Spec), line) + } + + assert.NotContains(t, string(result.Spec), "azldev-go-vendor.toml") + assert.NotContains(t, string(result.Spec), "azldev-go-module-root") + assert.Contains(t, string(result.Spec), "GOPROXY=off") + assert.NotContains(t, string(result.PrepSpec), "Source5:") + assert.NotContains(t, string(result.PrepSpec), "tar -xf %{S:5}") + assert.Contains(t, string(result.PrepSpec), "sed -i 's/old/new/' go.mod") + assert.Contains(t, string(result.PrepSpec), "%{shescape:app-%{version}-vendor.tar.bz2}") + assert.Contains(t, string(result.PrepSpec), `pwd -P > "%{_topdir}/azldev-go-module-root"`) + } +} + +func TestExistingGoVendoringRejectsAmbiguousInputs(t *testing.T) { + for name, input := range map[string]string{ + "multiple extractions": strings.ReplaceAll(existingVendorSpec(), "%goprep -p1", "%goprep -p1\ntar -xf %{S:9}"), + "no extraction": strings.ReplaceAll(existingVendorSpec(), "tar -xf %{S:5}", ""), + "mismatched config": strings.ReplaceAll(existingVendorSpec(), + "%go_vendor_license_check -c %{S:8}", "%go_vendor_license_check -c %{S:9}"), + "dynamic config": strings.ReplaceAll(existingVendorSpec(), "go-vendor-tools.toml", "%{name}.toml"), + "unknown compression": strings.ReplaceAll(existingVendorSpec(), "-vendor.tar.bz2", "-vendor.zip"), + "macro license": strings.ReplaceAll(existingVendorSpec(), "License: MIT AND Apache-2.0", "License: %{license}"), + "missing source": strings.ReplaceAll(existingVendorSpec(), "Source5:", "Source6:"), + "conditional extraction": strings.ReplaceAll(existingVendorSpec(), + "tar -xf %{S:5}", "%if 1\ntar -xf %{S:5}\n%endif"), + "conditional source": strings.ReplaceAll(existingVendorSpec(), + "Source5: app-%{version}-vendor.tar.bz2", "%if 1\nSource5: app-%{version}-vendor.tar.bz2\n%endif"), + "conditional license": strings.ReplaceAll(existingVendorSpec(), + "License: MIT AND Apache-2.0", "%if 1\nLicense: MIT AND Apache-2.0\n%endif"), + } { + t.Run(name, func(t *testing.T) { + result, err := spec.ConvertGoVendor([]byte(input), spec.GoVendorOptions{ + ArchiveName: "app-vendor.tar.bz2", ConfigName: "azldev-go-vendor.toml", + }) + require.Error(t, err) + assert.Nil(t, result) + }) + } +} diff --git a/internal/rpm/spec/govendor_test.go b/internal/rpm/spec/govendor_test.go index cfed507aa..c366de034 100644 --- a/internal/rpm/spec/govendor_test.go +++ b/internal/rpm/spec/govendor_test.go @@ -97,7 +97,10 @@ func TestConvertGoVendorPreservesUpstream(t *testing.T) { assert.Contains(t, string(result.PrepSpec), "Release: %autorelease") assert.Contains(t, string(result.PrepSpec), `pwd -P > "%{_topdir}/azldev-go-module-root"`) assert.Contains(t, string(result.PrepSpec), "sed -i 's/old/new/' go.mod") - assert.NotContains(t, string(result.PrepSpec), "app-vendor.tar.bz2") + assert.NotContains(t, string(result.PrepSpec), "Source2: app-vendor.tar.bz2") + assert.Contains(t, string(result.PrepSpec), "%{shescape:app-vendor.tar.bz2}") + assert.False(t, result.AllowReplacement) + assert.Empty(t, result.ExpectedLicense) assert.Contains(t, string(result.PrepSpec), "%autochangelog") _, err = spec.OpenSpec(strings.NewReader(output), spec.WithEditor(spec.EditorStructural)) From b9cb5a6bcc7f6d40bf88a92d9122034f00a91100 Mon Sep 17 00:00:00 2001 From: Antonio Salinas <173939267+Tonisal-byte@users.noreply.github.com> Date: Mon, 2 Nov 2026 11:20:22 -0800 Subject: [PATCH 3/5] feat: load Go vendor settings from config files Replace the inline tool-config option with config-file, resolved relative to its declaring component configuration. Explicit files replace upstream settings, including empty files, and missing or invalid files fail without fallback. Hash file contents rather than checkout paths for lock fingerprints and lock-free change detection. Preserve upstream settings when no file is selected, and update the schema, documentation, emitted guidance, and regression coverage. Validation: unit tests and all quality checks passed. The changed-code mutation audit killed all 16 mutations. Containerd source generation with a separate vendor config reproduced the prior outputs, and a file-only edit invalidated its isolated lock fingerprint. Docker scenario initialization remains unavailable in this environment. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/user/how-to/vendor-go-application.md | 34 ++++++++-- docs/user/reference/config/components.md | 27 ++++++-- .../app/azldev/agentskill/agentskill_test.go | 2 + .../agentskill/content/comp-toml.md.tmpl | 8 ++- .../content/withoutlockfile/comp-toml.md.tmpl | 8 ++- .../cmds/component/changed_upstreamcommit.go | 14 ++++ .../changed_upstreamcommit_internal_test.go | 22 +++++++ .../cmds/component/history_customizations.go | 4 +- .../cmds/component/history_internal_test.go | 1 - .../component/preparerctx_internal_test.go | 4 +- internal/app/azldev/core/sources/govendor.go | 20 ++++-- .../sources/govendor_config_internal_test.go | 41 ++++++++++++ .../core/sources/govendor_internal_test.go | 16 ++++- internal/fingerprint/doc.go | 2 +- internal/fingerprint/fingerprint.go | 16 ++++- internal/fingerprint/govendor_test.go | 62 ++++++++++++++++++ internal/govendor/generate.go | 6 +- internal/projectconfig/component.go | 3 + internal/projectconfig/fingerprint_test.go | 3 + internal/projectconfig/govendor.go | 60 +++++++++++++---- .../govendor_loader_internal_test.go | 56 ++++++++++++++++ internal/projectconfig/govendor_test.go | 64 ++++++++++++++++--- ...shots_config_generate-schema_stdout_1.snap | 9 ++- schemas/azldev.schema.json | 9 ++- 24 files changed, 433 insertions(+), 58 deletions(-) create mode 100644 internal/app/azldev/core/sources/govendor_config_internal_test.go create mode 100644 internal/fingerprint/govendor_test.go create mode 100644 internal/projectconfig/govendor_loader_internal_test.go diff --git a/docs/user/how-to/vendor-go-application.md b/docs/user/how-to/vendor-go-application.md index 068673213..3892e8426 100644 --- a/docs/user/how-to/vendor-go-application.md +++ b/docs/user/how-to/vendor-go-application.md @@ -15,6 +15,31 @@ Add this to the component definition: enabled = true ``` +For custom vendor settings, use a separate file beside the component definition: + +```toml +# example.comp.toml +[components.example.go-vendor] +enabled = true +config-file = "go-vendor-tools.toml" +``` + +```toml +# go-vendor-tools.toml +[archive] +use_module_proxy = true + +[licensing] +detector = "askalono" +``` + +This file replaces, rather than merges with, upstream settings. Carry forward +any required upstream license exceptions. Omit `config-file` to reuse those +settings unchanged. Paths are relative to the declaring component configuration, +including when inherited from defaults; no adjacent file is loaded implicitly. +File-only edits affect the component fingerprint. Inline `tool-config` values +are no longer accepted. + Then prepare the complete build inputs: ```sh @@ -35,7 +60,7 @@ This retains upstream patches and edits to module files. It then calls `go_vendor_archive create` on the prepared module tree, extracts the result for license detection, and runs `go_vendor_license` without interactive prompts. Unrecognized licenses fail preparation; add reviewed, hash-qualified exceptions -to `tool-config` rather than suppressing detection. +to the file selected by `config-file` rather than suppressing detection. The output includes the converted spec, all its original sources and patches, `-vendor.tar.bz2`, and `azldev-go-vendor.toml`. The archive is recorded @@ -84,9 +109,10 @@ the replacement does not consume the old vendor payload. RPM expands the upstream archive filename; the regenerated archive replaces that file and its `sources` checksum rather than adding a second vendor archive. -When `tool-config` is omitted, the upstream go-vendor-tools configuration is used, -including its reviewed license exceptions. An explicit `tool-config` replaces -that configuration. The detected licenses are verified against the existing +When `config-file` is omitted, the upstream go-vendor-tools configuration is used, +including its reviewed license exceptions. An explicit file replaces that configuration +even when empty; missing files or invalid TOML fail without falling back. +The detected licenses are verified against the existing main-package `License:` expression; mismatches fail instead of rewriting it. The source-fetch phase currently still downloads the upstream vendor archive alongside the other upstream sources, before regenerating its contents. diff --git a/docs/user/reference/config/components.md b/docs/user/reference/config/components.md index d16fffc7b..530b9cea3 100644 --- a/docs/user/reference/config/components.md +++ b/docs/user/reference/config/components.md @@ -182,22 +182,33 @@ enabled = true | `enabled` | boolean | `false` | Enable conversion and vendor archive generation. An explicit `false` overrides inherited enablement. | | `module-dir` | string | `""` | Relative directory containing `go.mod`, resolved from the working directory at the end of the upstream `%prep` section. Empty uses that directory itself. No absolute paths, traversal, or RPM macros. | | `prep-packages` | array of string | `[]` | Additional RPM packages needed by upstream `%prep` commands or the selected license detector. Installed only in the isolated generation chroot. | -| `tool-config` | string | `""` | Inline [go-vendor-tools TOML](https://fedora.gitlab.io/sigs/go/go-vendor-tools/config/), for dependency overrides, archive options, and reviewed license exceptions. When omitted for an already-vendored spec, reuse its upstream tool configuration. An explicit value replaces it. Use `module-dir`, not `general.go_mod_dir`. | +| `config-file` | string | `""` | Path to a separate [go-vendor-tools TOML file](https://fedora.gitlab.io/sigs/go/go-vendor-tools/config/), relative to the configuration file that declares it. Its contents replace upstream tool settings; they are not merged. When no file is configured, reuse an already-vendored spec's upstream settings or use tool defaults for a new conversion. | -The tool configuration is inline so its contents participate in component -fingerprinting without checkout-dependent paths. Configured command hooks execute -in the generation chroot; treat this configuration as trusted build code. +Custom vendor settings must live in a separate file, typically +`go-vendor-tools.toml` beside the component definition. Its contents participate +in the component fingerprint; absolute checkout paths do not. Inherited paths +retain their original declaration directory. Editing only this file makes the +lock fingerprint stale and requires the normal update/render workflow. ```toml +# example.comp.toml [components.example.go-vendor] enabled = true -prep-packages = ["git"] -tool-config = ''' +config-file = "go-vendor-tools.toml" +``` + +```toml +# go-vendor-tools.toml [archive.dependency_overrides] "example.com/dependency" = "v1.2.3" -''' ``` +The configured file must exist and contain valid TOML. An explicitly selected +empty file is an empty replacement, not a request to fall back to upstream. +Use `module-dir`, not `general.go_mod_dir`, for the module location. +Configured command hooks execute in the generation chroot; treat this file as +trusted build code. The old inline `tool-config` string/table is no longer supported. + For previously unbundled specs, preparation emits `-vendor.tar.bz2` and `azldev-go-vendor.toml`, allocating unused `Source` numbers automatically. These names are reserved for generated output; do not also declare them in `source-files` or supply them through @@ -209,6 +220,8 @@ For supported specs already using go-vendor-tools, preparation instead regenerat the existing vendor source, retaining its Source number, RPM-expanded filename, and configuration sidecar. It updates that archive's `sources` entry in place and verifies the existing `License:` expression without rewriting it. +When `config-file` is set, its contents are staged under the configuration filename +referenced by the prepared spec, retaining the existing Source number. All archive/spec/loose-file overlays run before conversion. Source preparation, builds, rendering (including `--check-only`), and source diffs use the same diff --git a/internal/app/azldev/agentskill/agentskill_test.go b/internal/app/azldev/agentskill/agentskill_test.go index 7ebe18bdf..0836c6751 100644 --- a/internal/app/azldev/agentskill/agentskill_test.go +++ b/internal/app/azldev/agentskill/agentskill_test.go @@ -168,6 +168,8 @@ func TestComponentTomlSkillDocumentsGoVendoring(t *testing.T) { assert.Contains(t, doc, "does\nnot run `go2rpm`") assert.Contains(t, doc, "Rendering and source diffs also generate") assert.Contains(t, doc, "reviewed license exceptions") + assert.Contains(t, doc, `config-file = "go-vendor-tools.toml"`) + assert.Contains(t, doc, "Inline `tool-config` is not supported.") } } diff --git a/internal/app/azldev/agentskill/content/comp-toml.md.tmpl b/internal/app/azldev/agentskill/content/comp-toml.md.tmpl index a6dbf477e..b724ea0ef 100644 --- a/internal/app/azldev/agentskill/content/comp-toml.md.tmpl +++ b/internal/app/azldev/agentskill/content/comp-toml.md.tmpl @@ -43,8 +43,12 @@ not run `go2rpm`. Rendering and source diffs also generate the vendor archive an require downloads. Review the generated cumulative `License:` expression. Use `module-dir` for a module below the final prep working directory, -`prep-packages` for additional chroot tools, and inline `tool-config` for -go-vendor-tools dependency overrides or reviewed license exceptions. +`prep-packages` for additional chroot tools, and `config-file = "go-vendor-tools.toml"` +for custom settings in a separate native TOML file beside the component definition. +Keep reviewed license exceptions in that file. +The path is relative to the declaring config, and file contents are fingerprinted. +It replaces upstream settings without merging; an explicit empty file is still a +replacement, and a missing file is an error. Inline `tool-config` is not supported. Generated `-vendor.tar.bz2` and `azldev-go-vendor.toml` must not also be declared as `source-files`. For supported already-vendored specs, regenerate the existing vendor Source instead, reuse upstream tool configuration unless overridden, diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl index 96690b936..cd0ec3e48 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl @@ -43,8 +43,12 @@ not run `go2rpm`. Rendering and source diffs also generate the vendor archive an require downloads. Review the generated cumulative `License:` expression. Use `module-dir` for a module below the final prep working directory, -`prep-packages` for additional chroot tools, and inline `tool-config` for -go-vendor-tools dependency overrides or reviewed license exceptions. +`prep-packages` for additional chroot tools, and `config-file = "go-vendor-tools.toml"` +for custom settings in a separate native TOML file beside the component definition. +Keep reviewed license exceptions in that file. +The path is relative to the declaring config, and file contents are fingerprinted. +It replaces upstream settings without merging; an explicit empty file is still a +replacement, and a missing file is an error. Inline `tool-config` is not supported. Generated `-vendor.tar.bz2` and `azldev-go-vendor.toml` must not also be declared as `source-files`. For supported already-vendored specs, regenerate the existing vendor Source instead, reuse upstream tool configuration unless overridden, diff --git a/internal/app/azldev/cmds/component/changed_upstreamcommit.go b/internal/app/azldev/cmds/component/changed_upstreamcommit.go index ce8e19ebc..ce18e0d5e 100644 --- a/internal/app/azldev/cmds/component/changed_upstreamcommit.go +++ b/internal/app/azldev/cmds/component/changed_upstreamcommit.go @@ -118,6 +118,7 @@ type componentComparisonInputs struct { Config projectconfig.ComponentConfig `json:"config"` SourceIdentity string `json:"sourceIdentity,omitempty"` OverlaySourceHashes map[string]string `json:"overlaySourceHashes,omitempty"` + GoVendorConfigHash string `json:"goVendorConfigHash,omitempty"` ReleaseVer string `json:"releaseVer,omitempty"` } @@ -252,6 +253,13 @@ func buildComponentComparisonInputs( SourceIdentity: component.EffectiveUpstreamCommit(), } + var err error + + inputs.GoVendorConfigHash, err = component.GoVendor.SourceContentIdentity(fs) + if err != nil { + return inputs, fmt.Errorf("hashing Go vendor configuration:\n%w", err) + } + if component.Spec.SourceType == projectconfig.SpecSourceTypeLocal { identity, err := sourceproviders.ResolveLocalSourceIdentity( fs, filepath.Dir(component.Spec.Path), @@ -314,7 +322,13 @@ func normalizeComponentForComparison(component projectconfig.ComponentConfig) pr normalized.Build.Hints = projectconfig.ComponentBuildHints{} normalized.OverlayFiles = nil normalized.Publish = projectconfig.ComponentPublishConfig{} + normalized.Tests = nil + if component.GoVendor != nil { + vendor := *component.GoVendor + vendor.ConfigFile = "" + normalized.GoVendor = &vendor + } normalized.Overlays = slices.Clone(component.Overlays) for idx := range normalized.Overlays { diff --git a/internal/app/azldev/cmds/component/changed_upstreamcommit_internal_test.go b/internal/app/azldev/cmds/component/changed_upstreamcommit_internal_test.go index 357f3321b..1680f2088 100644 --- a/internal/app/azldev/cmds/component/changed_upstreamcommit_internal_test.go +++ b/internal/app/azldev/cmds/component/changed_upstreamcommit_internal_test.go @@ -209,6 +209,28 @@ func TestBuildComponentComparisonInputs_ContentIdentities(t *testing.T) { assert.NotEqual(t, second.OverlaySourceHashes, third.OverlaySourceHashes) } +func TestBuildComponentComparisonInputs_GoVendorFile(t *testing.T) { + env := testutils.NewTestEnvWithoutLockfile(t) + enabled := true + component := projectconfig.ComponentConfig{GoVendor: &projectconfig.GoVendorConfig{ + Enabled: &enabled, ConfigFile: "/project/vendor.toml", + }} + require.NoError(t, fileutils.WriteFile(env.FS(), component.GoVendor.ConfigFile, + []byte("[archive]\ntidy = false\n"), fileperms.PublicFile)) + first, err := buildComponentComparisonInputs(env.FS(), env.Env, &component) + require.NoError(t, err) + assert.Empty(t, first.Config.GoVendor.ConfigFile) + assert.Equal(t, "/project/vendor.toml", component.GoVendor.ConfigFile) + require.NotEmpty(t, first.GoVendorConfigHash) + + require.NoError(t, fileutils.WriteFile(env.FS(), component.GoVendor.ConfigFile, + []byte("[archive]\ntidy = true\n"), fileperms.PublicFile)) + second, err := buildComponentComparisonInputs(env.FS(), env.Env, &component) + require.NoError(t, err) + assert.NotEqual(t, first.GoVendorConfigHash, second.GoVendorConfigHash) + assert.Equal(t, first.Config, second.Config) +} + func TestLoadHistoricalProject_UsesNormalIncludesAndMerging(t *testing.T) { rootConfig := []byte(`includes = [ "config/components.toml", diff --git a/internal/app/azldev/cmds/component/history_customizations.go b/internal/app/azldev/cmds/component/history_customizations.go index d1950fc34..8a051a54d 100644 --- a/internal/app/azldev/cmds/component/history_customizations.go +++ b/internal/app/azldev/cmds/component/history_customizations.go @@ -47,8 +47,8 @@ func appendGoVendorItems(items []CustomizationItem, config *projectconfig.GoVend items = append(items, CustomizationItem{Kind: "go-vendor.module-dir", Value: config.ModuleDir}) } - if config.ToolConfig != "" { - items = append(items, CustomizationItem{Kind: "go-vendor.tool-config", Value: config.ToolConfig}) + if config.ConfigFile != "" { + items = append(items, CustomizationItem{Kind: "go-vendor.config-file", Value: config.ConfigFile}) } for _, name := range config.PrepPackages { diff --git a/internal/app/azldev/cmds/component/history_internal_test.go b/internal/app/azldev/cmds/component/history_internal_test.go index eb9ef98fd..858d509f3 100644 --- a/internal/app/azldev/cmds/component/history_internal_test.go +++ b/internal/app/azldev/cmds/component/history_internal_test.go @@ -107,7 +107,6 @@ func TestCustomizationCollectorsCoverEveryFingerprintableField(t *testing.T) { "ComponentConfig.GoVendor": "appendGoVendorItems", "GoVendorConfig.Enabled": "go-vendor.enabled", "GoVendorConfig.ModuleDir": "go-vendor.module-dir", - "GoVendorConfig.ToolConfig": "go-vendor.tool-config", "GoVendorConfig.PrepPackages": "go-vendor.prep-packages", // ComponentBuildConfig. diff --git a/internal/app/azldev/cmds/component/preparerctx_internal_test.go b/internal/app/azldev/cmds/component/preparerctx_internal_test.go index 7118b0017..715286b8f 100644 --- a/internal/app/azldev/cmds/component/preparerctx_internal_test.go +++ b/internal/app/azldev/cmds/component/preparerctx_internal_test.go @@ -52,12 +52,12 @@ func TestGoVendorPreparerOptions(t *testing.T) { func TestGoVendorHistoryCustomizations(t *testing.T) { config := &projectconfig.GoVendorConfig{ Enabled: lo.ToPtr(true), ModuleDir: "cmd/app", - ToolConfig: "[archive]\ntidy = false", PrepPackages: []string{"git"}, + ConfigFile: "/project/go-vendor-tools.toml", PrepPackages: []string{"git"}, } assert.Equal(t, []CustomizationItem{ {Kind: "go-vendor.enabled", Value: "true"}, {Kind: "go-vendor.module-dir", Value: "cmd/app"}, - {Kind: "go-vendor.tool-config", Value: config.ToolConfig}, + {Kind: "go-vendor.config-file", Value: config.ConfigFile}, {Kind: "go-vendor.prep-packages", Value: "git"}, }, collectCustomizations("app", &projectconfig.ComponentConfig{GoVendor: config})) assert.Empty(t, appendGoVendorItems(nil, &projectconfig.GoVendorConfig{})) diff --git a/internal/app/azldev/core/sources/govendor.go b/internal/app/azldev/core/sources/govendor.go index 4d2e42b54..7709ac8e7 100644 --- a/internal/app/azldev/core/sources/govendor.go +++ b/internal/app/azldev/core/sources/govendor.go @@ -14,6 +14,7 @@ import ( "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" "github.com/microsoft/azure-linux-dev-tools/internal/govendor" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders/fedorasource" "github.com/microsoft/azure-linux-dev-tools/internal/rpm/mock" "github.com/microsoft/azure-linux-dev-tools/internal/rpm/spec" @@ -66,7 +67,7 @@ func (p *sourcePreparerImpl) prepareGoVendor( return fmt.Errorf("planning Go vendoring for %#q:\n%w", component.GetName(), err) } - toolConfig, err := p.resolveGoVendorConfig(outputDir, plan, config.ToolConfig) + toolConfig, err := p.resolveGoVendorConfig(outputDir, plan, config) if err != nil { return err } @@ -82,7 +83,7 @@ func (p *sourcePreparerImpl) prepareGoVendor( artifact, err := govendor.Generate(ctx, p.fs, p.goVendorRunner, govendor.Request{ SourceDir: outputDir, WorkDir: scratch, SpecName: filepath.Base(specPath), PrepSpec: plan.PrepSpec, ArchiveFile: plan.ArchiveFile, ArchiveSuffix: plan.ArchiveSuffix, ModuleDir: config.ModuleDir, - ToolConfig: toolConfig, Packages: config.PrepPackages, ExpectedLicense: plan.ExpectedLicense, + ConfigContents: toolConfig, Packages: config.PrepPackages, ExpectedLicense: plan.ExpectedLicense, }) if err != nil { return fmt.Errorf("generating Go vendor sources:\n%w", err) @@ -92,7 +93,7 @@ func (p *sourcePreparerImpl) prepareGoVendor( } func (p *sourcePreparerImpl) resolveGoVendorConfig( - outputDir string, plan *spec.GoVendorSpec, configured string, + outputDir string, plan *spec.GoVendorSpec, config *projectconfig.GoVendorConfig, ) (string, error) { if !plan.AllowReplacement { if err := p.checkGoVendorSourceCollision(outputDir, plan.ArchiveFile, plan.ConfigName); err != nil { @@ -100,8 +101,17 @@ func (p *sourcePreparerImpl) resolveGoVendorConfig( } } - if configured != "" || !plan.AllowReplacement { - return configured, nil + if config.ConfigFile != "" { + content, err := config.ReadConfigFile(p.fs) + if err != nil { + return "", fmt.Errorf("loading Go vendor configuration:\n%w", err) + } + + return content, nil + } + + if !plan.AllowReplacement { + return "", nil } data, err := fileutils.ReadFile(p.fs, filepath.Join(outputDir, plan.ConfigName)) diff --git a/internal/app/azldev/core/sources/govendor_config_internal_test.go b/internal/app/azldev/core/sources/govendor_config_internal_test.go new file mode 100644 index 000000000..f4cb285df --- /dev/null +++ b/internal/app/azldev/core/sources/govendor_config_internal_test.go @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package sources + +import ( + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/global/testctx" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/rpm/spec" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGoVendorConfigFileSelection(t *testing.T) { + for _, local := range []string{"", "[archive]\ntidy = false\n"} { + ctx := testctx.NewCtx() + require.NoError(t, fileutils.WriteFile(ctx.FS(), "/prepared/upstream.toml", + []byte("[licensing]\ndetector = 'askalono'\n"), fileperms.PublicFile)) + require.NoError(t, fileutils.WriteFile(ctx.FS(), "/project/local.toml", []byte(local), fileperms.PublicFile)) + preparer := &sourcePreparerImpl{fs: ctx.FS()} + plan := &spec.GoVendorSpec{AllowReplacement: true, ConfigName: "upstream.toml"} + config := &projectconfig.GoVendorConfig{ConfigFile: "/project/local.toml"} + actual, err := preparer.resolveGoVendorConfig("/prepared", plan, config) + require.NoError(t, err) + assert.Equal(t, local, actual, "explicit file replaces rather than merges upstream settings") + + config.ConfigFile = "" + actual, err = preparer.resolveGoVendorConfig("/prepared", plan, config) + require.NoError(t, err) + assert.Contains(t, actual, "askalono") + + config.ConfigFile = "/project/missing.toml" + _, err = preparer.resolveGoVendorConfig("/prepared", plan, config) + require.Error(t, err, "missing explicit file must not fall back to upstream") + assert.Contains(t, err.Error(), config.ConfigFile) + } +} diff --git a/internal/app/azldev/core/sources/govendor_internal_test.go b/internal/app/azldev/core/sources/govendor_internal_test.go index 6509507b2..963aeeecc 100644 --- a/internal/app/azldev/core/sources/govendor_internal_test.go +++ b/internal/app/azldev/core/sources/govendor_internal_test.go @@ -45,12 +45,18 @@ func TestPrepareSourcesGoVendor(t *testing.T) { component := components_testutils.NewMockComponent(ctrl) manager := sourceproviders_test.NewMockSourceManager(ctrl) config := &projectconfig.ComponentConfig{ - GoVendor: &projectconfig.GoVendorConfig{Enabled: lo.ToPtr(true), PrepPackages: []string{"git"}}, + GoVendor: &projectconfig.GoVendorConfig{ + Enabled: lo.ToPtr(true), PrepPackages: []string{"git"}, ConfigFile: "/project/vendor.toml", + }, Overlays: []projectconfig.ComponentOverlay{{ Type: "spec-set-tag", Tag: "Summary", Value: "overlaid", }}, } + const settings = "# reviewed settings\n[archive]\nuse_module_proxy = true\n" + require.NoError(t, fileutils.WriteFile(ctx.FS(), config.GoVendor.ConfigFile, + []byte(settings), fileperms.PublicFile)) + component.EXPECT().GetName().AnyTimes().Return("app") component.EXPECT().GetConfig().AnyTimes().Return(config) manager.EXPECT().FetchComponent(gomock.Any(), component, "/output", gomock.Any()).DoAndReturn( @@ -78,6 +84,10 @@ func TestPrepareSourcesGoVendor(t *testing.T) { assert.Contains(t, cmd.Args, "--enable-network") assert.Contains(t, strings.Join(cmd.Args, " "), "go_vendor_archive") staging := goVendorTestStaging(t, ctx) + stagedConfig, err := fileutils.ReadFile(ctx.FS(), filepath.Join(staging, govendor.ConfigFilename)) + require.NoError(t, err) + assert.Equal(t, settings, string(stagedConfig)) + prep, err := fileutils.ReadFile(ctx.FS(), filepath.Join(filepath.Dir(staging), "sources", "app.spec")) require.NoError(t, err) assert.Contains(t, string(prep), "Summary: overlaid", "overlays must run before generation") @@ -109,6 +119,10 @@ func TestPrepareSourcesGoVendor(t *testing.T) { assert.Contains(t, string(output), "tar -xf %{SOURCE1}") assert.NotContains(t, string(output), "BuildRequires: golang(") + publishedConfig, err := fileutils.ReadFile(ctx.FS(), "/output/"+govendor.ConfigFilename) + require.NoError(t, err) + assert.Equal(t, settings, string(publishedConfig)) + digest, err := fileutils.ComputeFileHash(ctx.FS(), fileutils.HashTypeSHA512, "/output/app-vendor.tar.bz2") require.NoError(t, err) sources, err := fileutils.ReadFile(ctx.FS(), "/output/sources") diff --git a/internal/fingerprint/doc.go b/internal/fingerprint/doc.go index fc655afa2..5ad05185b 100644 --- a/internal/fingerprint/doc.go +++ b/internal/fingerprint/doc.go @@ -3,7 +3,7 @@ // Package fingerprint computes deterministic identity fingerprints for components. // A fingerprint captures all resolved build inputs so that changes to any input -// (config fields, spec content, overlay files, distro context, upstream refs, or +// (config fields, spec content, overlay/vendor config files, distro context, upstream refs, or // ManualBump) produce a different fingerprint. // // The primary entry point is [ComputeIdentity], which takes a resolved diff --git a/internal/fingerprint/fingerprint.go b/internal/fingerprint/fingerprint.go index 391559d66..6ab13bb14 100644 --- a/internal/fingerprint/fingerprint.go +++ b/internal/fingerprint/fingerprint.go @@ -42,6 +42,9 @@ type ComponentInputs struct { // source file's basename and content. Keyed by index rather than path to avoid // checkout-location dependence. OverlayFileHashes map[string]string `json:"overlayFileHashes,omitempty"` + // GoVendorConfigHash records the contents of an explicitly selected vendor + // configuration without depending on its checkout-specific path. + GoVendorConfigHash string `json:"goVendorConfigHash,omitempty"` // ManualBump is the manual rebuild counter from the lock file. Almost always 0; // used for mass-rebuild scenarios. ManualBump int `json:"manualBump"` @@ -62,13 +65,13 @@ type IdentityOptions struct { // // This is caller-provided because resolving it requires network access (upstream clone) or // filesystem traversal (local content hash). [ComputeIdentity] is a pure combiner — it does - // not perform I/O beyond reading overlay files. Callers should resolve source identity via + // not perform I/O beyond reading declared input files. Callers should resolve source identity via // SourceManager.ResolveSourceIdentity before calling [ComputeIdentity]. SourceIdentity string } // ComputeIdentity computes the fingerprint for a component from its resolved config -// and additional context. The fs parameter is used to read overlay source file +// and additional context. The fs parameter is used to read declared input file // contents for hashing; spec content identity is provided via [IdentityOptions.SourceIdentity]. // // This function is a deterministic combiner: given the same resolved inputs it always @@ -141,6 +144,11 @@ func ComputeIdentity( inputs.OverlayFileHashes = overlayHashes + inputs.GoVendorConfigHash, err = component.GoVendor.SourceContentIdentity(fs) + if err != nil { + return nil, fmt.Errorf("hashing Go vendor configuration:\n%w", err) + } + // 5. Combine all inputs into the overall fingerprint. return &ComponentIdentity{ Fingerprint: combineInputs(inputs), @@ -158,6 +166,10 @@ func combineInputs(inputs ComponentInputs) string { writeField(hasher, "manual_bump", strconv.Itoa(inputs.ManualBump)) writeField(hasher, "release_ver", inputs.ReleaseVer) + if inputs.GoVendorConfigHash != "" { + writeField(hasher, "go_vendor_config", inputs.GoVendorConfigHash) + } + // Overlay file hashes in sorted key order for determinism. if len(inputs.OverlayFileHashes) > 0 { keys := make([]string, 0, len(inputs.OverlayFileHashes)) diff --git a/internal/fingerprint/govendor_test.go b/internal/fingerprint/govendor_test.go new file mode 100644 index 000000000..cd47c3838 --- /dev/null +++ b/internal/fingerprint/govendor_test.go @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package fingerprint_test + +import ( + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/fingerprint" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/samber/lo" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGoVendorConfigFileFingerprint(t *testing.T) { + ctx := newTestFS(t, map[string]string{ + "/checkout-one/vendor.toml": "[archive]\ntidy = false\n", + "/checkout-two/vendor.toml": "[archive]\ntidy = false\n", + "/empty.toml": "", + }) + component := projectconfig.ComponentConfig{GoVendor: &projectconfig.GoVendorConfig{ + Enabled: lo.ToPtr(true), ConfigFile: "/checkout-one/vendor.toml", + }} + compute := func(config projectconfig.ComponentConfig) *fingerprint.ComponentIdentity { + t.Helper() + + result, err := fingerprint.ComputeIdentity(ctx.FS(), config, testReleaseVer, fingerprint.IdentityOptions{}) + require.NoError(t, err) + + return result + } + initial := compute(component) + require.NotEmpty(t, initial.Inputs.GoVendorConfigHash) + assert.Equal(t, "/checkout-one/vendor.toml", component.GoVendor.ConfigFile) + component.GoVendor.ConfigFile = "/checkout-two/vendor.toml" + assert.Equal(t, initial.Fingerprint, compute(component).Fingerprint, "checkout location must not affect identity") + + require.NoError(t, fileutils.WriteFile(ctx.FS(), component.GoVendor.ConfigFile, + []byte("[archive]\ntidy = true\n"), fileperms.PublicFile)) + assert.NotEqual(t, initial.Fingerprint, compute(component).Fingerprint, + "file-only edits must invalidate the fingerprint") + component.GoVendor.ConfigFile = "/empty.toml" + emptyFile := compute(component) + component.GoVendor.ConfigFile = "" + noFile := compute(component) + assert.Empty(t, noFile.Inputs.GoVendorConfigHash) + assert.NotEqual(t, emptyFile.Fingerprint, noFile.Fingerprint, "an empty override is not upstream fallback") +} + +func TestGoVendorConfigFileFingerprintErrors(t *testing.T) { + ctx := newTestFS(t, map[string]string{"/bad.toml": "["}) + for _, path := range []string{"/missing.toml", "/bad.toml"} { + _, err := fingerprint.ComputeIdentity(ctx.FS(), projectconfig.ComponentConfig{ + GoVendor: &projectconfig.GoVendorConfig{Enabled: lo.ToPtr(true), ConfigFile: path}, + }, testReleaseVer, fingerprint.IdentityOptions{}) + require.Error(t, err) + assert.Contains(t, err.Error(), path) + } +} diff --git a/internal/govendor/generate.go b/internal/govendor/generate.go index 0f888ddfe..68c69889d 100644 --- a/internal/govendor/generate.go +++ b/internal/govendor/generate.go @@ -37,7 +37,7 @@ const ( type Request struct { SourceDir, WorkDir, SpecName string ArchiveFile, ArchiveSuffix string - ModuleDir, ToolConfig string + ModuleDir, ConfigContents string ExpectedLicense string PrepSpec []byte Packages []string @@ -90,7 +90,7 @@ func stage(fs opctx.FS, request Request, sourceCopy, staging string) error { for path, data := range map[string][]byte{ filepath.Join(sourceCopy, request.SpecName): request.PrepSpec, - filepath.Join(staging, ConfigFilename): []byte(request.ToolConfig), + filepath.Join(staging, ConfigFilename): []byte(request.ConfigContents), } { if err := fileutils.WriteFile(fs, path, data, fileperms.PublicFile); err != nil { return fmt.Errorf("staging Go vendor input %#q:\n%w", path, err) @@ -134,7 +134,7 @@ func execute(ctx context.Context, runner *mock.Runner, request Request) (err err output, err := cmd.RunAndGetOutput(ctx) if err != nil { return fmt.Errorf("go vendor generation or license detection failed; use 'prep-packages' for missing prep tools "+ - "and 'tool-config' for reviewed license exceptions:\n%s\n%w", output, err) + "and 'config-file' for reviewed license exceptions:\n%s\n%w", output, err) } return nil diff --git a/internal/projectconfig/component.go b/internal/projectconfig/component.go index d3b2404aa..be9e8c0ef 100644 --- a/internal/projectconfig/component.go +++ b/internal/projectconfig/component.go @@ -653,6 +653,9 @@ func (c *ComponentConfig) WithAbsolutePaths(referenceDir string) *ComponentConfi // Fix up paths. result.Spec.Path = makeAbsolute(referenceDir, result.Spec.Path) + if result.GoVendor != nil { + result.GoVendor.ConfigFile = makeAbsolute(referenceDir, result.GoVendor.ConfigFile) + } for i := range result.SourceFiles { origin := &result.SourceFiles[i].Origin diff --git a/internal/projectconfig/fingerprint_test.go b/internal/projectconfig/fingerprint_test.go index dcf89f830..881ea95fb 100644 --- a/internal/projectconfig/fingerprint_test.go +++ b/internal/projectconfig/fingerprint_test.go @@ -106,6 +106,9 @@ func TestAllFingerprintedFieldsHaveDecision(t *testing.T) { // SpecSource.Path — absolute path that varies by checkout location. // Spec content identity is captured separately via SourceIdentity. "SpecSource.Path": true, + + // GoVendorConfig.ConfigFile — absolute declaration path; contents are hashed separately. + "GoVendorConfig.ConfigFile": true, } // Collect all actual exclusions found via reflection, and flag invalid tag values. diff --git a/internal/projectconfig/govendor.go b/internal/projectconfig/govendor.go index cf9fc4eae..415a2a7c7 100644 --- a/internal/projectconfig/govendor.go +++ b/internal/projectconfig/govendor.go @@ -4,10 +4,13 @@ package projectconfig import ( + "crypto/sha256" + "encoding/hex" "errors" "fmt" "strings" + "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" "github.com/pelletier/go-toml/v2" ) @@ -18,8 +21,9 @@ type GoVendorConfig struct { Enabled *bool `toml:"enabled,omitempty" json:"enabled,omitempty" jsonschema:"title=Enabled,description=Enable Go application vendoring"` // ModuleDir is relative to the directory left by the spec's prep section. ModuleDir string `toml:"module-dir,omitempty" json:"moduleDir,omitempty" jsonschema:"title=Module directory,description=Relative module directory within the prepared source tree; defaults to its root"` - // ToolConfig is inline go-vendor-tools TOML, including reviewed license exceptions. - ToolConfig string `toml:"tool-config,omitempty" json:"toolConfig,omitempty" jsonschema:"title=Tool configuration,description=Inline go-vendor-tools TOML for dependency overrides and licensing settings"` + // ConfigFile identifies a native go-vendor-tools TOML file, relative to the declaring + // config file. Resolved to an absolute path; its contents are fingerprinted separately. + ConfigFile string `toml:"config-file,omitempty" json:"configFile,omitempty" jsonschema:"title=Vendor configuration file,description=Path to a go-vendor-tools TOML file relative to the declaring component configuration. Replaces upstream settings when specified,example=go-vendor-tools.toml" fingerprint:"-"` // PrepPackages supplies additional tools used by upstream prep commands. PrepPackages []string `toml:"prep-packages,omitempty" json:"prepPackages,omitempty" jsonschema:"title=Prep packages,description=Additional RPM packages needed to execute the upstream prep section in mock"` } @@ -29,7 +33,7 @@ func (c *GoVendorConfig) IsEnabled() bool { return c != nil && c.Enabled != nil && *c.Enabled } -// Validate checks paths and tool configuration before any source processing. +// Validate checks configuration values before any source processing. func (c *GoVendorConfig) Validate() error { if c == nil { return nil @@ -45,15 +49,8 @@ func (c *GoVendorConfig) Validate() error { } } - var toolConfig map[string]any - if err := toml.Unmarshal([]byte(c.ToolConfig), &toolConfig); err != nil { - return fmt.Errorf("invalid 'tool-config' TOML:\n%w", err) - } - - if general, ok := toolConfig["general"].(map[string]any); ok { - if _, exists := general["go_mod_dir"]; exists { - return errors.New("use 'module-dir' instead of 'general.go_mod_dir' in 'tool-config'") - } + if strings.ContainsAny(c.ConfigFile, "\x00\r\n") { + return errors.New("'config-file' must be a local file path without NUL or newlines") } for _, name := range c.PrepPackages { @@ -64,3 +61,42 @@ func (c *GoVendorConfig) Validate() error { return nil } + +// SourceContentIdentity hashes the selected configuration independently of its path. +// No configured file yields no identity; an explicitly empty file still has a hash. +func (c *GoVendorConfig) SourceContentIdentity(fs opctx.FS) (string, error) { + if c == nil || c.ConfigFile == "" { + return "", nil + } + + content, err := c.ReadConfigFile(fs) + if err != nil { + return "", err + } + + digest := sha256.Sum256([]byte(content)) + + return hex.EncodeToString(digest[:]), nil +} + +// ReadConfigFile reads and validates the explicitly selected native configuration. +// An empty file is a valid, intentional replacement; callers must not fall back to upstream settings. +func (c *GoVendorConfig) ReadConfigFile(fs opctx.FS) (string, error) { + data, err := fileutils.ReadFile(fs, c.ConfigFile) + if err != nil { + return "", fmt.Errorf("reading 'go-vendor.config-file' %#q:\n%w", c.ConfigFile, err) + } + + var settings map[string]any + if err := toml.Unmarshal(data, &settings); err != nil { + return "", fmt.Errorf("invalid TOML in 'go-vendor.config-file' %#q:\n%w", c.ConfigFile, err) + } + + if general, ok := settings["general"].(map[string]any); ok { + if _, exists := general["go_mod_dir"]; exists { + return "", fmt.Errorf("use 'module-dir' instead of 'general.go_mod_dir' in %#q", c.ConfigFile) + } + } + + return string(data), nil +} diff --git a/internal/projectconfig/govendor_loader_internal_test.go b/internal/projectconfig/govendor_loader_internal_test.go new file mode 100644 index 000000000..a0af07a33 --- /dev/null +++ b/internal/projectconfig/govendor_loader_internal_test.go @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package projectconfig + +import ( + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/global/testctx" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGoVendorConfigFileDeclarationAndInheritance(t *testing.T) { + ctx := testctx.NewCtx() + + files := map[string]string{ + "/project/azldev.toml": `includes = ["defaults/group.toml", "components/app.comp.toml"]`, + "/project/defaults/group.toml": ` +[component-groups.base] +components = ["app"] +[component-groups.base.default-component-config.go-vendor] +enabled = true +config-file = "go-vendor-tools.toml" +`, + "/project/components/app.comp.toml": `[components.app]`, + } + for path, content := range files { + require.NoError(t, fileutils.WriteFile(ctx.FS(), path, []byte(content), fileperms.PublicFile)) + } + + for _, withoutLockfile := range []bool{false, true} { + config, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{withoutLockfile: withoutLockfile}, + "/project/azldev.toml") + require.NoError(t, err) + resolved, err := ResolveComponentConfig(config.Components["app"], config.DefaultComponentConfig, + ComponentConfig{}, config.ComponentGroups, []string{"base"}) + require.NoError(t, err) + assert.Equal(t, "/project/defaults/go-vendor-tools.toml", resolved.GoVendor.ConfigFile) + assert.True(t, resolved.GoVendor.IsEnabled()) + } +} + +func TestGoVendorInlineConfigurationIsRejected(t *testing.T) { + for _, contents := range []string{ + "[components.app.go-vendor]\nenabled = true\ntool-config = '[archive]'", + "[components.app.go-vendor.tool-config.archive]\nuse_module_proxy = true", + } { + ctx := testctx.NewCtx() + require.NoError(t, fileutils.WriteFile(ctx.FS(), "/project/azldev.toml", []byte(contents), fileperms.PublicFile)) + _, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, "/project/azldev.toml") + require.Error(t, err) + } +} diff --git a/internal/projectconfig/govendor_test.go b/internal/projectconfig/govendor_test.go index e94ead8c6..02feb6943 100644 --- a/internal/projectconfig/govendor_test.go +++ b/internal/projectconfig/govendor_test.go @@ -4,9 +4,14 @@ package projectconfig_test import ( + "os" + "path/filepath" "testing" + "github.com/microsoft/azure-linux-dev-tools/internal/global/testctx" "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" "github.com/mitchellh/hashstructure/v2" "github.com/pelletier/go-toml/v2" "github.com/samber/lo" @@ -21,14 +26,14 @@ func TestGoVendorConfig(t *testing.T) { enabled = true module-dir = "cmd/app" prep-packages = ["git"] -tool-config = ''' -[archive.dependency_overrides] -"example.com/module" = "v1.2.3" -''' +config-file = "go-vendor-tools.toml" `), &config)) require.True(t, config.GoVendor.IsEnabled()) require.NoError(t, config.GoVendor.Validate()) - require.Equal(t, config.GoVendor, config.WithAbsolutePaths("/project").GoVendor) + resolved := config.WithAbsolutePaths("/project") + assert.Equal(t, filepath.Join("/project", "go-vendor-tools.toml"), resolved.GoVendor.ConfigFile) + assert.Equal(t, "go-vendor-tools.toml", config.GoVendor.ConfigFile) + assert.Equal(t, resolved.GoVendor, resolved.WithAbsolutePaths("/elsewhere").GoVendor) require.NoError(t, config.MergeUpdatesFrom(&projectconfig.ComponentConfig{ GoVendor: &projectconfig.GoVendorConfig{Enabled: lo.ToPtr(false)}, })) @@ -40,8 +45,8 @@ func TestGoVendorConfigValidation(t *testing.T) { {ModuleDir: "../outside"}, {ModuleDir: "/absolute"}, {ModuleDir: "%{macro}"}, - {ToolConfig: "["}, - {ToolConfig: "[general]\ngo_mod_dir = 'subdir'"}, + {ConfigFile: "bad\x00path"}, + {ConfigFile: "bad\npath"}, {PrepPackages: []string{"--bad"}}, {PrepPackages: []string{""}}, } { @@ -69,7 +74,6 @@ func TestGoVendorFingerprint(t *testing.T) { for _, config := range []*projectconfig.GoVendorConfig{ {Enabled: lo.ToPtr(false)}, {Enabled: lo.ToPtr(true), ModuleDir: "sub"}, - {Enabled: lo.ToPtr(true), ToolConfig: "[archive]\ntidy = false"}, {Enabled: lo.ToPtr(true), PrepPackages: []string{"git"}}, } { changed := base @@ -79,3 +83,47 @@ func TestGoVendorFingerprint(t *testing.T) { assert.NotEqual(t, original, hash) } } + +func TestGoVendorConfigFileContentIdentity(t *testing.T) { + ctx := testctx.NewCtx() + + var absent *projectconfig.GoVendorConfig + + identity, err := absent.SourceContentIdentity(ctx.FS()) + require.NoError(t, err) + assert.Empty(t, identity) + + config := &projectconfig.GoVendorConfig{} + identity, err = config.SourceContentIdentity(ctx.FS()) + require.NoError(t, err) + assert.Empty(t, identity) + + config.ConfigFile = "/project/vendor.toml" + _, err = config.SourceContentIdentity(ctx.FS()) + require.ErrorIs(t, err, os.ErrNotExist) + + for _, contents := range []string{"", "# reviewed\n[archive]\nuse_module_proxy = true\n"} { + require.NoError(t, fileutils.WriteFile(ctx.FS(), config.ConfigFile, []byte(contents), fileperms.PublicFile)) + actual, err := config.ReadConfigFile(ctx.FS()) + require.NoError(t, err) + assert.Equal(t, contents, actual) + + identity, err = config.SourceContentIdentity(ctx.FS()) + require.NoError(t, err) + expected, err := fileutils.ComputeFileHash(ctx.FS(), fileutils.HashTypeSHA256, config.ConfigFile) + require.NoError(t, err) + assert.Equal(t, expected, identity) + } +} + +func TestGoVendorConfigFileValidation(t *testing.T) { + for _, contents := range []string{"[", "[general]\ngo_mod_dir = 'subdir'"} { + ctx := testctx.NewCtx() + require.NoError(t, fileutils.WriteFile(ctx.FS(), "/project/vendor.toml", []byte(contents), fileperms.PublicFile)) + + config := projectconfig.GoVendorConfig{ConfigFile: "/project/vendor.toml"} + _, err := config.ReadConfigFile(ctx.FS()) + require.Error(t, err) + assert.Contains(t, err.Error(), config.ConfigFile) + } +} diff --git a/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap b/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap index 65c59d83a..9259ce684 100755 --- a/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap @@ -699,10 +699,13 @@ "title": "Module directory", "description": "Relative module directory within the prepared source tree; defaults to its root" }, - "tool-config": { + "config-file": { "type": "string", - "title": "Tool configuration", - "description": "Inline go-vendor-tools TOML for dependency overrides and licensing settings" + "title": "Vendor configuration file", + "description": "Path to a go-vendor-tools TOML file relative to the declaring component configuration. Replaces upstream settings when specified", + "examples": [ + "go-vendor-tools.toml" + ] }, "prep-packages": { "items": { diff --git a/schemas/azldev.schema.json b/schemas/azldev.schema.json index 65c59d83a..9259ce684 100644 --- a/schemas/azldev.schema.json +++ b/schemas/azldev.schema.json @@ -699,10 +699,13 @@ "title": "Module directory", "description": "Relative module directory within the prepared source tree; defaults to its root" }, - "tool-config": { + "config-file": { "type": "string", - "title": "Tool configuration", - "description": "Inline go-vendor-tools TOML for dependency overrides and licensing settings" + "title": "Vendor configuration file", + "description": "Path to a go-vendor-tools TOML file relative to the declaring component configuration. Replaces upstream settings when specified", + "examples": [ + "go-vendor-tools.toml" + ] }, "prep-packages": { "items": { From 26eb3559daec4c35288cdd3d8dee2979b652eaeb Mon Sep 17 00:00:00 2001 From: Antonio Salinas <173939267+Tonisal-byte@users.noreply.github.com> Date: Mon, 2 Nov 2026 12:58:50 -0800 Subject: [PATCH 4/5] fix: harden Go vendor source regeneration Address the eight unique findings in PR #424: reject source-file collisions, pin expanded archive names, snapshot regular chroot outputs without following symlinks, preserve module-aware checks, extract generated metadata after prep edits, reject ambiguous extraction layouts, honor native module subdirectories, and run archive hooks from staged dist-git. Add regression coverage and update user documentation and emitted agent guidance. Unit and quality checks pass; changed-code mutation audit kills all 37 mutations. The containerd update/prepare/render/build flow and isolated RPM daemon smoke test pass, including staged-sidecar hook checks. Ran ./magefile.go scenarioUpdate before pushing; the schema snapshot is current. Full Docker-dependent scenarios remain blocked by the unavailable Docker provider. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- docs/user/how-to/vendor-go-application.md | 19 ++- docs/user/reference/config/components.md | 12 +- .../app/azldev/agentskill/agentskill_test.go | 2 + .../agentskill/content/comp-toml.md.tmpl | 4 + .../content/withoutlockfile/comp-toml.md.tmpl | 4 + internal/app/azldev/core/sources/govendor.go | 8 +- .../govendor_existing_internal_test.go | 46 ++++++- internal/govendor/generate.go | 80 +++++++++++- internal/govendor/generate_internal_test.go | 123 ++++++++++++++++++ internal/rpm/spec/govendor.go | 29 +++-- internal/rpm/spec/govendor_existing.go | 30 ++++- internal/rpm/spec/govendor_existing_test.go | 46 ++++++- internal/rpm/spec/govendor_test.go | 5 +- internal/utils/fileutils/copy.go | 17 ++- internal/utils/fileutils/file.go | 26 ++++ internal/utils/fileutils/nofollow_test.go | 74 +++++++++++ 16 files changed, 488 insertions(+), 37 deletions(-) create mode 100644 internal/utils/fileutils/nofollow_test.go diff --git a/docs/user/how-to/vendor-go-application.md b/docs/user/how-to/vendor-go-application.md index 3892e8426..4cb103c32 100644 --- a/docs/user/how-to/vendor-go-application.md +++ b/docs/user/how-to/vendor-go-application.md @@ -59,6 +59,8 @@ there and runs the existing `%prep` section with `rpmbuild -bp --nodeps`. This retains upstream patches and edits to module files. It then calls `go_vendor_archive create` on the prepared module tree, extracts the result for license detection, and runs `go_vendor_license` without interactive prompts. +The archive tool runs from staged dist-git, so hooks can find preserved patches +and sidecars through `GVT_CWD`. Unrecognized licenses fail preparation; add reviewed, hash-qualified exceptions to the file selected by `config-file` rather than suppressing detection. @@ -102,11 +104,17 @@ For an already-vendored spec, azldev recognizes a single referenced by `%go_vendor_license_* -c %{S:M}` commands. The Source numbers need not be 1 and 2. The configuration must name a plain sidecar file. -The original Source declarations, extraction command, license/file-list macros, -module-mode settings, and custom packaging are retained. The temporary prep spec +The original Source numbers, license/file-list macros, module-mode settings, and +custom packaging are retained. The vendor extraction moves to the end of `%prep`, +after module-file patches and edits, because the replacement archive already +contains their results. Directory changes after the original extraction and +secondary-source extraction with `%setup -a/-b` or `%autosetup -a/-b` are rejected +as ambiguous; adapt these layouts with an overlay. The temporary prep spec omits the vendor Source declaration and its extraction command, so generating the replacement does not consume the old vendor payload. RPM expands the -upstream archive filename; the regenerated archive replaces that file and its +upstream archive filename and pins the vendor Source tag to that expanded name, +so subsequent release/history changes cannot desynchronize the spec and payload. +The regenerated archive replaces that file and its `sources` checksum rather than adding a second vendor archive. When `config-file` is omitted, the upstream go-vendor-tools configuration is used, @@ -114,6 +122,11 @@ including its reviewed license exceptions. An explicit file replaces that config even when empty; missing files or invalid TOML fail without falling back. The detected licenses are verified against the existing main-package `License:` expression; mismatches fail instead of rewriting it. +Upstream `general.go_mod_dir` is honored by generation and licensing, preserving +the module subdirectory within the archive. A conflicting `module-dir` fails; +use a separate `config-file` to replace upstream settings when changing the layout. +Neither the regenerated archive nor its configuration sidecar may also be +declared in `source-files`, even with `replace-upstream = true`. The source-fetch phase currently still downloads the upstream vendor archive alongside the other upstream sources, before regenerating its contents. diff --git a/docs/user/reference/config/components.md b/docs/user/reference/config/components.md index 530b9cea3..f483f671f 100644 --- a/docs/user/reference/config/components.md +++ b/docs/user/reference/config/components.md @@ -205,7 +205,9 @@ config-file = "go-vendor-tools.toml" The configured file must exist and contain valid TOML. An explicitly selected empty file is an empty replacement, not a request to fall back to upstream. -Use `module-dir`, not `general.go_mod_dir`, for the module location. +In an explicitly selected file, use `module-dir`, not `general.go_mod_dir`, for +the module location. Reused upstream `general.go_mod_dir` settings remain honored; +an inconsistent component `module-dir` is rejected. Configured command hooks execute in the generation chroot; treat this file as trusted build code. The old inline `tool-config` string/table is no longer supported. @@ -218,8 +220,14 @@ The recorded digest identifies that output, not a separately pinned vendor lock. For supported specs already using go-vendor-tools, preparation instead regenerates the existing vendor source, retaining its Source number, RPM-expanded filename, -and configuration sidecar. It updates that archive's `sources` entry in place and +and configuration sidecar. The Source tag is pinned to the generated filename +before release/history updates, and extraction moves after prep's module edits +to avoid applying patches twice. Ambiguous secondary-source setup and +post-extraction directory changes require a prep overlay. +It updates that archive's `sources` entry in place and verifies the existing `License:` expression without rewriting it. +These generated archive/config filenames must not also appear in `source-files`, +including entries with `replace-upstream = true`. When `config-file` is set, its contents are staged under the configuration filename referenced by the prepared spec, retaining the existing Source number. diff --git a/internal/app/azldev/agentskill/agentskill_test.go b/internal/app/azldev/agentskill/agentskill_test.go index 0836c6751..5805e5282 100644 --- a/internal/app/azldev/agentskill/agentskill_test.go +++ b/internal/app/azldev/agentskill/agentskill_test.go @@ -170,6 +170,8 @@ func TestComponentTomlSkillDocumentsGoVendoring(t *testing.T) { assert.Contains(t, doc, "reviewed license exceptions") assert.Contains(t, doc, `config-file = "go-vendor-tools.toml"`) assert.Contains(t, doc, "Inline `tool-config` is not supported.") + assert.Contains(t, doc, "Upstream `general.go_mod_dir` is honored") + assert.Contains(t, doc, "Source filename is pinned before") } } diff --git a/internal/app/azldev/agentskill/content/comp-toml.md.tmpl b/internal/app/azldev/agentskill/content/comp-toml.md.tmpl index b724ea0ef..7059b2ae3 100644 --- a/internal/app/azldev/agentskill/content/comp-toml.md.tmpl +++ b/internal/app/azldev/agentskill/content/comp-toml.md.tmpl @@ -53,6 +53,10 @@ Generated `-vendor.tar.bz2` and `azldev-go-vendor.toml` must not also declared as `source-files`. For supported already-vendored specs, regenerate the existing vendor Source instead, reuse upstream tool configuration unless overridden, and verify rather than replace the reviewed `License:` expression. +The regenerated archive and configuration must not also be `source-files` entries. +Upstream `general.go_mod_dir` is honored; conflicting `module-dir` values fail. +Vendor extraction runs after prep edits, and its Source filename is pinned before +release/history updates. Ambiguous secondary-source setup requires an overlay. Unsupported constructs (notably automatic Go devel subpackages, flagged `%gocheck`, or ambiguous sections) require explicit overlays; never replace the upstream spec wholesale to bypass a conversion error. diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl index cd0ec3e48..2ac8df159 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl @@ -53,6 +53,10 @@ Generated `-vendor.tar.bz2` and `azldev-go-vendor.toml` must not also declared as `source-files`. For supported already-vendored specs, regenerate the existing vendor Source instead, reuse upstream tool configuration unless overridden, and verify rather than replace the reviewed `License:` expression. +The regenerated archive and configuration must not also be `source-files` entries. +Upstream `general.go_mod_dir` is honored; conflicting `module-dir` values fail. +Vendor extraction runs after prep edits, and its Source filename is pinned before +release/history updates. Ambiguous secondary-source setup requires an overlay. Unsupported constructs (notably automatic Go devel subpackages, flagged `%gocheck`, or ambiguous sections) require explicit overlays; never replace the upstream spec wholesale to bypass a conversion error. diff --git a/internal/app/azldev/core/sources/govendor.go b/internal/app/azldev/core/sources/govendor.go index 7709ac8e7..fb0619758 100644 --- a/internal/app/azldev/core/sources/govendor.go +++ b/internal/app/azldev/core/sources/govendor.go @@ -89,6 +89,12 @@ func (p *sourcePreparerImpl) prepareGoVendor( return fmt.Errorf("generating Go vendor sources:\n%w", err) } + for _, ref := range component.GetConfig().SourceFiles { + if ref.Filename == artifact.Name || ref.Filename == plan.ConfigName { + return fmt.Errorf("'source-files' entry %#q conflicts with generated Go vendor sources", ref.Filename) + } + } + return p.publishGoVendor(outputDir, filepath.Base(specPath), toolConfig, plan, artifact) } @@ -129,7 +135,7 @@ func (p *sourcePreparerImpl) publishGoVendor( return fmt.Errorf("generated Go archive %#q does not match planned name %#q", artifact.Name, plan.ArchiveFile) } - finalSpec, err := plan.FinalizeLicense(artifact.License) + finalSpec, err := plan.Finalize(artifact.Name, artifact.License) if err != nil { return fmt.Errorf("finalizing Go vendor spec:\n%w", err) } diff --git a/internal/app/azldev/core/sources/govendor_existing_internal_test.go b/internal/app/azldev/core/sources/govendor_existing_internal_test.go index b9d47d92b..3ab85fd8a 100644 --- a/internal/app/azldev/core/sources/govendor_existing_internal_test.go +++ b/internal/app/azldev/core/sources/govendor_existing_internal_test.go @@ -23,12 +23,30 @@ import ( ) func TestPrepareExistingGoVendorReplacesPayload(t *testing.T) { + for _, conflict := range []string{"", "app-1.2-vendor.tar.bz2", "go-vendor-tools.toml"} { + t.Run("source-file-"+conflict, func(t *testing.T) { + testPrepareExistingGoVendor(t, conflict) + }) + } +} + +func testPrepareExistingGoVendor(t *testing.T, conflict string) { + t.Helper() + ctx := newMockProcessorTestCtx() component := components_testutils.NewMockComponent(gomock.NewController(t)) component.EXPECT().GetName().AnyTimes().Return("app") - component.EXPECT().GetConfig().AnyTimes().Return(&projectconfig.ComponentConfig{ + + config := &projectconfig.ComponentConfig{ GoVendor: &projectconfig.GoVendorConfig{Enabled: lo.ToPtr(true)}, - }) + } + if conflict != "" { + config.SourceFiles = []projectconfig.SourceFileReference{{ + Filename: conflict, ReplaceUpstream: true, Hash: strings.Repeat("a", 128), HashType: fileutils.HashTypeSHA512, + }} + } + + component.EXPECT().GetConfig().AnyTimes().Return(config) input := strings.Join([]string{ "Name: app", "Version: 1.2", "Release: 1", "License: MIT AND Apache-2.0", @@ -75,14 +93,34 @@ func TestPrepareExistingGoVendorReplacesPayload(t *testing.T) { return "", nil } preparer := &sourcePreparerImpl{fs: ctx.FS(), dryRunnable: ctx, goVendorRunner: mock.NewRunner(ctx, "/mock.cfg")} - require.NoError(t, preparer.prepareGoVendor(ctx, component, "/output")) + + err := preparer.prepareGoVendor(ctx, component, "/output") + if conflict != "" { + require.ErrorContains(t, err, "'source-files'") + + unchanged, readErr := fileutils.ReadFile(ctx.FS(), "/output/app.spec") + require.NoError(t, readErr) + assert.Equal(t, input, string(unchanged)) + + payload, readErr := fileutils.ReadFile(ctx.FS(), "/output/"+archiveName) + require.NoError(t, readErr) + assert.Equal(t, "old vendor", string(payload)) + + manifest, readErr := fileutils.ReadFile(ctx.FS(), "/output/sources") + require.NoError(t, readErr) + assert.Equal(t, oldEntry+"\n", string(manifest)) + + return + } + + require.NoError(t, err) payload, err := fileutils.ReadFile(ctx.FS(), "/output/"+archiveName) require.NoError(t, err) assert.Equal(t, "new vendor", string(payload)) finalSpec, err := fileutils.ReadFile(ctx.FS(), "/output/app.spec") require.NoError(t, err) - assert.Contains(t, string(finalSpec), "Source1: app-%{version}-vendor.tar.bz2") + assert.Contains(t, string(finalSpec), "Source1: "+archiveName) assert.Contains(t, string(finalSpec), "License: MIT AND Apache-2.0") assert.NotContains(t, string(finalSpec), "azldev-go-vendor.toml") diff --git a/internal/govendor/generate.go b/internal/govendor/generate.go index 68c69889d..3d8fa4a02 100644 --- a/internal/govendor/generate.go +++ b/internal/govendor/generate.go @@ -20,6 +20,7 @@ import ( "github.com/microsoft/azure-linux-dev-tools/internal/rpm/spectool" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/pelletier/go-toml/v2" ) const ( @@ -41,6 +42,7 @@ type Request struct { ExpectedLicense string PrepSpec []byte Packages []string + configModuleDir string } // Artifact is validated generation output, ready for publication by the caller. @@ -51,6 +53,11 @@ type Artifact struct { // Generate prepares inputs and generates an archive in one isolated mock root. // It never changes SourceDir or publishes a spec or source manifest. func Generate(ctx context.Context, fs opctx.FS, base *mock.Runner, request Request) (Artifact, error) { + request, err := resolveModuleDir(request) + if err != nil { + return Artifact{}, err + } + staging := filepath.Join(request.WorkDir, "work") sourceCopy := filepath.Join(request.WorkDir, "sources") @@ -65,11 +72,73 @@ func Generate(ctx context.Context, fs opctx.FS, base *mock.Runner, request Reque return Artifact{}, err } - if err := validateInputs(fs, staging, request.SourceDir); err != nil { + // Snapshot only regular outputs into a host-only directory. Subsequent + // validation, hashing and publication never reopen chroot-writable paths. + trusted := filepath.Join(request.WorkDir, "artifacts") + if err := fileutils.MkdirAll(fs, trusted); err != nil { + return Artifact{}, fmt.Errorf("creating trusted Go vendor output directory:\n%w", err) + } + + for _, name := range []string{request.ArchiveFile, "archive-name", "license", "sources-list"} { + if err := fileutils.CopyFileCrossFS(nilDryRun{}, fs, filepath.Join(staging, name), + fs, filepath.Join(trusted, name), fileutils.CopyFileOptions{NoFollow: true}); err != nil { + return Artifact{}, fmt.Errorf("collecting Go vendor output %#q:\n%w", name, err) + } + } + + if err := validateInputs(fs, trusted, request.SourceDir); err != nil { return Artifact{}, err } - return readArtifact(fs, staging, request) + return readArtifact(fs, trusted, request) +} + +type nilDryRun struct{} + +func (nilDryRun) DryRun() bool { return false } + +func resolveModuleDir(request Request) (Request, error) { + var settings map[string]any + if err := toml.Unmarshal([]byte(request.ConfigContents), &settings); err != nil { + return request, fmt.Errorf("parsing Go vendor tool configuration:\n%w", err) + } + + general, _ := settings["general"].(map[string]any) + if raw, exists := general["go_mod_dir"]; exists { + directory, ok := raw.(string) + if !ok { + return request, errors.New("go vendor 'general.go_mod_dir' must be a string") + } + + request.configModuleDir = directory + } + + for _, directory := range []string{request.ModuleDir, request.configModuleDir} { + if directory == "" || directory == "." { + continue + } + + if err := fileutils.ValidateRelPath(directory); err != nil { + return request, fmt.Errorf("invalid Go vendor module directory:\n%w", err) + } + + if strings.ContainsAny(directory, "%\r\n") { + return request, errors.New("go vendor module directory must not contain RPM macros or newlines") + } + } + + if request.configModuleDir != "" { + if request.ModuleDir != "" && request.ModuleDir != request.configModuleDir { + return request, errors.New("'module-dir' conflicts with upstream 'general.go_mod_dir'; " + + "use 'config-file' to replace the upstream settings") + } + + // Native tools preserve this subdirectory in the archive and use it for + // licensing. Keep their root and config intact, without descending twice. + request.ModuleDir = "" + } + + return request, nil } func stage(fs opctx.FS, request Request, sourceCopy, staging string) error { @@ -169,10 +238,13 @@ func generationCommand(request Request, targetArch string) []string { `IFS= read -r module_root < ` + topDir + "/azldev-go-module-root", `cd "$module_root"`, "cd " + quote(cmp.Or(request.ModuleDir, ".")), - "test -f go.mod", + quote("test", "-f", filepath.Join(request.configModuleDir, "go.mod")), + `module_root=$(pwd -P)`, "export GOTOOLCHAIN=local", + "cd " + sourceMount, quote("go_vendor_archive", "create", "--config", outputMount+"/"+ConfigFilename, - "--output", outputMount+"/"+request.ArchiveFile, "."), + "--output", outputMount+"/"+request.ArchiveFile) + ` "$module_root"`, + `cd "$module_root"`, quote("tar", "-xf", outputMount+"/"+request.ArchiveFile), quote(licenseArgs...) + " > " + outputMount + "/license", "cd " + sourceMount, diff --git a/internal/govendor/generate_internal_test.go b/internal/govendor/generate_internal_test.go index 68df88413..1d3f7b49e 100644 --- a/internal/govendor/generate_internal_test.go +++ b/internal/govendor/generate_internal_test.go @@ -4,6 +4,7 @@ package govendor import ( + "os" "os/exec" "path/filepath" "strings" @@ -54,6 +55,8 @@ func TestGenerateArtifact(t *testing.T) { } require.NoError(t, err) + assert.Equal(t, "/scratch/artifacts/vendor.tar.bz2", artifact.Path, + "published artifacts must come from the host-only snapshot") assert.Equal(t, "app-vendor.tar.bz2", artifact.Name) assert.Equal(t, "MIT", artifact.License) digest, err := fileutils.ComputeFileHash(ctx.FS(), fileutils.HashTypeSHA512, artifact.Path) @@ -78,6 +81,9 @@ func TestGenerationCommand(t *testing.T) { assert.Less(t, strings.Index(script, "rpmbuild"), strings.Index(script, "go_vendor_archive")) assert.Less(t, strings.Index(script, "go_vendor_archive"), strings.Index(script, "go_vendor_license")) assert.Contains(t, script, "cd cmd/app") + assert.Contains(t, script, + "cd "+sourceMount+"\ngo_vendor_archive create --config "+outputMount+"/"+ConfigFilename+ + " --output "+outputMount+`/app-vendor.tar.bz2 "$module_root"`+"\n"+`cd "$module_root"`) assert.Contains(t, script, "--target aarch64") assert.Contains(t, script, "--no-prompt") assert.Contains(t, script, "--autofill off") @@ -86,3 +92,120 @@ func TestGenerationCommand(t *testing.T) { assert.Equal(t, expected != "", strings.Contains(script, "--verify")) } } + +func TestResolveModuleDir(t *testing.T) { + for _, testCase := range []struct { + name, config, module, want string + wantErr bool + }{ + {name: "default"}, + {name: "upstream", config: "[general]\ngo_mod_dir = 'cmd/app'\n[archive]\ntidy = false"}, + {name: "matching", config: "[general]\ngo_mod_dir = 'cmd/app'", module: "cmd/app"}, + {name: "conflicting", config: "[general]\ngo_mod_dir = 'upstream'", module: "cmd/app", wantErr: true}, + {name: "explicit", module: "cmd/app", want: "cmd/app"}, + {name: "root", config: "[general]\ngo_mod_dir = '.'"}, + {name: "absolute", config: "[general]\ngo_mod_dir = '/etc'", wantErr: true}, + {name: "traversal", config: "[general]\ngo_mod_dir = '../outside'", wantErr: true}, + {name: "macro", config: "[general]\ngo_mod_dir = '%{bad}'", wantErr: true}, + {name: "type", config: "[general]\ngo_mod_dir = 42", wantErr: true}, + {name: "toml", config: "[broken", wantErr: true}, + } { + t.Run(testCase.name, func(t *testing.T) { + original := Request{ModuleDir: testCase.module, ConfigContents: testCase.config} + + actual, err := resolveModuleDir(original) + if testCase.wantErr { + require.Error(t, err) + + return + } + + require.NoError(t, err) + assert.Equal(t, testCase.want, actual.ModuleDir) + assert.Equal(t, testCase.config, actual.ConfigContents) + assert.Equal(t, testCase.config, original.ConfigContents) + }) + } +} + +// Kernel-enforced O_NOFOLLOW cannot be exercised by the in-memory filesystem. +func TestGenerateRejectsSymlinkOutputs(t *testing.T) { + for _, name := range []string{"vendor.tar.bz2", "archive-name", "license", "sources-list"} { + t.Run(name, func(t *testing.T) { + ctx := testctx.NewCtx(testctx.WithHostFS()) + ctx.CmdFactory.RegisterCommandInSearchPath(mock.MockBinary) + + root := t.TempDir() + request := Request{ + SourceDir: filepath.Join(root, "input"), WorkDir: filepath.Join(root, "scratch"), SpecName: "app.spec", + ArchiveFile: "vendor.tar.bz2", ArchiveSuffix: ".tar.bz2", PrepSpec: []byte("prepared"), + } + require.NoError(t, fileutils.MkdirAll(ctx.FS(), request.SourceDir)) + require.NoError(t, fileutils.WriteFile(ctx.FS(), filepath.Join(request.SourceDir, "app.spec"), + []byte("original"), fileperms.PublicFile)) + target := filepath.Join(root, "host-only") + require.NoError(t, fileutils.WriteFile(ctx.FS(), target, []byte("host-only contents"), fileperms.PublicFile)) + ctx.CmdFactory.RunAndGetOutputHandler = func(_ *exec.Cmd) (string, error) { + for filename, content := range map[string]string{ + request.ArchiveFile: "archive", "archive-name": "app-vendor.tar.bz2\n", + "license": "MIT\n", "sources-list": "Source0: app.spec\n", + } { + path := filepath.Join(request.WorkDir, "work", filename) + if filename == name { + require.NoError(t, os.Symlink(target, path)) + } else { + require.NoError(t, fileutils.WriteFile(ctx.FS(), path, []byte(content), fileperms.PublicFile)) + } + } + + return "", nil + } + artifact, err := Generate(ctx, ctx.FS(), mock.NewRunner(ctx, "/mock.cfg"), request) + require.ErrorContains(t, err, "without following symlinks") + assert.Empty(t, artifact.Path) + }) + } +} + +func TestCollectOutputRejectsNonRegularFile(t *testing.T) { + ctx := testctx.NewCtx() + require.NoError(t, fileutils.MkdirAll(ctx.FS(), "/input/directory")) + err := fileutils.CopyFile(ctx, ctx.FS(), "/input/directory", "/output/file", + fileutils.CopyFileOptions{NoFollow: true}) + require.ErrorContains(t, err, "not a regular file") +} + +func TestGenerateUsesUpstreamModuleWithoutChangingInputs(t *testing.T) { + ctx := testctx.NewCtx() + ctx.CmdFactory.RegisterCommandInSearchPath(mock.MockBinary) + require.NoError(t, fileutils.WriteFile(ctx.FS(), "/input/app.spec", []byte("original"), fileperms.PublicFile)) + + settings := "[general]\ngo_mod_dir = 'cmd/app'\n" + ctx.CmdFactory.RunAndGetOutputHandler = func(command *exec.Cmd) (string, error) { + script := strings.Join(command.Args, " ") + assert.Contains(t, script, "test -f cmd/app/go.mod") + assert.NotContains(t, script, "cd cmd/app") + + config, err := fileutils.ReadFile(ctx.FS(), "/scratch/work/"+ConfigFilename) + require.NoError(t, err) + assert.Equal(t, settings, string(config)) + + for name, content := range map[string]string{ + "vendor.tar.bz2": "archive", "archive-name": "app-vendor.tar.bz2\n", + "license": "MIT\n", "sources-list": "Source0: app.spec\n", + } { + require.NoError(t, fileutils.WriteFile(ctx.FS(), filepath.Join("/scratch/work", name), + []byte(content), fileperms.PublicFile)) + } + + return "", nil + } + request := Request{ + SourceDir: "/input", WorkDir: "/scratch", SpecName: "app.spec", + ArchiveFile: "vendor.tar.bz2", ConfigContents: settings, + } + _, err := Generate(ctx, ctx.FS(), mock.NewRunner(ctx, "/mock.cfg"), request) + require.NoError(t, err) + assert.Equal(t, settings, request.ConfigContents) + assert.Empty(t, request.ModuleDir) +} diff --git a/internal/rpm/spec/govendor.go b/internal/rpm/spec/govendor.go index 492faec23..2422df37e 100644 --- a/internal/rpm/spec/govendor.go +++ b/internal/rpm/spec/govendor.go @@ -29,6 +29,7 @@ type GoVendorSpec struct { ArchiveFile string // Filename used inside generation staging. ArchiveSuffix string // Required suffix of the RPM-expanded destination name, if any. ConfigName string + ArchiveSource string // Existing Source tag to pin to the generated filename. ExpectedLicense string // Empty: detect and update. Otherwise: verify and preserve. AllowReplacement bool // Permit replacing the existing vendor payload and configuration. } @@ -73,7 +74,7 @@ func ConvertGoVendor(data []byte, options GoVendorOptions) (*GoVendorSpec, error return nil, err } - if err := convertGoVendorLines(tree); err != nil { + if err := convertGoVendorLines(tree, options.ModuleDir); err != nil { return nil, err } @@ -189,20 +190,24 @@ func goVendorPrepSpec(tree *specTree, archiveValue string) []byte { return result } -// FinalizeLicense preserves a verified upstream expression or writes the detected -// expression for a newly converted spec. It does not mutate the plan. -func (p *GoVendorSpec) FinalizeLicense(expression string) ([]byte, error) { - if p.ExpectedLicense != "" { - return p.Spec, nil - } - +// Finalize pins an existing archive reference before release/history processing, +// and updates the license only for newly converted specs. It does not mutate the plan. +func (p *GoVendorSpec) Finalize(archiveName, expression string) ([]byte, error) { editor, err := openStructuralSpec(strings.NewReader(string(p.Spec))) if err != nil { return nil, err } - if err := editor.UpdateExistingTag("", "License", expression); err != nil { - return nil, err + if p.ArchiveSource != "" { + if err := editor.UpdateExistingTag("", p.ArchiveSource, archiveName); err != nil { + return nil, err + } + } + + if p.ExpectedLicense == "" { + if err := editor.UpdateExistingTag("", "License", expression); err != nil { + return nil, err + } } return []byte(strings.Join(editor.rawLines, "\n") + "\n"), nil @@ -298,7 +303,7 @@ func validateGoVendorBlock(blk *block) error { return nil } -func convertGoVendorLines(tree *specTree) error { +func convertGoVendorLines(tree *specTree, moduleDir string) error { return tree.VisitAllLines(func(section, _ string, line *lineHandle) error { text := strings.TrimSpace(line.Text) if strings.HasPrefix(text, "#") { @@ -326,7 +331,7 @@ func convertGoVendorLines(tree *specTree) error { return fmt.Errorf("unsupported Go check command %#q; use a module-aware check overlay", text) } - line.Replace("%gotest ./...") + line.Replace(goVendorInModule(moduleDir, "%gotest ./...")) } return nil diff --git a/internal/rpm/spec/govendor_existing.go b/internal/rpm/spec/govendor_existing.go index b7a237c0a..3013eacb7 100644 --- a/internal/rpm/spec/govendor_existing.go +++ b/internal/rpm/spec/govendor_existing.go @@ -18,6 +18,8 @@ var ( existingVendorLicense = regexp.MustCompile( `^%go_vendor_license_(?:buildrequires|install|check)\s+-c\s+%\{(?:S:|SOURCE)([0-9]+)\}\s*$`) existingVendorExtract = regexp.MustCompile(`^tar\s+-xf\s+%\{(?:S:|SOURCE)([0-9]+)\}\s*$`) + existingVendorSetup = regexp.MustCompile(`%(?:\{[!?]*)?(?:(?:auto)?setup|goprep)\b.*\s-[a-zA-Z]*[ab]`) + existingVendorChdir = regexp.MustCompile(`^(?:cd|pushd|popd|%(?:auto)?setup|%goprep)\b`) ) func prepareExistingGoVendor(tree *specTree, configSource string) (*GoVendorSpec, error) { @@ -34,7 +36,7 @@ func prepareExistingGoVendor(tree *specTree, configSource string) (*GoVendorSpec return nil, fmt.Errorf("existing Go vendor config Source%s must name a plain file, got %#q", configSource, configName) } - extraction, sourceNumber, err := existingVendorExtraction(tree) + extraction, sourceNumber, err := takeExistingVendorExtraction(tree) if err != nil { return nil, err } @@ -65,6 +67,9 @@ func prepareExistingGoVendor(tree *specTree, configSource string) (*GoVendorSpec } } + // Regeneration already includes prep's module edits. Extract the resulting + // module files after those edits so the RPM build cannot apply them twice. + tree.Section("%prep", "").AppendLines([]string{extraction}) addGoVendorEnvironment(tree) finalSpec := serializeGoVendorTree(tree) @@ -79,7 +84,7 @@ func prepareExistingGoVendor(tree *specTree, configSource string) (*GoVendorSpec return &GoVendorSpec{ Spec: finalSpec, PrepSpec: goVendorPrepSpec(tree, archiveValue), ArchiveFile: "azldev-generated-vendor" + suffix, ArchiveSuffix: suffix, ConfigName: configName, - ExpectedLicense: license, AllowReplacement: true, + ArchiveSource: "Source" + sourceNumber, ExpectedLicense: license, AllowReplacement: true, }, nil } @@ -126,19 +131,34 @@ func existingVendorConfigSource(tree *specTree) (string, error) { return source, nil } -func existingVendorExtraction(tree *specTree) (line, source string, err error) { +func takeExistingVendorExtraction(tree *specTree) (line, source string, err error) { count := 0 err = tree.Section("%prep", "").VisitLines(func(candidate *lineHandle) error { - matches := existingVendorExtract.FindStringSubmatch(strings.TrimSpace(candidate.Text)) + text := strings.TrimSpace(candidate.Text) + if existingVendorSetup.MatchString(text) { + return errors.New("secondary-source extraction in '%setup' or '%autosetup' makes Go vendoring ambiguous") + } + + if count > 0 && existingVendorChdir.MatchString(text) { + return errors.New("directory changes after Go vendor extraction require an explicit prep overlay") + } + + matches := existingVendorExtract.FindStringSubmatch(text) + if strings.HasPrefix(text, "tar ") && strings.Contains(text, "%{S") && len(matches) == 0 { + return fmt.Errorf("unsupported secondary-source extraction %#q", text) + } + if len(matches) != 0 { if !slices.Contains(tree.Section("%prep", "").block.Children, candidate.block) { return errors.New("conditional Go vendor extraction requires an explicit preparation strategy") } count++ - line = strings.TrimSpace(candidate.Text) + line = text source = matches[1] + + candidate.Remove() } return nil diff --git a/internal/rpm/spec/govendor_existing_test.go b/internal/rpm/spec/govendor_existing_test.go index 3b1927f95..280c9b4c0 100644 --- a/internal/rpm/spec/govendor_existing_test.go +++ b/internal/rpm/spec/govendor_existing_test.go @@ -61,8 +61,13 @@ func TestExistingGoVendoringPreservesPackaging(t *testing.T) { func TestExistingGoVendoringRejectsAmbiguousInputs(t *testing.T) { for name, input := range map[string]string{ - "multiple extractions": strings.ReplaceAll(existingVendorSpec(), "%goprep -p1", "%goprep -p1\ntar -xf %{S:9}"), - "no extraction": strings.ReplaceAll(existingVendorSpec(), "tar -xf %{S:5}", ""), + "multiple extractions": strings.ReplaceAll(existingVendorSpec(), "%goprep -p1", "%goprep -p1\ntar -xf %{S:9}"), + "autosetup secondary": strings.ReplaceAll(existingVendorSpec(), "%goprep -p1", "%autosetup -a 1"), + "setup secondary": strings.ReplaceAll(existingVendorSpec(), "%goprep -p1", "%setup -qb1"), + "braced autosetup secondary": strings.ReplaceAll(existingVendorSpec(), "%goprep -p1", "%{autosetup -a 1}"), + "other tar flags": strings.ReplaceAll(existingVendorSpec(), "%goprep -p1", "%goprep -p1\ntar -xjf %{S:9}"), + "directory after extraction": strings.ReplaceAll(existingVendorSpec(), "sed -i 's/old/new/' go.mod", "cd submodule"), + "no extraction": strings.ReplaceAll(existingVendorSpec(), "tar -xf %{S:5}", ""), "mismatched config": strings.ReplaceAll(existingVendorSpec(), "%go_vendor_license_check -c %{S:8}", "%go_vendor_license_check -c %{S:9}"), "dynamic config": strings.ReplaceAll(existingVendorSpec(), "go-vendor-tools.toml", "%{name}.toml"), @@ -85,3 +90,40 @@ func TestExistingGoVendoringRejectsAmbiguousInputs(t *testing.T) { }) } } + +func TestExistingGoVendorExtractsAfterModulePatches(t *testing.T) { + input := strings.ReplaceAll(existingVendorSpec(), "%goprep -p1", "%setup -q") + input = strings.ReplaceAll(input, "sed -i 's/old/new/' go.mod", "%autopatch -p1\nsed -i 's/old/new/' go.mod") + result, err := spec.ConvertGoVendor([]byte(input), spec.GoVendorOptions{ + ArchiveName: "app-vendor.tar.bz2", ConfigName: "vendor.toml", + }) + require.NoError(t, err) + + output := string(result.Spec) + assert.Less(t, strings.Index(output, "%autopatch -p1"), strings.Index(output, "tar -xf %{S:5}")) + assert.Less(t, strings.Index(output, "sed -i"), strings.Index(output, "tar -xf %{S:5}")) + assert.Less(t, strings.Index(output, "cp %{S:9} ."), strings.Index(output, "tar -xf %{S:5}")) + assert.Contains(t, string(result.PrepSpec), "%autopatch -p1") + assert.NotContains(t, string(result.PrepSpec), "tar -xf %{S:5}") +} + +func TestExistingGoVendorPinsExpandedArchiveBeforeReleaseChanges(t *testing.T) { + input := strings.ReplaceAll(existingVendorSpec(), "app-%{version}-vendor", "app-%{version}-%{release}-vendor") + result, err := spec.ConvertGoVendor([]byte(input), spec.GoVendorOptions{ + ArchiveName: "app-vendor.tar.bz2", ConfigName: "vendor.toml", + }) + require.NoError(t, err) + final, err := result.Finalize("app-1.2-7-vendor.tar.bz2", "MIT AND Apache-2.0") + require.NoError(t, err) + assert.Contains(t, string(final), "Source5: app-1.2-7-vendor.tar.bz2") + assert.NotContains(t, string(final), "app-%{version}-%{release}-vendor") + assert.Contains(t, string(result.Spec), "Source5: app-%{version}-%{release}-vendor.tar.bz2", + "finalization must not mutate the plan") + editor, err := spec.OpenSpec(strings.NewReader(string(final)), spec.WithEditor(spec.EditorStructural)) + require.NoError(t, err) + require.NoError(t, editor.UpdateExistingTag("", "Release", "8")) + + var output strings.Builder + require.NoError(t, editor.Serialize(&output)) + assert.Contains(t, output.String(), "Source5: app-1.2-7-vendor.tar.bz2") +} diff --git a/internal/rpm/spec/govendor_test.go b/internal/rpm/spec/govendor_test.go index c366de034..a29a44a3f 100644 --- a/internal/rpm/spec/govendor_test.go +++ b/internal/rpm/spec/govendor_test.go @@ -78,7 +78,8 @@ func TestConvertGoVendorPreservesUpstream(t *testing.T) { "%package tools", "Requires: app = %{version}-%{release}", "%goprep -p1", "sed -i 's/old/new/' go.mod", "echo custom-build-dependency", "%gobuild -o app %{goipath}", "install -D app %{buildroot}%{_bindir}/app", - "%if %{with check}\n%gotest ./...\n./custom-test\n%endif", + "%if %{with check}\n" + + `( cd "$(cat "%{_topdir}/azldev-go-module-root")"` + "\n%gotest ./...\n)\n./custom-test\n%endif", "%files tools\n%{_bindir}/tool", "%autochangelog", "Source2: app-vendor.tar.bz2", "Source3: vendor.toml", "tar -xf %{SOURCE2}", "%go_vendor_license_buildrequires -c %{SOURCE3}", @@ -153,6 +154,8 @@ func TestConvertGoVendorModuleDirAndStaticRequires(t *testing.T) { require.NoError(t, err) assert.Contains(t, string(result.Spec), "tar -xf %{SOURCE2} -C cmd/app") assert.Contains(t, string(result.Spec), "cd cmd/app\n%go_vendor_license_install") + assert.Contains(t, string(result.Spec), + `( cd "$(cat "%{_topdir}/azldev-go-module-root")"`+"\ncd cmd/app\n%gotest ./...\n)") assert.Contains(t, string(result.Spec), `( cd "$(cat "%{_topdir}/azldev-go-module-root")"`) assert.Contains(t, string(result.Spec), "%generate_buildrequires\n") } diff --git a/internal/utils/fileutils/copy.go b/internal/utils/fileutils/copy.go index 790822976..d0002e7c0 100644 --- a/internal/utils/fileutils/copy.go +++ b/internal/utils/fileutils/copy.go @@ -18,11 +18,14 @@ import ( type CopyFileOptions struct { // Whether or not to preserve POSIX mode bits on files. Does *not* apply to directories. PreserveFileMode bool + // NoFollow requires a regular source file and rejects final-component symlinks. + NoFollow bool } // Copies the contents of an existing file to a new file at the given destination path. // May preserve mode bits if "options" says so but does *not* preserve ownership, timestamps, -// or other metadata. *Does* follow symlinks. Is enlightened to skip copying in dry-run mode. +// or other metadata. Follows source symlinks unless [CopyFileOptions.NoFollow] is set. +// Is enlightened to skip copying in dry-run mode. func CopyFile(dryRunnable opctx.DryRunnable, fs opctx.FS, sourcePath, destPath string, options CopyFileOptions) error { return CopyFileCrossFS(dryRunnable, fs, sourcePath, fs, destPath, options) } @@ -30,7 +33,8 @@ func CopyFile(dryRunnable opctx.DryRunnable, fs opctx.FS, sourcePath, destPath s // Copies the contents of an existing file to a new file at the given destination path, // potentially across two [opctx.FS] filesystem instances. May preserve mode bits if // "options" says so but does *not* preserve ownership, timestamps, or other metadata. -// *Does* follow symlinks. Is enlightened to skip copying in dry-run mode. +// Follows source symlinks unless [CopyFileOptions.NoFollow] is set. +// Is enlightened to skip copying in dry-run mode. func CopyFileCrossFS( dryRunnable opctx.DryRunnable, sourceFS opctx.FS, sourcePath string, @@ -67,7 +71,14 @@ func copyFileCrossFS( destFileMode = stat.Mode() } - sourceFile, err := sourceFS.Open(sourcePath) + var sourceFile opctx.File + + if options.NoFollow { + sourceFile, err = OpenRegularFileNoFollow(sourceFS, sourcePath) + } else { + sourceFile, err = sourceFS.Open(sourcePath) + } + if err != nil { err = fmt.Errorf("failed to open copy source '%s':\n%w", sourcePath, err) diff --git a/internal/utils/fileutils/file.go b/internal/utils/fileutils/file.go index 52fbcbd3e..50e9d5dd3 100644 --- a/internal/utils/fileutils/file.go +++ b/internal/utils/fileutils/file.go @@ -14,6 +14,7 @@ import ( "github.com/bmatcuk/doublestar/v4" "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx" "github.com/spf13/afero" + "golang.org/x/sys/unix" ) // Adaptation of [os.WriteFile] that works with [opctx.FS]. @@ -28,6 +29,31 @@ func ReadFile(fs opctx.FS, filename string) ([]byte, error) { return afero.ReadFile(fs, filename) } +// OpenRegularFileNoFollow atomically rejects final-component symlinks and checks +// the opened descriptor rather than a path that could change before reading. +// The parent directory must be trusted. Nonblocking mode avoids opening a FIFO indefinitely. +func OpenRegularFileNoFollow(fs opctx.FS, filename string) (opctx.File, error) { + file, err := fs.OpenFile(filename, os.O_RDONLY|unix.O_NOFOLLOW|unix.O_NONBLOCK, 0) + if err != nil { + return nil, fmt.Errorf("opening regular file %#q without following symlinks:\n%w", filename, err) + } + + info, err := file.Stat() + if err != nil { + file.Close() + + return nil, fmt.Errorf("checking opened file %#q:\n%w", filename, err) + } + + if !info.Mode().IsRegular() { + file.Close() + + return nil, fmt.Errorf("%#q is not a regular file", filename) + } + + return file, nil +} + // Adaptation of [filepath.Glob] that works with [opctx.FS] and also uses the // [doublestar] package to supports standard double-star globs ('**'). func Glob(fs opctx.FS, pattern string, opts ...doublestar.GlobOption) ([]string, error) { diff --git a/internal/utils/fileutils/nofollow_test.go b/internal/utils/fileutils/nofollow_test.go new file mode 100644 index 000000000..9e87e3afc --- /dev/null +++ b/internal/utils/fileutils/nofollow_test.go @@ -0,0 +1,74 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package fileutils_test + +import ( + "io" + "os" + "path/filepath" + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx/opctx_test" + "github.com/microsoft/azure-linux-dev-tools/internal/global/testctx" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + "golang.org/x/sys/unix" +) + +func TestOpenRegularFileNoFollow(t *testing.T) { + ctx := testctx.NewCtx() + require.NoError(t, fileutils.WriteFile(ctx.FS(), "/regular", []byte("contents"), fileperms.PublicFile)) + file, err := fileutils.OpenRegularFileNoFollow(ctx.FS(), "/regular") + require.NoError(t, err) + + defer file.Close() + + data, err := io.ReadAll(file) + require.NoError(t, err) + assert.Equal(t, "contents", string(data)) + + missing, err := fileutils.OpenRegularFileNoFollow(ctx.FS(), "/missing") + require.ErrorIs(t, err, os.ErrNotExist) + assert.Nil(t, missing) + require.NoError(t, fileutils.MkdirAll(ctx.FS(), "/directory")) + directory, err := fileutils.OpenRegularFileNoFollow(ctx.FS(), "/directory") + require.ErrorContains(t, err, "not a regular file") + assert.Nil(t, directory) +} + +func TestOpenRegularFileNoFollowFlagsAndStatErrors(t *testing.T) { + ctrl := gomock.NewController(t) + mockFS := opctx_test.NewMockFS(ctrl) + flags := os.O_RDONLY | unix.O_NOFOLLOW | unix.O_NONBLOCK + mockFS.EXPECT().OpenFile("/denied", flags, os.FileMode(0)).Return(nil, os.ErrPermission) + file, err := fileutils.OpenRegularFileNoFollow(mockFS, "/denied") + require.ErrorIs(t, err, os.ErrPermission) + assert.Nil(t, file) + + opened := opctx_test.NewMockFile(ctrl) + mockFS.EXPECT().OpenFile("/bad-stat", flags, os.FileMode(0)).Return(opened, nil) + opened.EXPECT().Stat().Return(nil, os.ErrPermission) + opened.EXPECT().Close().Return(nil) + + file, err = fileutils.OpenRegularFileNoFollow(mockFS, "/bad-stat") + require.ErrorIs(t, err, os.ErrPermission) + assert.Nil(t, file) +} + +// Kernel-enforced symlink rejection requires a real filesystem. +func TestCopyFileNoFollowRejectsSymlinks(t *testing.T) { + ctx := testctx.NewCtx(testctx.WithHostFS()) + root := t.TempDir() + target, link, output := filepath.Join(root, "target"), filepath.Join(root, "link"), filepath.Join(root, "output") + require.NoError(t, fileutils.WriteFile(ctx.FS(), target, []byte("host-only contents"), fileperms.PublicFile)) + require.NoError(t, os.Symlink(target, link)) + err := fileutils.CopyFile(ctx, ctx.FS(), link, output, fileutils.CopyFileOptions{NoFollow: true}) + require.ErrorContains(t, err, "without following symlinks") + exists, err := fileutils.Exists(ctx.FS(), output) + require.NoError(t, err) + assert.False(t, exists) +} From 3e512ffa63a22c3271e1c5d81c9480accf6ecc10 Mon Sep 17 00:00:00 2001 From: Antonio Salinas <173939267+Tonisal-byte@users.noreply.github.com> Date: Mon, 2 Nov 2026 13:26:29 -0800 Subject: [PATCH 5/5] test: refresh container schema snapshot for Go vendoring The Go vendor configuration intentionally extends the schema, but scenarioUpdate refreshed only the local baseline when Docker was unavailable. CI reached the container schema check and failed on the missing 37 lines. Update the container baseline to match the actual generated CLI schema and document that incomplete Docker runs can leave this second snapshot stale. Both stdout snapshots and the checked-in schema match the rebuilt CLI output byte-for-byte. Local scenario/scenarioUpdate runs remain blocked by the unavailable Docker provider. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/copilot-instructions.md | 3 ++ ...ainer_config_generate-schema_stdout_1.snap | 37 +++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 8f5fe6df3..493ccaf2d 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -37,5 +37,8 @@ The TOML config files in `defaultconfigs/` are loaded via `internal/projectconfi **IMPORTANT**: Code generation runs automatically with build/test commands. `mage generate` (runs `go generate` for each package in parallel) is a prerequisite for building and runs automatically with `mage build` and `mage unit`. `mage docs` rebuilds the binary and updates the JSON schema (`schemas/azldev.schema.json`) and CLI docs (`docs/user/reference/cli/`). Run `mage docs` explicitly after changing config structs or Cobra command descriptions so that checked-in generated files stay current (checked by PR gates). **CRITICAL**: Run `mage scenarioUpdate` when test expectations change (updates snapshots). +Schema changes affect both `TestSnapshots` and `TestSnapshotsContainer` baselines. +If Docker is unavailable, the update can refresh only local snapshots before failing; +check both baselines against the generated schema and report the incomplete container run. Follow conventional commit format for all changes AND PR titles. A github action will fail the PR if the title does not adhere to the format. diff --git a/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap b/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap index 9cf47bd35..9259ce684 100755 --- a/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap @@ -150,6 +150,11 @@ "title": "Source files", "description": "Source files to download for this component" }, + "go-vendor": { + "$ref": "#/$defs/GoVendorConfig", + "title": "Go vendoring", + "description": "Convert a Go application spec and generate its vendor archive during source preparation" + }, "packages": { "additionalProperties": { "$ref": "#/$defs/PackageConfig" @@ -682,6 +687,38 @@ "additionalProperties": false, "type": "object" }, + "GoVendorConfig": { + "properties": { + "enabled": { + "type": "boolean", + "title": "Enabled", + "description": "Enable Go application vendoring" + }, + "module-dir": { + "type": "string", + "title": "Module directory", + "description": "Relative module directory within the prepared source tree; defaults to its root" + }, + "config-file": { + "type": "string", + "title": "Vendor configuration file", + "description": "Path to a go-vendor-tools TOML file relative to the declaring component configuration. Replaces upstream settings when specified", + "examples": [ + "go-vendor-tools.toml" + ] + }, + "prep-packages": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Prep packages", + "description": "Additional RPM packages needed to execute the upstream prep section in mock" + } + }, + "additionalProperties": false, + "type": "object" + }, "ImageCapabilities": { "properties": { "machine-bootable": {