diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 8f5fe6df..493ccaf2 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -37,5 +37,8 @@ The TOML config files in `defaultconfigs/` are loaded via `internal/projectconfi **IMPORTANT**: Code generation runs automatically with build/test commands. `mage generate` (runs `go generate` for each package in parallel) is a prerequisite for building and runs automatically with `mage build` and `mage unit`. `mage docs` rebuilds the binary and updates the JSON schema (`schemas/azldev.schema.json`) and CLI docs (`docs/user/reference/cli/`). Run `mage docs` explicitly after changing config structs or Cobra command descriptions so that checked-in generated files stay current (checked by PR gates). **CRITICAL**: Run `mage scenarioUpdate` when test expectations change (updates snapshots). +Schema changes affect both `TestSnapshots` and `TestSnapshotsContainer` baselines. +If Docker is unavailable, the update can refresh only local snapshots before failing; +check both baselines against the generated schema and report the incomplete container run. Follow conventional commit format for all changes AND PR titles. A github action will fail the PR if the title does not adhere to the format. diff --git a/.github/instructions/go.instructions.md b/.github/instructions/go.instructions.md index 439a7642..7c0f3cb4 100644 --- a/.github/instructions/go.instructions.md +++ b/.github/instructions/go.instructions.md @@ -105,6 +105,9 @@ description: "Instructions for working on the azldev Go codebase. IMPORTANT: Alw ## Component Fingerprinting — `fingerprint:"-"` Tags +New component fields must survive `ComponentConfig.WithAbsolutePaths` without aliasing +mutable data. Its automatic clone deliberately shares only config-file back-references. + Structs in `internal/projectconfig/` are hashed by `hashstructure.Hash()` to detect component changes. Fields **included by default** (safe: false positive > false negative). When adding a new field to a fingerprinted struct, ask: **"Does changing this field change the build output?"** diff --git a/docs/user/how-to/vendor-go-application.md b/docs/user/how-to/vendor-go-application.md new file mode 100644 index 00000000..4cb103c3 --- /dev/null +++ b/docs/user/how-to/vendor-go-application.md @@ -0,0 +1,155 @@ +# Vendor a Go application + +Use `go-vendor` to bundle a Go application's module dependencies instead of +building against separately packaged `golang(...)` libraries. This is a targeted +conversion of the selected upstream spec, not a `go2rpm` wrapper or a newly +generated replacement spec. The same setting regenerates sources for supported +specs that already use go-vendor-tools. + +## Enable and prepare + +Add this to the component definition: + +```toml +[components.example.go-vendor] +enabled = true +``` + +For custom vendor settings, use a separate file beside the component definition: + +```toml +# example.comp.toml +[components.example.go-vendor] +enabled = true +config-file = "go-vendor-tools.toml" +``` + +```toml +# go-vendor-tools.toml +[archive] +use_module_proxy = true + +[licensing] +detector = "askalono" +``` + +This file replaces, rather than merges with, upstream settings. Carry forward +any required upstream license exceptions. Omit `config-file` to reuse those +settings unchanged. Paths are relative to the declaring component configuration, +including when inherited from defaults; no adjacent file is loaded implicitly. +File-only edits affect the component fingerprint. Inline `tool-config` values +are no longer accepted. + +Then prepare the complete build inputs: + +```sh +azldev component prepare-sources -p example -o prepared-example --without-git +``` + +The project/build distro must have a working mock configuration and provide +`go-vendor-tools`, `askalono-cli`, `golang`, `go-rpm-macros`, `rpm-build`, and +`rpmdevtools`, `rpmautospec`, `git`, and `python3-click`. These are installed in a +fresh, network-enabled generation chroot; +neither `go2rpm` nor the Go vendor tools are required on the host. Configure +`prep-packages` for additional tools required by upstream prep scripts. + +The pipeline fetches the selected upstream sources, applies overlays, and copies +them into isolated staging. It processes automatic release/changelog metadata +there and runs the existing `%prep` section with `rpmbuild -bp --nodeps`. +This retains upstream patches and edits to module files. It then calls +`go_vendor_archive create` on the prepared module tree, extracts the result for +license detection, and runs `go_vendor_license` without interactive prompts. +The archive tool runs from staged dist-git, so hooks can find preserved patches +and sidecars through `GVT_CWD`. +Unrecognized licenses fail preparation; add reviewed, hash-qualified exceptions +to the file selected by `config-file` rather than suppressing detection. + +The output includes the converted spec, all its original sources and patches, +`-vendor.tar.bz2`, and `azldev-go-vendor.toml`. The archive is recorded +with SHA-512 in `sources` and excluded from synthetic git commits with `.gitignore`. +The upstream release/changelog is retained; ordinary azldev release handling +still applies afterwards. + +## What is converted + +The converter removes standalone `BuildRequires: golang(import/path)` tags, +including version constraints, and bare `%go_generate_buildrequires` invocations. +It changes a bare `%gocheck` to `%gotest ./...`, adds vendor extraction at the end +of `%prep`, and enables module/vendor mode for subsequent build scripts. +It adds the Go compiler/macros and vendor tooling build requirements, dynamic +license backend requirements, license installation/file-list handling, and a +license check in `%build` so disabling `%check` does not bypass license validation. +The main `License:` tag is updated to the detected cumulative SPDX expression. + +Unrelated runtime dependencies, C-library build dependencies, custom commands, +patches, metadata, and ordinary subpackages are retained. Go vendoring does not +eliminate dependencies on shared libraries used by cgo. + +The initial supported scope is application specs with one unconditional +main-package `%prep`, `%build`, `%install`, and `%files` section. The prep section +must finish in the module root, or use `module-dir` for a relative subdirectory. +Custom build/install commands must already address that same module. Conditional +commands within those sections are retained. + +Automatic Go development subpackages (`%gopkg`, `%goinstall`, Go file-list macros), +flagged `%gocheck` commands, compound/rich Go BuildRequires, +explicit conflicting module-mode settings, and ambiguous/conditional +target sections are rejected. Use narrowly scoped overlays to adapt these cases; +azldev does not silently drop subpackages or rewrite arbitrary shell scripts. + +## Regenerate an existing vendor archive + +For an already-vendored spec, azldev recognizes a single +`tar -xf %{S:N}` (or `%{SOURCEN}`) prep command and the configuration source +referenced by `%go_vendor_license_* -c %{S:M}` commands. The Source numbers need +not be 1 and 2. The configuration must name a plain sidecar file. + +The original Source numbers, license/file-list macros, module-mode settings, and +custom packaging are retained. The vendor extraction moves to the end of `%prep`, +after module-file patches and edits, because the replacement archive already +contains their results. Directory changes after the original extraction and +secondary-source extraction with `%setup -a/-b` or `%autosetup -a/-b` are rejected +as ambiguous; adapt these layouts with an overlay. The temporary prep spec +omits the vendor Source declaration and its extraction command, so generating +the replacement does not consume the old vendor payload. RPM expands the +upstream archive filename and pins the vendor Source tag to that expanded name, +so subsequent release/history changes cannot desynchronize the spec and payload. +The regenerated archive replaces that file and its +`sources` checksum rather than adding a second vendor archive. + +When `config-file` is omitted, the upstream go-vendor-tools configuration is used, +including its reviewed license exceptions. An explicit file replaces that configuration +even when empty; missing files or invalid TOML fail without falling back. +The detected licenses are verified against the existing +main-package `License:` expression; mismatches fail instead of rewriting it. +Upstream `general.go_mod_dir` is honored by generation and licensing, preserving +the module subdirectory within the archive. A conflicting `module-dir` fails; +use a separate `config-file` to replace upstream settings when changing the layout. +Neither the regenerated archive nor its configuration sidecar may also be +declared in `source-files`, even with `replace-upstream = true`. +The source-fetch phase currently still downloads the upstream vendor archive +alongside the other upstream sources, before regenerating its contents. + +## Review and upgrade + +Run `azldev component diff-sources -p example` to inspect the conversion, or +`azldev component render -p example` to refresh checked-in rendered output. +Rendering and source diffs also generate the archive: they need network access and +can be substantially slower for enabled components. +Rendered output retains the `sources` checksums but excludes payloads listed in +that manifest; archives remain preparation/build artifacts rather than checked-in files. + +Review the cumulative license expression and the retained build/install/check +commands before building. Automatic detection is not approval of a dependency's +license. The converter sets `GO111MODULE=on`, `-mod=vendor`, `GOPROXY=off`, +`GOSUMDB=off`, and `GOTOOLCHAIN=local` for build scripts; custom commands must not +override these settings to download dependencies. + +The same TOML applies after an upstream version update: no archive version, +import path, Source number, or spec line number is embedded in the declaration. +Generation reruns against the newly selected and patched inputs. Review changed +vendor hashes and licenses; an upstream packaging change may require updating +an overlay or resolving a newly unsupported construct. + +See [Go vendoring configuration](../reference/config/components.md#go-vendoring) +for module layout, dependency overrides, and license exceptions. diff --git a/docs/user/reference/cli/azldev_component_prepare-sources.md b/docs/user/reference/cli/azldev_component_prepare-sources.md index 45f9f8cb..b133f34c 100644 --- a/docs/user/reference/cli/azldev_component_prepare-sources.md +++ b/docs/user/reference/cli/azldev_component_prepare-sources.md @@ -41,8 +41,8 @@ azldev component prepare-sources [flags] -o, --output-dir string output directory --rpmdev-bumpspec Use rpmdev-bumpspec instead of the legacy static release calculation --skip-lock-validation skip lock file consistency checks - --skip-overlays skip applying overlays to prepared sources - --skip-sources skip downloading fetched sources when preparing the package (useful to extract dist-git metadata when source files are not needed) + --skip-overlays skip overlays and Go vendor conversion, preserving upstream packaging + --skip-sources skip downloading sources unless Go vendor conversion requires them -s, --spec-path stringArray Spec path --without-git Disable dist-git repository creation (enabled by default) ``` diff --git a/docs/user/reference/cli/azldev_component_render.md b/docs/user/reference/cli/azldev_component_render.md index 88a9d515..239018bd 100644 --- a/docs/user/reference/cli/azldev_component_render.md +++ b/docs/user/reference/cli/azldev_component_render.md @@ -16,9 +16,9 @@ Within the output directory, components are organized into letter-prefixed subdirectories based on the first character of their name (e.g., specs/c/curl, specs/v/vim). -Unlike prepare-sources, render skips downloading source tarballs from the -lookaside cache — only spec files, patches, scripts, and other git-tracked -sidecar files are included. Multiple components can be rendered at once. +Render normally skips source downloads. Go vendoring downloads and prepares sources for archive generation +and license checks, but excludes payloads from checked-in output. Only specs, patches, scripts, checksums, +and sidecar files are included. Multiple components can be rendered at once. When rendering all components (-a), the --clean-stale flag prunes orphan rendered-spec directories (per-component dirs that no longer correspond to diff --git a/docs/user/reference/config/components.md b/docs/user/reference/config/components.md index ce06e6f6..f483f671 100644 --- a/docs/user/reference/config/components.md +++ b/docs/user/reference/config/components.md @@ -15,6 +15,7 @@ A component definition tells azldev where to find the spec file, how to customiz | Build config | `build` | [BuildConfig](#build-configuration) | No | Build-time options (macros, conditionals, check config) | | Render config | `render` | [RenderConfig](#render-configuration) | No | Options controlling spec rendering behavior | | Source files | `source-files` | array of [SourceFileReference](#source-file-references) | No | Additional source files to download for this component | +| Go vendoring | `go-vendor` | [GoVendorConfig](#go-vendoring) | No | Convert a supported Go application spec in place and generate bundled dependencies | | Package overrides | `packages` | map of string → [PackageConfig](package-groups.md#package-config) | No | Exact per-package configuration overrides; highest priority in the resolution order | | Tests | `tests` | [ComponentTests](#component-tests) | No | Test references that apply to this component (see [Tests and Test Groups](tests.md)) | @@ -165,6 +166,77 @@ skip-file-filter = true > **Note:** This should only be used for specs with macros that `spectool` cannot resolve. For most components, the default filtering behavior is correct and keeps the rendered output clean. +## Go Vendoring + +Go vendoring is opt-in and preserves the upstream spec rather than replacing it +with a generated spec. See [Vendor a Go application](../../how-to/vendor-go-application.md) +for the workflow and supported spec patterns. + +```toml +[components.example.go-vendor] +enabled = true +``` + +| TOML Key | Type | Default | Description | +|----------|------|---------|-------------| +| `enabled` | boolean | `false` | Enable conversion and vendor archive generation. An explicit `false` overrides inherited enablement. | +| `module-dir` | string | `""` | Relative directory containing `go.mod`, resolved from the working directory at the end of the upstream `%prep` section. Empty uses that directory itself. No absolute paths, traversal, or RPM macros. | +| `prep-packages` | array of string | `[]` | Additional RPM packages needed by upstream `%prep` commands or the selected license detector. Installed only in the isolated generation chroot. | +| `config-file` | string | `""` | Path to a separate [go-vendor-tools TOML file](https://fedora.gitlab.io/sigs/go/go-vendor-tools/config/), relative to the configuration file that declares it. Its contents replace upstream tool settings; they are not merged. When no file is configured, reuse an already-vendored spec's upstream settings or use tool defaults for a new conversion. | + +Custom vendor settings must live in a separate file, typically +`go-vendor-tools.toml` beside the component definition. Its contents participate +in the component fingerprint; absolute checkout paths do not. Inherited paths +retain their original declaration directory. Editing only this file makes the +lock fingerprint stale and requires the normal update/render workflow. + +```toml +# example.comp.toml +[components.example.go-vendor] +enabled = true +config-file = "go-vendor-tools.toml" +``` + +```toml +# go-vendor-tools.toml +[archive.dependency_overrides] +"example.com/dependency" = "v1.2.3" +``` + +The configured file must exist and contain valid TOML. An explicitly selected +empty file is an empty replacement, not a request to fall back to upstream. +In an explicitly selected file, use `module-dir`, not `general.go_mod_dir`, for +the module location. Reused upstream `general.go_mod_dir` settings remain honored; +an inconsistent component `module-dir` is rejected. +Configured command hooks execute in the generation chroot; treat this file as +trusted build code. The old inline `tool-config` string/table is no longer supported. + +For previously unbundled specs, preparation emits `-vendor.tar.bz2` and `azldev-go-vendor.toml`, +allocating unused `Source` numbers automatically. These names are reserved for +generated output; do not also declare them in `source-files` or supply them through +overlays. The archive's SHA-512 digest is recorded in the prepared `sources` file. +Generation runs afresh on every preparation; there is no filename-based cache. +The recorded digest identifies that output, not a separately pinned vendor lock. + +For supported specs already using go-vendor-tools, preparation instead regenerates +the existing vendor source, retaining its Source number, RPM-expanded filename, +and configuration sidecar. The Source tag is pinned to the generated filename +before release/history updates, and extraction moves after prep's module edits +to avoid applying patches twice. Ambiguous secondary-source setup and +post-extraction directory changes require a prep overlay. +It updates that archive's `sources` entry in place and +verifies the existing `License:` expression without rewriting it. +These generated archive/config filenames must not also appear in `source-files`, +including entries with `replace-upstream = true`. +When `config-file` is set, its contents are staged under the configuration filename +referenced by the prepared spec, retaining the existing Source number. + +All archive/spec/loose-file overlays run before conversion. Source preparation, +builds, rendering (including `--check-only`), and source diffs use the same +conversion. Enabled components require downloads even with `--skip-sources`, +because preparation and license detection need the effective upstream sources. +`--skip-overlays` also skips conversion, leaving the upstream packaging unchanged. + ## Build Configuration The `[components..build]` section controls build-time options for a component. diff --git a/internal/app/azldev/agentskill/agentskill_test.go b/internal/app/azldev/agentskill/agentskill_test.go index a263a100..5805e528 100644 --- a/internal/app/azldev/agentskill/agentskill_test.go +++ b/internal/app/azldev/agentskill/agentskill_test.go @@ -160,6 +160,21 @@ func TestComponentTomlSkillDocumentsRPMDevBumpspecPreview(t *testing.T) { assert.Contains(t, doc, "no bumpspec runtime discovery occurs") } +func TestComponentTomlSkillDocumentsGoVendoring(t *testing.T) { + for _, withoutLockfile := range []bool{false, true} { + doc, err := agentskill.NewCatalog(withoutLockfile).SkillDocument("azldev-comp-toml", testParams()) + require.NoError(t, err) + assert.Contains(t, doc, "go-vendor.enabled") + assert.Contains(t, doc, "does\nnot run `go2rpm`") + assert.Contains(t, doc, "Rendering and source diffs also generate") + assert.Contains(t, doc, "reviewed license exceptions") + assert.Contains(t, doc, `config-file = "go-vendor-tools.toml"`) + assert.Contains(t, doc, "Inline `tool-config` is not supported.") + assert.Contains(t, doc, "Upstream `general.go_mod_dir` is honored") + assert.Contains(t, doc, "Source filename is pinned before") + } +} + func TestImageSkillDocumentsSKUGroupGuidance(t *testing.T) { doc, err := agentskill.SkillDocument("azldev-image", testParams()) require.NoError(t, err) diff --git a/internal/app/azldev/agentskill/content/comp-toml.md.tmpl b/internal/app/azldev/agentskill/content/comp-toml.md.tmpl index 0def4bf5..7059b2ae 100644 --- a/internal/app/azldev/agentskill/content/comp-toml.md.tmpl +++ b/internal/app/azldev/agentskill/content/comp-toml.md.tmpl @@ -32,6 +32,34 @@ Add sub-tables only for what you change. The fields you will reach for most: | `build.defines` / `build.with` / `build.without` | RPM macro and bcond build tweaks | | `release.calculation` | how the `Release` tag is managed | | `render.skip-file-filter` | rendering edge-case escape hatch | +| `go-vendor.enabled` | opt-in Go application spec conversion and vendor archive generation | + +## Go application vendoring + +Set `[components..go-vendor]` with `enabled = true` to preserve the upstream +spec while converting supported application packaging to bundled modules. +Preparation runs the patched upstream `%prep` before `go_vendor_archive`; it does +not run `go2rpm`. Rendering and source diffs also generate the vendor archive and +require downloads. Review the generated cumulative `License:` expression. + +Use `module-dir` for a module below the final prep working directory, +`prep-packages` for additional chroot tools, and `config-file = "go-vendor-tools.toml"` +for custom settings in a separate native TOML file beside the component definition. +Keep reviewed license exceptions in that file. +The path is relative to the declaring config, and file contents are fingerprinted. +It replaces upstream settings without merging; an explicit empty file is still a +replacement, and a missing file is an error. Inline `tool-config` is not supported. +Generated `-vendor.tar.bz2` and `azldev-go-vendor.toml` must not also be +declared as `source-files`. For supported already-vendored specs, regenerate the +existing vendor Source instead, reuse upstream tool configuration unless overridden, +and verify rather than replace the reviewed `License:` expression. +The regenerated archive and configuration must not also be `source-files` entries. +Upstream `general.go_mod_dir` is honored; conflicting `module-dir` values fail. +Vendor extraction runs after prep edits, and its Source filename is pinned before +release/history updates. Ambiguous secondary-source setup requires an overlay. +Unsupported constructs (notably automatic Go devel +subpackages, flagged `%gocheck`, or ambiguous sections) require explicit overlays; +never replace the upstream spec wholesale to bypass a conversion error. ## Spec source diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl index 0e33a9e6..2ac8df15 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/comp-toml.md.tmpl @@ -32,6 +32,34 @@ Add sub-tables only for what you change. The fields you will reach for most: | `build.defines` / `build.with` / `build.without` | RPM macro and bcond build tweaks | | `release.calculation` | how the `Release` tag is managed | | `render.skip-file-filter` | rendering edge-case escape hatch | +| `go-vendor.enabled` | opt-in Go application spec conversion and vendor archive generation | + +## Go application vendoring + +Set `[components..go-vendor]` with `enabled = true` to preserve the upstream +spec while converting supported application packaging to bundled modules. +Preparation runs the patched upstream `%prep` before `go_vendor_archive`; it does +not run `go2rpm`. Rendering and source diffs also generate the vendor archive and +require downloads. Review the generated cumulative `License:` expression. + +Use `module-dir` for a module below the final prep working directory, +`prep-packages` for additional chroot tools, and `config-file = "go-vendor-tools.toml"` +for custom settings in a separate native TOML file beside the component definition. +Keep reviewed license exceptions in that file. +The path is relative to the declaring config, and file contents are fingerprinted. +It replaces upstream settings without merging; an explicit empty file is still a +replacement, and a missing file is an error. Inline `tool-config` is not supported. +Generated `-vendor.tar.bz2` and `azldev-go-vendor.toml` must not also be +declared as `source-files`. For supported already-vendored specs, regenerate the +existing vendor Source instead, reuse upstream tool configuration unless overridden, +and verify rather than replace the reviewed `License:` expression. +The regenerated archive and configuration must not also be `source-files` entries. +Upstream `general.go_mod_dir` is honored; conflicting `module-dir` values fail. +Vendor extraction runs after prep edits, and its Source filename is pinned before +release/history updates. Ambiguous secondary-source setup requires an overlay. +Unsupported constructs (notably automatic Go devel +subpackages, flagged `%gocheck`, or ambiguous sections) require explicit overlays; +never replace the upstream spec wholesale to bypass a conversion error. ## Spec source diff --git a/internal/app/azldev/cmds/component/build.go b/internal/app/azldev/cmds/component/build.go index 2c9f9917..f2d95527 100644 --- a/internal/app/azldev/cmds/component/build.go +++ b/internal/app/azldev/cmds/component/build.go @@ -288,7 +288,7 @@ func buildComponent( preparerOpts = append(preparerOpts, sources.WithMockProcessor(mockProcessor)) - sourcePreparer, err := newSourcePreparer(sourceManager, env.FS(), env, env, append( + sourcePreparer, err := newComponentSourcePreparer(env, component, sourceManager, options.MockConfigOpts, append( preparerOpts, sources.WithSpecEditor(options.specEditorMode()), )...) diff --git a/internal/app/azldev/cmds/component/changed_upstreamcommit.go b/internal/app/azldev/cmds/component/changed_upstreamcommit.go index ce8e19eb..ce18e0d5 100644 --- a/internal/app/azldev/cmds/component/changed_upstreamcommit.go +++ b/internal/app/azldev/cmds/component/changed_upstreamcommit.go @@ -118,6 +118,7 @@ type componentComparisonInputs struct { Config projectconfig.ComponentConfig `json:"config"` SourceIdentity string `json:"sourceIdentity,omitempty"` OverlaySourceHashes map[string]string `json:"overlaySourceHashes,omitempty"` + GoVendorConfigHash string `json:"goVendorConfigHash,omitempty"` ReleaseVer string `json:"releaseVer,omitempty"` } @@ -252,6 +253,13 @@ func buildComponentComparisonInputs( SourceIdentity: component.EffectiveUpstreamCommit(), } + var err error + + inputs.GoVendorConfigHash, err = component.GoVendor.SourceContentIdentity(fs) + if err != nil { + return inputs, fmt.Errorf("hashing Go vendor configuration:\n%w", err) + } + if component.Spec.SourceType == projectconfig.SpecSourceTypeLocal { identity, err := sourceproviders.ResolveLocalSourceIdentity( fs, filepath.Dir(component.Spec.Path), @@ -314,7 +322,13 @@ func normalizeComponentForComparison(component projectconfig.ComponentConfig) pr normalized.Build.Hints = projectconfig.ComponentBuildHints{} normalized.OverlayFiles = nil normalized.Publish = projectconfig.ComponentPublishConfig{} + normalized.Tests = nil + if component.GoVendor != nil { + vendor := *component.GoVendor + vendor.ConfigFile = "" + normalized.GoVendor = &vendor + } normalized.Overlays = slices.Clone(component.Overlays) for idx := range normalized.Overlays { diff --git a/internal/app/azldev/cmds/component/changed_upstreamcommit_internal_test.go b/internal/app/azldev/cmds/component/changed_upstreamcommit_internal_test.go index 357f3321..1680f208 100644 --- a/internal/app/azldev/cmds/component/changed_upstreamcommit_internal_test.go +++ b/internal/app/azldev/cmds/component/changed_upstreamcommit_internal_test.go @@ -209,6 +209,28 @@ func TestBuildComponentComparisonInputs_ContentIdentities(t *testing.T) { assert.NotEqual(t, second.OverlaySourceHashes, third.OverlaySourceHashes) } +func TestBuildComponentComparisonInputs_GoVendorFile(t *testing.T) { + env := testutils.NewTestEnvWithoutLockfile(t) + enabled := true + component := projectconfig.ComponentConfig{GoVendor: &projectconfig.GoVendorConfig{ + Enabled: &enabled, ConfigFile: "/project/vendor.toml", + }} + require.NoError(t, fileutils.WriteFile(env.FS(), component.GoVendor.ConfigFile, + []byte("[archive]\ntidy = false\n"), fileperms.PublicFile)) + first, err := buildComponentComparisonInputs(env.FS(), env.Env, &component) + require.NoError(t, err) + assert.Empty(t, first.Config.GoVendor.ConfigFile) + assert.Equal(t, "/project/vendor.toml", component.GoVendor.ConfigFile) + require.NotEmpty(t, first.GoVendorConfigHash) + + require.NoError(t, fileutils.WriteFile(env.FS(), component.GoVendor.ConfigFile, + []byte("[archive]\ntidy = true\n"), fileperms.PublicFile)) + second, err := buildComponentComparisonInputs(env.FS(), env.Env, &component) + require.NoError(t, err) + assert.NotEqual(t, first.GoVendorConfigHash, second.GoVendorConfigHash) + assert.Equal(t, first.Config, second.Config) +} + func TestLoadHistoricalProject_UsesNormalIncludesAndMerging(t *testing.T) { rootConfig := []byte(`includes = [ "config/components.toml", diff --git a/internal/app/azldev/cmds/component/diffsources.go b/internal/app/azldev/cmds/component/diffsources.go index 7fd72805..24a2dd37 100644 --- a/internal/app/azldev/cmds/component/diffsources.go +++ b/internal/app/azldev/cmds/component/diffsources.go @@ -101,7 +101,7 @@ func DiffComponentSources(env *azldev.Env, options *DiffSourcesOptions) (interfa return nil, fmt.Errorf("failed to create source manager:\n%w", err) } - preparer, err := newSourcePreparer(sourceManager, env.FS(), env, env, + preparer, err := newComponentSourcePreparer(env, component, sourceManager, nil, sources.WithUpstreamProvenance(sources.FedoraDistTag(distro.Ref.Name, distro.Version.ReleaseVer)), sources.WithSpecEditor(options.specEditorMode())) if err != nil { diff --git a/internal/app/azldev/cmds/component/history_customizations.go b/internal/app/azldev/cmds/component/history_customizations.go index 2e90e7eb..8a051a54 100644 --- a/internal/app/azldev/cmds/component/history_customizations.go +++ b/internal/app/azldev/cmds/component/history_customizations.go @@ -29,6 +29,31 @@ func collectCustomizations(name string, config *projectconfig.ComponentConfig) [ items = appendRenderItems(items, config.Render) items = appendPackageItems(items, config.Packages) items = appendSourceFileItems(items, config.SourceFiles) + items = appendGoVendorItems(items, config.GoVendor) + + return items +} + +func appendGoVendorItems(items []CustomizationItem, config *projectconfig.GoVendorConfig) []CustomizationItem { + if config == nil { + return items + } + + if config.Enabled != nil { + items = append(items, CustomizationItem{Kind: "go-vendor.enabled", Value: strconv.FormatBool(*config.Enabled)}) + } + + if config.ModuleDir != "" { + items = append(items, CustomizationItem{Kind: "go-vendor.module-dir", Value: config.ModuleDir}) + } + + if config.ConfigFile != "" { + items = append(items, CustomizationItem{Kind: "go-vendor.config-file", Value: config.ConfigFile}) + } + + for _, name := range config.PrepPackages { + items = append(items, CustomizationItem{Kind: "go-vendor.prep-packages", Value: name}) + } return items } diff --git a/internal/app/azldev/cmds/component/history_internal_test.go b/internal/app/azldev/cmds/component/history_internal_test.go index 24209419..858d509f 100644 --- a/internal/app/azldev/cmds/component/history_internal_test.go +++ b/internal/app/azldev/cmds/component/history_internal_test.go @@ -88,6 +88,7 @@ func TestCustomizationCollectorsCoverEveryFingerprintableField(t *testing.T) { reflect.TypeFor[projectconfig.ComponentRenderConfig](), reflect.TypeFor[projectconfig.SourceFileReference](), reflect.TypeFor[projectconfig.Origin](), + reflect.TypeFor[projectconfig.GoVendorConfig](), } // Maps "StructName.FieldName" -> short note describing how the field @@ -103,6 +104,10 @@ func TestCustomizationCollectorsCoverEveryFingerprintableField(t *testing.T) { "ComponentConfig.Render": "appendRenderItems (per-field via ComponentRenderConfig walk)", "ComponentConfig.SourceFiles": "appendSourceFileItems (opaque unit per source file)", "ComponentConfig.Packages": "appendPackageItems (opaque unit per package override)", + "ComponentConfig.GoVendor": "appendGoVendorItems", + "GoVendorConfig.Enabled": "go-vendor.enabled", + "GoVendorConfig.ModuleDir": "go-vendor.module-dir", + "GoVendorConfig.PrepPackages": "go-vendor.prep-packages", // ComponentBuildConfig. "ComponentBuildConfig.With": "build.with", diff --git a/internal/app/azldev/cmds/component/preparerctx.go b/internal/app/azldev/cmds/component/preparerctx.go index f98f7b67..777ff4c7 100644 --- a/internal/app/azldev/cmds/component/preparerctx.go +++ b/internal/app/azldev/cmds/component/preparerctx.go @@ -4,11 +4,37 @@ package component import ( + "errors" + "fmt" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/sources" "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders" + "github.com/microsoft/azure-linux-dev-tools/internal/rpm/mock" ) +func goVendorPreparerOptions( + env *azldev.Env, component components.Component, mockConfigOpts map[string]string, +) ([]sources.PreparerOption, error) { + if !component.GetConfig().GoVendor.IsEnabled() || env.DryRun() { + return nil, nil + } + + _, distro, err := env.Distro() + if err != nil { + return nil, fmt.Errorf("resolving Go vendor build distro:\n%w", err) + } + + if distro.MockConfigPath == "" { + return nil, errors.New("go vendoring requires a project distro with a mock configuration") + } + + runner := mock.NewRunner(env, distro.MockConfigPath).WithConfigOpts(mockConfigOpts) + + return []sources.PreparerOption{sources.WithGoVendorRunner(runner)}, nil +} + // gitRepoPreparerOptions returns the [sources.PreparerOption] values that enable // synthetic dist-git history for the current mode. // diff --git a/internal/app/azldev/cmds/component/preparerctx_internal_test.go b/internal/app/azldev/cmds/component/preparerctx_internal_test.go new file mode 100644 index 00000000..715286b8 --- /dev/null +++ b/internal/app/azldev/cmds/component/preparerctx_internal_test.go @@ -0,0 +1,64 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component + +import ( + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components/components_testutils" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/testutils" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/samber/lo" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +func TestGoVendorPreparerOptions(t *testing.T) { + const disabled = "disabled" + for _, mode := range []string{"enabled", disabled, "missing-mock", "missing-distro"} { + t.Run(mode, func(t *testing.T) { + env := testutils.NewTestEnv(t) + component := components_testutils.NewMockComponent(gomock.NewController(t)) + component.EXPECT().GetConfig().AnyTimes().Return(&projectconfig.ComponentConfig{ + GoVendor: &projectconfig.GoVendorConfig{Enabled: lo.ToPtr(mode != disabled)}, + }) + + switch mode { + case "missing-mock", disabled: + clearProjectMockConfig(env.Env.Config()) + case "missing-distro": + env.Env.Config().Project.DefaultDistro.Name = "absent" + } + + options, err := goVendorPreparerOptions(env.Env, component, nil) + + switch mode { + case "enabled": + require.NoError(t, err) + require.Len(t, options, 1) + case disabled: + require.NoError(t, err) + assert.Empty(t, options) + default: + require.Error(t, err) + assert.Empty(t, options) + } + }) + } +} + +func TestGoVendorHistoryCustomizations(t *testing.T) { + config := &projectconfig.GoVendorConfig{ + Enabled: lo.ToPtr(true), ModuleDir: "cmd/app", + ConfigFile: "/project/go-vendor-tools.toml", PrepPackages: []string{"git"}, + } + assert.Equal(t, []CustomizationItem{ + {Kind: "go-vendor.enabled", Value: "true"}, + {Kind: "go-vendor.module-dir", Value: "cmd/app"}, + {Kind: "go-vendor.config-file", Value: config.ConfigFile}, + {Kind: "go-vendor.prep-packages", Value: "git"}, + }, collectCustomizations("app", &projectconfig.ComponentConfig{GoVendor: config})) + assert.Empty(t, appendGoVendorItems(nil, &projectconfig.GoVendorConfig{})) +} diff --git a/internal/app/azldev/cmds/component/preparesources.go b/internal/app/azldev/cmds/component/preparesources.go index 317b7fb3..37b9668c 100644 --- a/internal/app/azldev/cmds/component/preparesources.go +++ b/internal/app/azldev/cmds/component/preparesources.go @@ -73,7 +73,8 @@ Only one component may be selected at a time.`, _ = cmd.MarkFlagRequired("output-dir") _ = cmd.MarkFlagDirname("output-dir") - cmd.Flags().BoolVar(&options.SkipOverlays, "skip-overlays", false, "skip applying overlays to prepared sources") + cmd.Flags().BoolVar(&options.SkipOverlays, "skip-overlays", false, + "skip overlays and Go vendor conversion, preserving upstream packaging") cmd.Flags().BoolVar(&options.WithoutGitRepo, "without-git", false, "Disable dist-git repository creation (enabled by default)") addRPMDevBumpspecFlag(cmd, &options.RPMDevBumpspec) @@ -82,8 +83,7 @@ Only one component may be selected at a time.`, cmd.Flags().BoolVar(&options.AllowNoHashes, "allow-no-hashes", false, "compute missing hashes by downloading source files from their origin") cmd.Flags().BoolVar(&options.SkipSources, "skip-sources", false, - "skip downloading fetched sources when preparing the package (useful to extract "+ - "dist-git metadata when source files are not needed)") + "skip downloading sources unless Go vendor conversion requires them") return cmd } @@ -142,6 +142,14 @@ func PrepareComponentSources(env *azldev.Env, options *PrepareSourcesOptions) er } preparerOpts := buildPreparerOptions(env, distro, options) + if !options.SkipOverlays { + vendorOpts, err := goVendorPreparerOptions(env, component, nil) + if err != nil { + return err + } + + preparerOpts = append(preparerOpts, vendorOpts...) + } preparer, err := newSourcePreparer(sourceManager, env.FS(), env, env, append( preparerOpts, diff --git a/internal/app/azldev/cmds/component/render.go b/internal/app/azldev/cmds/component/render.go index ff99942b..e9df2be1 100644 --- a/internal/app/azldev/cmds/component/render.go +++ b/internal/app/azldev/cmds/component/render.go @@ -64,9 +64,9 @@ Within the output directory, components are organized into letter-prefixed subdirectories based on the first character of their name (e.g., specs/c/curl, specs/v/vim). -Unlike prepare-sources, render skips downloading source tarballs from the -lookaside cache — only spec files, patches, scripts, and other git-tracked -sidecar files are included. Multiple components can be rendered at once. +Render normally skips source downloads. Go vendoring downloads and prepares sources for archive generation +and license checks, but excludes payloads from checked-in output. Only specs, patches, scripts, checksums, +and sidecar files are included. Multiple components can be rendered at once. When rendering all components (-a), the --clean-stale flag prunes orphan rendered-spec directories (per-component dirs that no longer correspond to @@ -531,8 +531,8 @@ func prepareComponentSources( // Prepare sources with overlays, skipping lookaside downloads. // WithGitRepo preserves upstream .git and creates synthetic history so // rpmautospec can expand %autorelease and %autochangelog correctly. - // WithSkipLookaside avoids expensive tarball downloads — only spec + - // sidecar files are needed for rendering. + // WithSkipLookaside avoids tarball downloads unless Go vendoring requires + // the prepared sources to generate the archive and cumulative license. preparerOpts := append(gitRepoPreparerOptions(env, distro), sources.WithSkipLookaside(), sources.WithUpstreamProvenance(sources.FedoraDistTag(distro.Ref.Name, distro.Version.ReleaseVer)), @@ -542,7 +542,7 @@ func prepareComponentSources( preparerOpts = append(preparerOpts, sources.WithRPMDevBumpspec(env, env.WorkDir(), "")) } - preparer, err := newSourcePreparer(sourceManager, env.FS(), env, env, append( + preparer, err := newComponentSourcePreparer(env, comp, sourceManager, nil, append( preparerOpts, sources.WithSpecEditor(specEditor), )...) @@ -765,6 +765,12 @@ func finishComponentRender( return false, fmt.Errorf("filtering unreferenced files for %#q:\n%w", componentName, filterErr) } + if err := removeGoVendorRenderPayloads( + env.FS(), componentDir, prep.comp.GetConfig().GoVendor.IsEnabled(), + ); err != nil { + return false, fmt.Errorf("removing generated render payloads for %#q:\n%w", componentName, err) + } + // Remove .git directory — must not appear in rendered output. // rpmautospec already read it during the batch mock phase. gitDir := filepath.Join(componentDir, ".git") diff --git a/internal/app/azldev/cmds/component/render_govendor.go b/internal/app/azldev/cmds/component/render_govendor.go new file mode 100644 index 00000000..bcfae200 --- /dev/null +++ b/internal/app/azldev/cmds/component/render_govendor.go @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component + +import ( + "errors" + "fmt" + "os" + "path/filepath" + + "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx" + "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders/fedorasource" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" +) + +// Vendoring needs real payloads during preparation, but rendered dist-git output +// must contain their manifest rather than copying source archives into the repository. +func removeGoVendorRenderPayloads(fs opctx.FS, directory string, enabled bool) error { + if !enabled { + return nil + } + + contents, err := fileutils.ReadFile(fs, filepath.Join(directory, fedorasource.SourcesFileName)) + if errors.Is(err, os.ErrNotExist) { + return nil + } + + if err != nil { + return fmt.Errorf("reading render source manifest:\n%w", err) + } + + lines, err := fedorasource.ReadSourcesFile(string(contents)) + if err != nil { + return fmt.Errorf("parsing render source manifest:\n%w", err) + } + + for _, line := range lines { + if line.Entry == nil { + continue + } + + if err := fileutils.ValidateFilename(line.Entry.Filename); err != nil { + return fmt.Errorf("invalid render source filename:\n%w", err) + } + + if err := fs.Remove(filepath.Join(directory, line.Entry.Filename)); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("removing render source payload %#q:\n%w", line.Entry.Filename, err) + } + } + + return nil +} diff --git a/internal/app/azldev/cmds/component/render_govendor_internal_test.go b/internal/app/azldev/cmds/component/render_govendor_internal_test.go new file mode 100644 index 00000000..e6b33633 --- /dev/null +++ b/internal/app/azldev/cmds/component/render_govendor_internal_test.go @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component + +import ( + "strings" + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/global/testctx" + "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders/fedorasource" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRemoveGoVendorRenderPayloads(t *testing.T) { + for _, enabled := range []bool{false, true} { + ctx := testctx.NewCtx() + + manifest := "# keep manifest\n" + + fedorasource.FormatSourcesEntry("upstream.tar.gz", fileutils.HashTypeSHA512, strings.Repeat("a", 128)) + "\n" + + fedorasource.FormatSourcesEntry("vendor.tar.bz2", fileutils.HashTypeSHA512, strings.Repeat("b", 128)) + "\n" + for name, content := range map[string]string{ + "app.spec": "spec", "config.toml": "[archive]", "sources": manifest, + "upstream.tar.gz": "source", "vendor.tar.bz2": "vendor", + } { + require.NoError(t, fileutils.WriteFile(ctx.FS(), "/render/"+name, []byte(content), fileperms.PublicFile)) + } + + require.NoError(t, removeGoVendorRenderPayloads(ctx.FS(), "/render", enabled)) + + for _, name := range []string{"upstream.tar.gz", "vendor.tar.bz2"} { + exists, err := fileutils.Exists(ctx.FS(), "/render/"+name) + require.NoError(t, err) + assert.Equal(t, !enabled, exists) + } + + for _, name := range []string{"app.spec", "config.toml", "sources"} { + exists, err := fileutils.Exists(ctx.FS(), "/render/"+name) + require.NoError(t, err) + assert.True(t, exists) + } + + data, err := fileutils.ReadFile(ctx.FS(), "/render/sources") + require.NoError(t, err) + assert.Equal(t, manifest, string(data)) + require.NoError(t, removeGoVendorRenderPayloads(ctx.FS(), "/render", enabled)) + } +} diff --git a/internal/app/azldev/cmds/component/sourcepreparer.go b/internal/app/azldev/cmds/component/sourcepreparer.go index a16e0a00..a1389026 100644 --- a/internal/app/azldev/cmds/component/sourcepreparer.go +++ b/internal/app/azldev/cmds/component/sourcepreparer.go @@ -3,7 +3,24 @@ package component -import "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/sources" +import ( + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/sources" + "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders" +) //nolint:gochecknoglobals // Replaced by isolated command tests to observe constructed source preparers. var newSourcePreparer = sources.NewPreparer + +func newComponentSourcePreparer( + env *azldev.Env, component components.Component, manager sourceproviders.SourceManager, + mockConfigOpts map[string]string, options ...sources.PreparerOption, +) (sources.SourcePreparer, error) { + vendorOpts, err := goVendorPreparerOptions(env, component, mockConfigOpts) + if err != nil { + return nil, err + } + + return newSourcePreparer(manager, env.FS(), env, env, append(options, vendorOpts...)...) +} diff --git a/internal/app/azldev/core/sources/govendor.go b/internal/app/azldev/core/sources/govendor.go new file mode 100644 index 00000000..fb061975 --- /dev/null +++ b/internal/app/azldev/core/sources/govendor.go @@ -0,0 +1,257 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package sources + +import ( + "context" + "errors" + "fmt" + "log/slog" + "path/filepath" + "slices" + "strings" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components" + "github.com/microsoft/azure-linux-dev-tools/internal/govendor" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders/fedorasource" + "github.com/microsoft/azure-linux-dev-tools/internal/rpm/mock" + "github.com/microsoft/azure-linux-dev-tools/internal/rpm/spec" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" +) + +// WithGoVendorRunner provides a template for an isolated generation root. +func WithGoVendorRunner(runner *mock.Runner) PreparerOption { + return func(p *sourcePreparerImpl) { p.goVendorRunner = runner } +} + +func (p *sourcePreparerImpl) prepareGoVendor( + ctx context.Context, component components.Component, outputDir string, +) (err error) { + config := component.GetConfig().GoVendor + if !config.IsEnabled() { + return nil + } + + if err := config.Validate(); err != nil { + return fmt.Errorf("invalid Go vendor configuration:\n%w", err) + } + + if p.dryRunnable.DryRun() { + slog.Info("Dry run; skipping Go vendor generation", "component", component.GetName()) + + return nil + } + + if p.goVendorRunner == nil { + return errors.New("go vendoring requires a mock runner with the project's build distro configuration") + } + + specPath, err := p.resolveSpecPath(component, outputDir) + if err != nil { + return err + } + + data, err := fileutils.ReadFile(p.fs, specPath) + if err != nil { + return fmt.Errorf("reading Go spec:\n%w", err) + } + + plan, err := spec.ConvertGoVendor(data, spec.GoVendorOptions{ + ArchiveName: component.GetName() + "-vendor.tar.bz2", + ConfigName: govendor.ConfigFilename, ModuleDir: config.ModuleDir, + }) + if err != nil { + return fmt.Errorf("planning Go vendoring for %#q:\n%w", component.GetName(), err) + } + + toolConfig, err := p.resolveGoVendorConfig(outputDir, plan, config) + if err != nil { + return err + } + + scratch, err := fileutils.MkdirTempInTempDir(p.fs, "azldev-go-vendor-*") + if err != nil { + return fmt.Errorf("creating Go vendor scratch directory:\n%w", err) + } + defer fileutils.RemoveAllAndUpdateErrorIfNil(p.fs, scratch, &err) + + slog.Info("Generating Go vendor archive", "spec", filepath.Base(specPath), "regenerating", plan.AllowReplacement) + + artifact, err := govendor.Generate(ctx, p.fs, p.goVendorRunner, govendor.Request{ + SourceDir: outputDir, WorkDir: scratch, SpecName: filepath.Base(specPath), PrepSpec: plan.PrepSpec, + ArchiveFile: plan.ArchiveFile, ArchiveSuffix: plan.ArchiveSuffix, ModuleDir: config.ModuleDir, + ConfigContents: toolConfig, Packages: config.PrepPackages, ExpectedLicense: plan.ExpectedLicense, + }) + if err != nil { + return fmt.Errorf("generating Go vendor sources:\n%w", err) + } + + for _, ref := range component.GetConfig().SourceFiles { + if ref.Filename == artifact.Name || ref.Filename == plan.ConfigName { + return fmt.Errorf("'source-files' entry %#q conflicts with generated Go vendor sources", ref.Filename) + } + } + + return p.publishGoVendor(outputDir, filepath.Base(specPath), toolConfig, plan, artifact) +} + +func (p *sourcePreparerImpl) resolveGoVendorConfig( + outputDir string, plan *spec.GoVendorSpec, config *projectconfig.GoVendorConfig, +) (string, error) { + if !plan.AllowReplacement { + if err := p.checkGoVendorSourceCollision(outputDir, plan.ArchiveFile, plan.ConfigName); err != nil { + return "", err + } + } + + if config.ConfigFile != "" { + content, err := config.ReadConfigFile(p.fs) + if err != nil { + return "", fmt.Errorf("loading Go vendor configuration:\n%w", err) + } + + return content, nil + } + + if !plan.AllowReplacement { + return "", nil + } + + data, err := fileutils.ReadFile(p.fs, filepath.Join(outputDir, plan.ConfigName)) + if err != nil { + return "", fmt.Errorf("reading upstream Go vendor configuration:\n%w", err) + } + + return string(data), nil +} + +func (p *sourcePreparerImpl) publishGoVendor( + outputDir, specName, toolConfig string, plan *spec.GoVendorSpec, artifact govendor.Artifact, +) error { + if !plan.AllowReplacement && artifact.Name != plan.ArchiveFile { + return fmt.Errorf("generated Go archive %#q does not match planned name %#q", artifact.Name, plan.ArchiveFile) + } + + finalSpec, err := plan.Finalize(artifact.Name, artifact.License) + if err != nil { + return fmt.Errorf("finalizing Go vendor spec:\n%w", err) + } + + if err := fileutils.CopyFile(p.dryRunnable, p.fs, artifact.Path, + filepath.Join(outputDir, artifact.Name), fileutils.CopyFileOptions{}); err != nil { + return fmt.Errorf("publishing Go vendor archive:\n%w", err) + } + + for name, contents := range map[string][]byte{specName: finalSpec, plan.ConfigName: []byte(toolConfig)} { + if err := fileutils.WriteFile(p.fs, filepath.Join(outputDir, name), contents, fileperms.PublicFile); err != nil { + return fmt.Errorf("publishing Go vendor file %#q:\n%w", name, err) + } + } + + return p.recordGoVendorArchive(outputDir, artifact, plan.AllowReplacement) +} + +func (p *sourcePreparerImpl) checkGoVendorSourceCollision(outputDir string, names ...string) error { + for _, name := range names { + exists, err := fileutils.Exists(p.fs, filepath.Join(outputDir, name)) + if err != nil { + return fmt.Errorf("checking generated Go source path:\n%w", err) + } + + if exists { + return fmt.Errorf("generated Go vendor file %#q conflicts with an existing source", name) + } + } + + content, err := p.readSourcesFileIfExists(filepath.Join(outputDir, fedorasource.SourcesFileName)) + if err != nil { + return err + } + + lines, err := fedorasource.ReadSourcesFile(content) + if err != nil { + return fmt.Errorf("reading Go vendor 'sources' entries:\n%w", err) + } + + for _, line := range lines { + if line.Entry != nil && slices.Contains(names, line.Entry.Filename) { + return fmt.Errorf("generated Go vendor file %#q conflicts with an upstream 'sources' entry", line.Entry.Filename) + } + } + + return nil +} + +func (p *sourcePreparerImpl) recordGoVendorArchive(outputDir string, artifact govendor.Artifact, replace bool) error { + path := filepath.Join(outputDir, fedorasource.SourcesFileName) + + content, err := p.readSourcesFileIfExists(path) + if err != nil { + return err + } + + content, err = upsertGoVendorEntry(content, artifact, replace) + if err != nil { + return err + } + + if err := fileutils.WriteFile(p.fs, path, []byte(content), fileperms.PublicFile); err != nil { + return fmt.Errorf("recording Go vendor digest:\n%w", err) + } + + slog.Info("Recorded generated Go vendor archive", "filename", artifact.Name, + "hashType", fileutils.HashTypeSHA512, "hash", artifact.Digest) + + ignorePath := filepath.Join(outputDir, ".gitignore") + + ignore, err := p.readSourcesFileIfExists(ignorePath) + if err != nil { + return err + } + + for _, line := range strings.Split(ignore, "\n") { + if strings.TrimPrefix(line, "/") == artifact.Name { + return nil + } + } + + if err := fileutils.WriteFile(p.fs, ignorePath, + []byte(strings.TrimRight(ignore, "\n")+"\n/"+artifact.Name+"\n"), fileperms.PublicFile); err != nil { + return fmt.Errorf("ignoring generated Go archive in git:\n%w", err) + } + + return nil +} + +func upsertGoVendorEntry(content string, artifact govendor.Artifact, replace bool) (string, error) { + lines, err := fedorasource.ReadSourcesFile(strings.TrimRight(content, "\n")) + if err != nil { + return "", fmt.Errorf("parsing Go vendor source manifest:\n%w", err) + } + + entry := fedorasource.FormatSourcesEntry(artifact.Name, fileutils.HashTypeSHA512, artifact.Digest) + raw := make([]string, 0, len(lines)+1) + found := false + + for _, line := range lines { + value := line.Raw + if line.Entry != nil && line.Entry.Filename == artifact.Name { + if !replace { + return "", fmt.Errorf("go vendor source %#q already exists", artifact.Name) + } + + value, found = entry, true + } + + raw = append(raw, value) + } + + if !found { + raw = append(raw, entry) + } + + return strings.Join(raw, "\n") + "\n", nil +} diff --git a/internal/app/azldev/core/sources/govendor_config_internal_test.go b/internal/app/azldev/core/sources/govendor_config_internal_test.go new file mode 100644 index 00000000..f4cb285d --- /dev/null +++ b/internal/app/azldev/core/sources/govendor_config_internal_test.go @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package sources + +import ( + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/global/testctx" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/rpm/spec" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGoVendorConfigFileSelection(t *testing.T) { + for _, local := range []string{"", "[archive]\ntidy = false\n"} { + ctx := testctx.NewCtx() + require.NoError(t, fileutils.WriteFile(ctx.FS(), "/prepared/upstream.toml", + []byte("[licensing]\ndetector = 'askalono'\n"), fileperms.PublicFile)) + require.NoError(t, fileutils.WriteFile(ctx.FS(), "/project/local.toml", []byte(local), fileperms.PublicFile)) + preparer := &sourcePreparerImpl{fs: ctx.FS()} + plan := &spec.GoVendorSpec{AllowReplacement: true, ConfigName: "upstream.toml"} + config := &projectconfig.GoVendorConfig{ConfigFile: "/project/local.toml"} + actual, err := preparer.resolveGoVendorConfig("/prepared", plan, config) + require.NoError(t, err) + assert.Equal(t, local, actual, "explicit file replaces rather than merges upstream settings") + + config.ConfigFile = "" + actual, err = preparer.resolveGoVendorConfig("/prepared", plan, config) + require.NoError(t, err) + assert.Contains(t, actual, "askalono") + + config.ConfigFile = "/project/missing.toml" + _, err = preparer.resolveGoVendorConfig("/prepared", plan, config) + require.Error(t, err, "missing explicit file must not fall back to upstream") + assert.Contains(t, err.Error(), config.ConfigFile) + } +} diff --git a/internal/app/azldev/core/sources/govendor_existing_internal_test.go b/internal/app/azldev/core/sources/govendor_existing_internal_test.go new file mode 100644 index 00000000..3ab85fd8 --- /dev/null +++ b/internal/app/azldev/core/sources/govendor_existing_internal_test.go @@ -0,0 +1,135 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package sources + +import ( + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components/components_testutils" + "github.com/microsoft/azure-linux-dev-tools/internal/govendor" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders/fedorasource" + "github.com/microsoft/azure-linux-dev-tools/internal/rpm/mock" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/samber/lo" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +func TestPrepareExistingGoVendorReplacesPayload(t *testing.T) { + for _, conflict := range []string{"", "app-1.2-vendor.tar.bz2", "go-vendor-tools.toml"} { + t.Run("source-file-"+conflict, func(t *testing.T) { + testPrepareExistingGoVendor(t, conflict) + }) + } +} + +func testPrepareExistingGoVendor(t *testing.T, conflict string) { + t.Helper() + + ctx := newMockProcessorTestCtx() + component := components_testutils.NewMockComponent(gomock.NewController(t)) + component.EXPECT().GetName().AnyTimes().Return("app") + + config := &projectconfig.ComponentConfig{ + GoVendor: &projectconfig.GoVendorConfig{Enabled: lo.ToPtr(true)}, + } + if conflict != "" { + config.SourceFiles = []projectconfig.SourceFileReference{{ + Filename: conflict, ReplaceUpstream: true, Hash: strings.Repeat("a", 128), HashType: fileutils.HashTypeSHA512, + }} + } + + component.EXPECT().GetConfig().AnyTimes().Return(config) + + input := strings.Join([]string{ + "Name: app", "Version: 1.2", "Release: 1", "License: MIT AND Apache-2.0", + "Source0: app.tar.gz", "Source1: app-%{version}-vendor.tar.bz2", "Source2: go-vendor-tools.toml", + "%description", "app", "%prep", "%goprep", "tar -xf %{S:1}", + "%build", "%global gomodulesmode GO111MODULE=on", "%gobuild -o app .", + "%install", "%go_vendor_license_install -c %{S:2}", + "%files -f %{go_vendor_license_filelist}", "/usr/bin/app", "", + }, "\n") + + const ( + archiveName = "app-1.2-vendor.tar.bz2" + upstreamConfig = "[licensing]\ndetector = 'askalono'\n" + ) + + oldEntry := fedorasource.FormatSourcesEntry(archiveName, fileutils.HashTypeSHA512, strings.Repeat("a", 128)) + for name, data := range map[string]string{ + "app.spec": input, "app.tar.gz": "upstream source", archiveName: "old vendor", + "go-vendor-tools.toml": upstreamConfig, "sources": oldEntry + "\n", ".gitignore": "/" + archiveName + "\n", + } { + require.NoError(t, fileutils.WriteFile(ctx.FS(), "/output/"+name, []byte(data), fileperms.PublicFile)) + } + + ctx.CmdFactory.RunAndGetOutputHandler = func(command *exec.Cmd) (string, error) { + assert.Contains(t, strings.Join(command.Args, " "), "--verify") + assert.Contains(t, strings.Join(command.Args, " "), "MIT AND Apache-2.0") + staging := goVendorTestStaging(t, ctx) + data, err := fileutils.ReadFile(ctx.FS(), filepath.Join(staging, govendor.ConfigFilename)) + require.NoError(t, err) + assert.Equal(t, upstreamConfig, string(data)) + + prep, err := fileutils.ReadFile(ctx.FS(), filepath.Join(filepath.Dir(staging), "sources", "app.spec")) + require.NoError(t, err) + assert.NotContains(t, string(prep), "Source1:") + assert.NotContains(t, string(prep), "tar -xf %{S:1}") + + for name, data := range map[string]string{ + "azldev-generated-vendor.tar.bz2": "new vendor", "archive-name": archiveName + "\n", + "license": "Apache-2.0 AND MIT\n", "sources-list": "Source0: app.tar.gz\nSource2: go-vendor-tools.toml\n", + } { + require.NoError(t, fileutils.WriteFile(ctx.FS(), filepath.Join(staging, name), []byte(data), fileperms.PublicFile)) + } + + return "", nil + } + preparer := &sourcePreparerImpl{fs: ctx.FS(), dryRunnable: ctx, goVendorRunner: mock.NewRunner(ctx, "/mock.cfg")} + + err := preparer.prepareGoVendor(ctx, component, "/output") + if conflict != "" { + require.ErrorContains(t, err, "'source-files'") + + unchanged, readErr := fileutils.ReadFile(ctx.FS(), "/output/app.spec") + require.NoError(t, readErr) + assert.Equal(t, input, string(unchanged)) + + payload, readErr := fileutils.ReadFile(ctx.FS(), "/output/"+archiveName) + require.NoError(t, readErr) + assert.Equal(t, "old vendor", string(payload)) + + manifest, readErr := fileutils.ReadFile(ctx.FS(), "/output/sources") + require.NoError(t, readErr) + assert.Equal(t, oldEntry+"\n", string(manifest)) + + return + } + + require.NoError(t, err) + payload, err := fileutils.ReadFile(ctx.FS(), "/output/"+archiveName) + require.NoError(t, err) + assert.Equal(t, "new vendor", string(payload)) + + finalSpec, err := fileutils.ReadFile(ctx.FS(), "/output/app.spec") + require.NoError(t, err) + assert.Contains(t, string(finalSpec), "Source1: "+archiveName) + assert.Contains(t, string(finalSpec), "License: MIT AND Apache-2.0") + assert.NotContains(t, string(finalSpec), "azldev-go-vendor.toml") + + sources, err := fileutils.ReadFile(ctx.FS(), "/output/sources") + require.NoError(t, err) + digest, err := fileutils.ComputeFileHash(ctx.FS(), fileutils.HashTypeSHA512, "/output/"+archiveName) + require.NoError(t, err) + assert.Equal(t, fedorasource.FormatSourcesEntry(archiveName, fileutils.HashTypeSHA512, digest)+"\n", string(sources)) + ignore, err := fileutils.ReadFile(ctx.FS(), "/output/.gitignore") + require.NoError(t, err) + assert.Equal(t, "/"+archiveName+"\n", string(ignore)) +} diff --git a/internal/app/azldev/core/sources/govendor_internal_test.go b/internal/app/azldev/core/sources/govendor_internal_test.go new file mode 100644 index 00000000..963aeeec --- /dev/null +++ b/internal/app/azldev/core/sources/govendor_internal_test.go @@ -0,0 +1,253 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package sources + +import ( + "context" + "errors" + "os/exec" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/components/components_testutils" + "github.com/microsoft/azure-linux-dev-tools/internal/global/testctx" + "github.com/microsoft/azure-linux-dev-tools/internal/govendor" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders" + "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders/fedorasource" + "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders/sourceproviders_test" + "github.com/microsoft/azure-linux-dev-tools/internal/rpm/mock" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/samber/lo" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" +) + +func goVendorTestSpec() string { + return strings.Join([]string{ + "Name: app", "Version: 1", "Release: 1", "Summary: app", "License: MIT", + "Source0: app.tar.gz", "Patch0: fix.patch", "BuildRequires: golang(example.com/dep)", + "%description", "app", "%prep", "%goprep -p1", "%build", "%gobuild -o app .", + "%install", "install -D app %{buildroot}%{_bindir}/app", "%files", "%{_bindir}/app", "", + }, "\n") +} + +func TestPrepareSourcesGoVendor(t *testing.T) { + for _, skipLookaside := range []bool{false, true} { + t.Run(map[bool]string{false: "prepare", true: "render"}[skipLookaside], func(t *testing.T) { + ctx := newMockProcessorTestCtx() + ctrl := gomock.NewController(t) + component := components_testutils.NewMockComponent(ctrl) + manager := sourceproviders_test.NewMockSourceManager(ctrl) + config := &projectconfig.ComponentConfig{ + GoVendor: &projectconfig.GoVendorConfig{ + Enabled: lo.ToPtr(true), PrepPackages: []string{"git"}, ConfigFile: "/project/vendor.toml", + }, + Overlays: []projectconfig.ComponentOverlay{{ + Type: "spec-set-tag", Tag: "Summary", Value: "overlaid", + }}, + } + + const settings = "# reviewed settings\n[archive]\nuse_module_proxy = true\n" + require.NoError(t, fileutils.WriteFile(ctx.FS(), config.GoVendor.ConfigFile, + []byte(settings), fileperms.PublicFile)) + + component.EXPECT().GetName().AnyTimes().Return("app") + component.EXPECT().GetConfig().AnyTimes().Return(config) + manager.EXPECT().FetchComponent(gomock.Any(), component, "/output", gomock.Any()).DoAndReturn( + func(_ context.Context, _ any, _ string, _ ...sourceproviders.FetchComponentOption) error { + for name, content := range map[string]string{ + "app.spec": goVendorTestSpec(), "app.tar.gz": "upstream", "fix.patch": "patch", + "sources": "# upstream\n", ".gitignore": "/app.tar.gz\n", + } { + require.NoError(t, fileutils.WriteFile(ctx.FS(), "/output/"+name, []byte(content), fileperms.PublicFile)) + } + + return nil + }) + manager.EXPECT().FetchFiles(gomock.Any(), component, "/output").Return(nil) + + var calls []string + + ctx.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + calls = append(calls, strings.Join(cmd.Args, " ")) + + return nil + } + ctx.CmdFactory.RunAndGetOutputHandler = func(cmd *exec.Cmd) (string, error) { + assert.Contains(t, cmd.Args, "--unpriv") + assert.Contains(t, cmd.Args, "--enable-network") + assert.Contains(t, strings.Join(cmd.Args, " "), "go_vendor_archive") + staging := goVendorTestStaging(t, ctx) + stagedConfig, err := fileutils.ReadFile(ctx.FS(), filepath.Join(staging, govendor.ConfigFilename)) + require.NoError(t, err) + assert.Equal(t, settings, string(stagedConfig)) + + prep, err := fileutils.ReadFile(ctx.FS(), filepath.Join(filepath.Dir(staging), "sources", "app.spec")) + require.NoError(t, err) + assert.Contains(t, string(prep), "Summary: overlaid", "overlays must run before generation") + + for name, content := range map[string]string{ + "app-vendor.tar.bz2": "generated archive", + "archive-name": "app-vendor.tar.bz2\n", + "license": "Apache-2.0 AND MIT\n", + "sources-list": "Source0: app.tar.gz\nPatch0: fix.patch\n", + } { + require.NoError(t, fileutils.WriteFile(ctx.FS(), + filepath.Join(staging, name), []byte(content), fileperms.PublicFile)) + } + + return "", nil + } + + options := []PreparerOption{WithGoVendorRunner(mock.NewRunner(ctx, "/mock.cfg"))} + if skipLookaside { + options = append(options, WithSkipLookaside()) + } + + preparer, err := NewPreparer(manager, ctx.FS(), ctx, ctx, options...) + require.NoError(t, err) + require.NoError(t, preparer.PrepareSources(ctx, component, "/output", true)) + output, err := fileutils.ReadFile(ctx.FS(), "/output/app.spec") + require.NoError(t, err) + assert.Contains(t, string(output), "License: Apache-2.0 AND MIT") + assert.Contains(t, string(output), "tar -xf %{SOURCE1}") + assert.NotContains(t, string(output), "BuildRequires: golang(") + + publishedConfig, err := fileutils.ReadFile(ctx.FS(), "/output/"+govendor.ConfigFilename) + require.NoError(t, err) + assert.Equal(t, settings, string(publishedConfig)) + + digest, err := fileutils.ComputeFileHash(ctx.FS(), fileutils.HashTypeSHA512, "/output/app-vendor.tar.bz2") + require.NoError(t, err) + sources, err := fileutils.ReadFile(ctx.FS(), "/output/sources") + require.NoError(t, err) + assert.Contains(t, string(sources), "# upstream\n") + assert.Contains(t, string(sources), + fedorasource.FormatSourcesEntry("app-vendor.tar.bz2", fileutils.HashTypeSHA512, digest)) + ignored, err := fileutils.ReadFile(ctx.FS(), "/output/.gitignore") + require.NoError(t, err) + assert.Equal(t, "/app.tar.gz\n/app-vendor.tar.bz2\n", string(ignored)) + assert.Contains(t, strings.Join(calls, "\n"), "--install go-vendor-tools askalono-cli") + assert.Contains(t, strings.Join(calls, "\n"), " git") + assert.Contains(t, strings.Join(calls, "\n"), "--scrub all") + assert.NotContains(t, strings.Join(calls, "\n"), "go2rpm") + + matches, err := fileutils.Glob(ctx.FS(), "/tmp/azldev-go-vendor-*") + require.NoError(t, err) + assert.Empty(t, matches) + }) + } +} + +func goVendorTestStaging(t *testing.T, ctx *testctx.TestCtx) string { + t.Helper() + + matches, err := fileutils.Glob(ctx.FS(), "/tmp/azldev-go-vendor-*/work/azldev-go-vendor.toml") + require.NoError(t, err) + require.Len(t, matches, 1) + + return filepath.Dir(matches[0]) +} + +func TestGoVendorFailuresDoNotPublishSpec(t *testing.T) { + failure := errors.New("tool failed") + + for _, stage := range []string{ + "init", "install", "generate", "missing-archive", "empty-archive", + "license", "missing-source", "source-list", "scrub", "archive-name", "unexpected-name", + } { + t.Run(stage, func(t *testing.T) { + ctx := newMockProcessorTestCtx() + ctrl := gomock.NewController(t) + component := components_testutils.NewMockComponent(ctrl) + component.EXPECT().GetName().AnyTimes().Return("app") + component.EXPECT().GetConfig().AnyTimes().Return(&projectconfig.ComponentConfig{ + GoVendor: &projectconfig.GoVendorConfig{Enabled: lo.ToPtr(true)}, + }) + require.NoError(t, fileutils.WriteFile(ctx.FS(), + "/output/app.spec", []byte(goVendorTestSpec()), fileperms.PublicFile)) + require.NoError(t, fileutils.WriteFile(ctx.FS(), "/output/app.tar.gz", []byte("source"), fileperms.PublicFile)) + ctx.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + if slices.Contains(cmd.Args, "--"+stage) { + return failure + } + + return nil + } + ctx.CmdFactory.RunAndGetOutputHandler = func(_ *exec.Cmd) (string, error) { + if stage == "generate" { + return "license failure details", failure + } + + staging := goVendorTestStaging(t, ctx) + files := map[string]string{ + "app-vendor.tar.bz2": "archive", "license": "MIT\n", "sources-list": "Source0: app.tar.gz\n", + "archive-name": "app-vendor.tar.bz2\n", + } + + switch stage { + case "missing-archive": + delete(files, "app-vendor.tar.bz2") + case "empty-archive": + files["app-vendor.tar.bz2"] = "" + case "license": + files["license"] = "bad\nLicense" + case "missing-source": + files["sources-list"] += "Patch0: absent.patch\n" + case "source-list": + files["sources-list"] = "Source0: /etc/passwd\n" + case "archive-name": + files["archive-name"] = "../outside.tar.bz2\n" + case "unexpected-name": + files["archive-name"] = "other.tar.bz2\n" + } + + for name, data := range files { + require.NoError(t, fileutils.WriteFile(ctx.FS(), filepath.Join(staging, name), []byte(data), fileperms.PublicFile)) + } + + return "", nil + } + p := &sourcePreparerImpl{fs: ctx.FS(), dryRunnable: ctx, goVendorRunner: mock.NewRunner(ctx, "/mock.cfg")} + err := p.prepareGoVendor(ctx, component, "/output") + require.Error(t, err) + + unchanged, readErr := fileutils.ReadFile(ctx.FS(), "/output/app.spec") + require.NoError(t, readErr) + assert.Equal(t, goVendorTestSpec(), string(unchanged)) + + exists, readErr := fileutils.Exists(ctx.FS(), "/output/app-vendor.tar.bz2") + require.NoError(t, readErr) + assert.False(t, exists) + }) + } +} + +func TestGoVendorSourceCollisions(t *testing.T) { + for _, name := range []string{"app-vendor.tar.bz2", govendor.ConfigFilename, "sources", "invalid-sources"} { + t.Run(name, func(t *testing.T) { + ctx := testctx.NewCtx() + preparer := &sourcePreparerImpl{fs: ctx.FS()} + content := "existing" + + filename := name + switch name { + case "sources": + content = fedorasource.FormatSourcesEntry( + "app-vendor.tar.bz2", fileutils.HashTypeSHA512, strings.Repeat("a", 128)) + "\n" + case "invalid-sources": + filename = "sources" + content = "SHA512 (broken" + } + + require.NoError(t, fileutils.WriteFile(ctx.FS(), "/output/"+filename, []byte(content), fileperms.PublicFile)) + require.Error(t, preparer.checkGoVendorSourceCollision("/output", "app-vendor.tar.bz2", govendor.ConfigFilename)) + }) + } +} diff --git a/internal/app/azldev/core/sources/sourceprep.go b/internal/app/azldev/core/sources/sourceprep.go index 4edeb30b..8759787d 100644 --- a/internal/app/azldev/core/sources/sourceprep.go +++ b/internal/app/azldev/core/sources/sourceprep.go @@ -24,6 +24,7 @@ import ( "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders" "github.com/microsoft/azure-linux-dev-tools/internal/providers/sourceproviders/fedorasource" + "github.com/microsoft/azure-linux-dev-tools/internal/rpm/mock" "github.com/microsoft/azure-linux-dev-tools/internal/rpm/spec" "github.com/microsoft/azure-linux-dev-tools/internal/utils/dirdiff" "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" @@ -133,7 +134,8 @@ func WithSpecEditor(mode spec.EditorMode) PreparerOption { // component fetching ([FedoraSourceDownloader.ExtractSourcesFromRepo]). // Git-tracked files (spec, patches, scripts, configs) are still fetched from // the upstream clone. This is useful for rendering, where only the spec and -// sidecar files are needed and downloading large source tarballs is unnecessary. +// sidecar files are normally needed. Enabled Go vendor conversion overrides this +// optimization because its archive and license expression require prepared sources. func WithSkipLookaside() PreparerOption { return func(p *sourcePreparerImpl) { p.skipLookaside = true @@ -184,11 +186,12 @@ func WithAllowNoHashes() PreparerOption { // Standard implementation of the [SourcePreparer] interface. type sourcePreparerImpl struct { - sourceManager sourceproviders.SourceManager - specEditor spec.EditorMode - fs opctx.FS - eventListener opctx.EventListener - dryRunnable opctx.DryRunnable + sourceManager sourceproviders.SourceManager + specEditor spec.EditorMode + fs opctx.FS + eventListener opctx.EventListener + dryRunnable opctx.DryRunnable + goVendorRunner *mock.Runner // withGitRepo, when true, enables dist-git creation by preserving the // upstream .git directory and generating synthetic commit history. @@ -297,6 +300,8 @@ func NewPreparer( func (p *sourcePreparerImpl) PrepareSources( ctx context.Context, component components.Component, outputDir string, applyOverlays bool, ) error { + p = p.withRequiredDownloads(component, applyOverlays) + if applyOverlays { if err := p.validateArchiveOverlayConfig(component); err != nil { return err @@ -360,6 +365,21 @@ func (p *sourcePreparerImpl) PrepareSources( return nil } +func (p *sourcePreparerImpl) withRequiredDownloads( + component components.Component, applyConversion bool, +) *sourcePreparerImpl { + if applyConversion && p.skipLookaside && component.GetConfig().GoVendor.IsEnabled() { + effective := *p + effective.skipLookaside = false + + slog.Info("Go vendoring requires source downloads, including during rendering", "component", component.GetName()) + + return &effective + } + + return p +} + func (p *sourcePreparerImpl) validateArchiveOverlayConfig(component components.Component) error { config := component.GetConfig() if err := config.ValidateArchiveOverlays(p.allowNoHashes); err != nil { @@ -394,6 +414,10 @@ func (p *sourcePreparerImpl) applyOverlaysToSources( component.GetName(), err) } + if err := p.prepareGoVendor(ctx, component, outputDir); err != nil { + return nil, fmt.Errorf("preparing Go vendor sources for component %#q:\n%w", component.GetName(), err) + } + return repackedArchives, nil } @@ -740,6 +764,8 @@ func removeSubmoduleEntries(fs opctx.FS, repo *gogit.Repository, repoDir string) func (p *sourcePreparerImpl) DiffSources( ctx context.Context, component components.Component, baseDir string, ) (result *dirdiff.DiffResult, err error) { + p = p.withRequiredDownloads(component, true) + event := p.eventListener.StartEvent("Computing overlay diff", "component", component.GetName()) defer event.End() diff --git a/internal/fingerprint/doc.go b/internal/fingerprint/doc.go index fc655afa..5ad05185 100644 --- a/internal/fingerprint/doc.go +++ b/internal/fingerprint/doc.go @@ -3,7 +3,7 @@ // Package fingerprint computes deterministic identity fingerprints for components. // A fingerprint captures all resolved build inputs so that changes to any input -// (config fields, spec content, overlay files, distro context, upstream refs, or +// (config fields, spec content, overlay/vendor config files, distro context, upstream refs, or // ManualBump) produce a different fingerprint. // // The primary entry point is [ComputeIdentity], which takes a resolved diff --git a/internal/fingerprint/fingerprint.go b/internal/fingerprint/fingerprint.go index 391559d6..6ab13bb1 100644 --- a/internal/fingerprint/fingerprint.go +++ b/internal/fingerprint/fingerprint.go @@ -42,6 +42,9 @@ type ComponentInputs struct { // source file's basename and content. Keyed by index rather than path to avoid // checkout-location dependence. OverlayFileHashes map[string]string `json:"overlayFileHashes,omitempty"` + // GoVendorConfigHash records the contents of an explicitly selected vendor + // configuration without depending on its checkout-specific path. + GoVendorConfigHash string `json:"goVendorConfigHash,omitempty"` // ManualBump is the manual rebuild counter from the lock file. Almost always 0; // used for mass-rebuild scenarios. ManualBump int `json:"manualBump"` @@ -62,13 +65,13 @@ type IdentityOptions struct { // // This is caller-provided because resolving it requires network access (upstream clone) or // filesystem traversal (local content hash). [ComputeIdentity] is a pure combiner — it does - // not perform I/O beyond reading overlay files. Callers should resolve source identity via + // not perform I/O beyond reading declared input files. Callers should resolve source identity via // SourceManager.ResolveSourceIdentity before calling [ComputeIdentity]. SourceIdentity string } // ComputeIdentity computes the fingerprint for a component from its resolved config -// and additional context. The fs parameter is used to read overlay source file +// and additional context. The fs parameter is used to read declared input file // contents for hashing; spec content identity is provided via [IdentityOptions.SourceIdentity]. // // This function is a deterministic combiner: given the same resolved inputs it always @@ -141,6 +144,11 @@ func ComputeIdentity( inputs.OverlayFileHashes = overlayHashes + inputs.GoVendorConfigHash, err = component.GoVendor.SourceContentIdentity(fs) + if err != nil { + return nil, fmt.Errorf("hashing Go vendor configuration:\n%w", err) + } + // 5. Combine all inputs into the overall fingerprint. return &ComponentIdentity{ Fingerprint: combineInputs(inputs), @@ -158,6 +166,10 @@ func combineInputs(inputs ComponentInputs) string { writeField(hasher, "manual_bump", strconv.Itoa(inputs.ManualBump)) writeField(hasher, "release_ver", inputs.ReleaseVer) + if inputs.GoVendorConfigHash != "" { + writeField(hasher, "go_vendor_config", inputs.GoVendorConfigHash) + } + // Overlay file hashes in sorted key order for determinism. if len(inputs.OverlayFileHashes) > 0 { keys := make([]string, 0, len(inputs.OverlayFileHashes)) diff --git a/internal/fingerprint/govendor_test.go b/internal/fingerprint/govendor_test.go new file mode 100644 index 00000000..cd47c383 --- /dev/null +++ b/internal/fingerprint/govendor_test.go @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package fingerprint_test + +import ( + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/fingerprint" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/samber/lo" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGoVendorConfigFileFingerprint(t *testing.T) { + ctx := newTestFS(t, map[string]string{ + "/checkout-one/vendor.toml": "[archive]\ntidy = false\n", + "/checkout-two/vendor.toml": "[archive]\ntidy = false\n", + "/empty.toml": "", + }) + component := projectconfig.ComponentConfig{GoVendor: &projectconfig.GoVendorConfig{ + Enabled: lo.ToPtr(true), ConfigFile: "/checkout-one/vendor.toml", + }} + compute := func(config projectconfig.ComponentConfig) *fingerprint.ComponentIdentity { + t.Helper() + + result, err := fingerprint.ComputeIdentity(ctx.FS(), config, testReleaseVer, fingerprint.IdentityOptions{}) + require.NoError(t, err) + + return result + } + initial := compute(component) + require.NotEmpty(t, initial.Inputs.GoVendorConfigHash) + assert.Equal(t, "/checkout-one/vendor.toml", component.GoVendor.ConfigFile) + component.GoVendor.ConfigFile = "/checkout-two/vendor.toml" + assert.Equal(t, initial.Fingerprint, compute(component).Fingerprint, "checkout location must not affect identity") + + require.NoError(t, fileutils.WriteFile(ctx.FS(), component.GoVendor.ConfigFile, + []byte("[archive]\ntidy = true\n"), fileperms.PublicFile)) + assert.NotEqual(t, initial.Fingerprint, compute(component).Fingerprint, + "file-only edits must invalidate the fingerprint") + component.GoVendor.ConfigFile = "/empty.toml" + emptyFile := compute(component) + component.GoVendor.ConfigFile = "" + noFile := compute(component) + assert.Empty(t, noFile.Inputs.GoVendorConfigHash) + assert.NotEqual(t, emptyFile.Fingerprint, noFile.Fingerprint, "an empty override is not upstream fallback") +} + +func TestGoVendorConfigFileFingerprintErrors(t *testing.T) { + ctx := newTestFS(t, map[string]string{"/bad.toml": "["}) + for _, path := range []string{"/missing.toml", "/bad.toml"} { + _, err := fingerprint.ComputeIdentity(ctx.FS(), projectconfig.ComponentConfig{ + GoVendor: &projectconfig.GoVendorConfig{Enabled: lo.ToPtr(true), ConfigFile: path}, + }, testReleaseVer, fingerprint.IdentityOptions{}) + require.Error(t, err) + assert.Contains(t, err.Error(), path) + } +} diff --git a/internal/govendor/generate.go b/internal/govendor/generate.go new file mode 100644 index 00000000..3d8fa4a0 --- /dev/null +++ b/internal/govendor/generate.go @@ -0,0 +1,325 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// Package govendor generates vendor artifacts without resolving components or editing specs. +package govendor + +import ( + "cmp" + "context" + "errors" + "fmt" + "log/slog" + "path/filepath" + "strings" + "time" + + "github.com/kballard/go-shellquote" + "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx" + "github.com/microsoft/azure-linux-dev-tools/internal/rpm/mock" + "github.com/microsoft/azure-linux-dev-tools/internal/rpm/spectool" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/pelletier/go-toml/v2" +) + +const ( + // ConfigFilename is the tool configuration filename inside generation staging. + ConfigFilename = "azldev-go-vendor.toml" + sourceMount = "/azldev-go-vendor" + outputMount = "/azldev-go-scratch" + topDir = "/tmp/azldev-go-rpm" + cleanupTimeout = 5 * time.Minute +) + +// Request supplies prepared spec bytes and input files. WorkDir must be a private +// scratch directory outside SourceDir; the caller owns its cleanup and dry-run policy. +// PrepSpec must record 'azldev-go-module-root' and 'azldev-go-archive-name' in RPM's topdir. +type Request struct { + SourceDir, WorkDir, SpecName string + ArchiveFile, ArchiveSuffix string + ModuleDir, ConfigContents string + ExpectedLicense string + PrepSpec []byte + Packages []string + configModuleDir string +} + +// Artifact is validated generation output, ready for publication by the caller. +type Artifact struct { + Path, Name, License, Digest string +} + +// Generate prepares inputs and generates an archive in one isolated mock root. +// It never changes SourceDir or publishes a spec or source manifest. +func Generate(ctx context.Context, fs opctx.FS, base *mock.Runner, request Request) (Artifact, error) { + request, err := resolveModuleDir(request) + if err != nil { + return Artifact{}, err + } + + staging := filepath.Join(request.WorkDir, "work") + + sourceCopy := filepath.Join(request.WorkDir, "sources") + if err := stage(fs, request, sourceCopy, staging); err != nil { + return Artifact{}, err + } + + runner := base.Clone().WithBaseDir(filepath.Join(request.WorkDir, "mock")).EnableNetwork().WithUnprivileged() + runner.AddBindMount(sourceCopy, sourceMount).AddBindMount(staging, outputMount) + + if err := execute(ctx, runner, request); err != nil { + return Artifact{}, err + } + + // Snapshot only regular outputs into a host-only directory. Subsequent + // validation, hashing and publication never reopen chroot-writable paths. + trusted := filepath.Join(request.WorkDir, "artifacts") + if err := fileutils.MkdirAll(fs, trusted); err != nil { + return Artifact{}, fmt.Errorf("creating trusted Go vendor output directory:\n%w", err) + } + + for _, name := range []string{request.ArchiveFile, "archive-name", "license", "sources-list"} { + if err := fileutils.CopyFileCrossFS(nilDryRun{}, fs, filepath.Join(staging, name), + fs, filepath.Join(trusted, name), fileutils.CopyFileOptions{NoFollow: true}); err != nil { + return Artifact{}, fmt.Errorf("collecting Go vendor output %#q:\n%w", name, err) + } + } + + if err := validateInputs(fs, trusted, request.SourceDir); err != nil { + return Artifact{}, err + } + + return readArtifact(fs, trusted, request) +} + +type nilDryRun struct{} + +func (nilDryRun) DryRun() bool { return false } + +func resolveModuleDir(request Request) (Request, error) { + var settings map[string]any + if err := toml.Unmarshal([]byte(request.ConfigContents), &settings); err != nil { + return request, fmt.Errorf("parsing Go vendor tool configuration:\n%w", err) + } + + general, _ := settings["general"].(map[string]any) + if raw, exists := general["go_mod_dir"]; exists { + directory, ok := raw.(string) + if !ok { + return request, errors.New("go vendor 'general.go_mod_dir' must be a string") + } + + request.configModuleDir = directory + } + + for _, directory := range []string{request.ModuleDir, request.configModuleDir} { + if directory == "" || directory == "." { + continue + } + + if err := fileutils.ValidateRelPath(directory); err != nil { + return request, fmt.Errorf("invalid Go vendor module directory:\n%w", err) + } + + if strings.ContainsAny(directory, "%\r\n") { + return request, errors.New("go vendor module directory must not contain RPM macros or newlines") + } + } + + if request.configModuleDir != "" { + if request.ModuleDir != "" && request.ModuleDir != request.configModuleDir { + return request, errors.New("'module-dir' conflicts with upstream 'general.go_mod_dir'; " + + "use 'config-file' to replace the upstream settings") + } + + // Native tools preserve this subdirectory in the archive and use it for + // licensing. Keep their root and config intact, without descending twice. + request.ModuleDir = "" + } + + return request, nil +} + +func stage(fs opctx.FS, request Request, sourceCopy, staging string) error { + for _, name := range []string{request.SpecName, request.ArchiveFile} { + if err := fileutils.ValidateFilename(name); err != nil { + return fmt.Errorf("invalid Go vendor staging filename:\n%w", err) + } + } + + if err := fileutils.MkdirAll(fs, staging); err != nil { + return fmt.Errorf("creating Go vendor staging directory:\n%w", err) + } + + if err := fileutils.CopyDirRecursiveCrossFS(fs, request.SourceDir, fs, sourceCopy, + fileutils.CopyDirOptions{CopyFileOptions: fileutils.CopyFileOptions{PreserveFileMode: true}}); err != nil { + return fmt.Errorf("copying Go sources for isolated prep:\n%w", err) + } + + for path, data := range map[string][]byte{ + filepath.Join(sourceCopy, request.SpecName): request.PrepSpec, + filepath.Join(staging, ConfigFilename): []byte(request.ConfigContents), + } { + if err := fileutils.WriteFile(fs, path, data, fileperms.PublicFile); err != nil { + return fmt.Errorf("staging Go vendor input %#q:\n%w", path, err) + } + } + + return nil +} + +func execute(ctx context.Context, runner *mock.Runner, request Request) (err error) { + defer func() { + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), cleanupTimeout) + defer cancel() + + if cleanupErr := runner.ScrubRoot(cleanupCtx); cleanupErr != nil { + slog.Error("Failed to clean Go vendor mock root", "error", cleanupErr) + + if err == nil { + err = cleanupErr + } + } + }() + + if err := runner.InitRoot(ctx); err != nil { + return fmt.Errorf("initializing Go vendor mock root:\n%w", err) + } + + packages := append([]string{ + "go-vendor-tools", "askalono-cli", "golang", "go-rpm-macros", "rpm-build", "rpmdevtools", + "rpmautospec", "git", "python3-click", + }, request.Packages...) + if err := runner.InstallPackages(ctx, packages); err != nil { + return fmt.Errorf("installing Go vendor prerequisites:\n%w", err) + } + + cmd, err := runner.CmdInChroot(ctx, generationCommand(request, runner.ConfigOpts()["target_arch"]), false) + if err != nil { + return fmt.Errorf("constructing Go vendor command:\n%w", err) + } + + output, err := cmd.RunAndGetOutput(ctx) + if err != nil { + return fmt.Errorf("go vendor generation or license detection failed; use 'prep-packages' for missing prep tools "+ + "and 'config-file' for reviewed license exceptions:\n%s\n%w", output, err) + } + + return nil +} + +func generationCommand(request Request, targetArch string) []string { + quote := shellquote.Join + + prepArgs := []string{ + "rpmbuild", "-bp", "--nodeps", "--define", "_topdir " + topDir, + "--define", "_builddir " + topDir + "/BUILD", "--define", "_sourcedir " + sourceMount, + } + if targetArch != "" { + prepArgs = append(prepArgs, "--target", targetArch) + } + + licenseArgs := []string{ + "go_vendor_license", "--config", outputMount + "/" + ConfigFilename, "--path", ".", + "report", "--no-prompt", "--autofill", "off", "expression", + } + if request.ExpectedLicense != "" { + licenseArgs = append(licenseArgs, "--verify", request.ExpectedLicense) + } + + commands := []string{ + "cd " + sourceMount, + "mkdir -p " + topDir + "/BUILD", + quote("git", "config", "--global", "--add", "safe.directory", sourceMount), + quote("rpmautospec", "process-distgit", request.SpecName, request.SpecName), + quote(append(prepArgs, sourceMount+"/"+request.SpecName)...), + quote("cp", topDir+"/azldev-go-archive-name", outputMount+"/archive-name"), + `IFS= read -r module_root < ` + topDir + "/azldev-go-module-root", + `cd "$module_root"`, + "cd " + quote(cmp.Or(request.ModuleDir, ".")), + quote("test", "-f", filepath.Join(request.configModuleDir, "go.mod")), + `module_root=$(pwd -P)`, + "export GOTOOLCHAIN=local", + "cd " + sourceMount, + quote("go_vendor_archive", "create", "--config", outputMount+"/"+ConfigFilename, + "--output", outputMount+"/"+request.ArchiveFile) + ` "$module_root"`, + `cd "$module_root"`, + quote("tar", "-xf", outputMount+"/"+request.ArchiveFile), + quote(licenseArgs...) + " > " + outputMount + "/license", + "cd " + sourceMount, + quote("spectool", "--define", "_sourcedir "+sourceMount, "-l", "-a", request.SpecName) + + " > " + outputMount + "/sources-list", + } + + return []string{"bash", "-euo", "pipefail", "-c", strings.Join(commands, "\n")} +} + +func readArtifact(fs opctx.FS, staging string, request Request) (Artifact, error) { + result := Artifact{Path: filepath.Join(staging, request.ArchiveFile)} + + info, err := fs.Stat(result.Path) + if err != nil { + return result, fmt.Errorf("missing generated Go vendor archive:\n%w", err) + } + + if !info.Mode().IsRegular() || info.Size() == 0 { + return result, errors.New("generated Go vendor archive must be a nonempty regular file") + } + + for name, destination := range map[string]*string{"archive-name": &result.Name, "license": &result.License} { + data, err := fileutils.ReadFile(fs, filepath.Join(staging, name)) + if err != nil { + return result, fmt.Errorf("reading Go vendor output %#q:\n%w", name, err) + } + + *destination = strings.TrimSpace(string(data)) + if *destination == "" || strings.ContainsAny(*destination, "%\r\n") { + return result, fmt.Errorf("invalid Go vendor output %#q: %#q", name, *destination) + } + } + + if err := fileutils.ValidateFilename(result.Name); err != nil { + return result, fmt.Errorf("invalid expanded Go vendor archive name:\n%w", err) + } + + if !strings.HasSuffix(result.Name, request.ArchiveSuffix) { + return result, fmt.Errorf("unexpected expanded Go vendor archive name %#q", result.Name) + } + + result.Digest, err = fileutils.ComputeFileHash(fs, fileutils.HashTypeSHA512, result.Path) + if err != nil { + return result, fmt.Errorf("hashing generated Go vendor archive:\n%w", err) + } + + return result, nil +} + +func validateInputs(fs opctx.FS, staging, sourceDir string) error { + data, err := fileutils.ReadFile(fs, filepath.Join(staging, "sources-list")) + if err != nil { + return fmt.Errorf("reading prepared Go source list:\n%w", err) + } + + for _, line := range strings.Split(strings.TrimSpace(string(data)), "\n") { + if line == "" { + continue + } + + files := spectool.ParseSpectoolOutput(line) + if len(files) != 1 || strings.Contains(files[0], "%") { + return fmt.Errorf("cannot resolve Go source/patch reference %#q", line) + } + + info, err := fs.Stat(filepath.Join(sourceDir, files[0])) + if err != nil { + return fmt.Errorf("missing prepared Go source/patch %#q:\n%w", files[0], err) + } + + if !info.Mode().IsRegular() { + return fmt.Errorf("go source/patch %#q is not a regular file", files[0]) + } + } + + return nil +} diff --git a/internal/govendor/generate_internal_test.go b/internal/govendor/generate_internal_test.go new file mode 100644 index 00000000..1d3f7b49 --- /dev/null +++ b/internal/govendor/generate_internal_test.go @@ -0,0 +1,211 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package govendor + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/global/testctx" + "github.com/microsoft/azure-linux-dev-tools/internal/rpm/mock" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGenerateArtifact(t *testing.T) { + for _, invalid := range []string{"", "archive-name", "license", "sources-list"} { + t.Run("invalid-"+invalid, func(t *testing.T) { + ctx := testctx.NewCtx() + ctx.CmdFactory.RegisterCommandInSearchPath(mock.MockBinary) + require.NoError(t, fileutils.WriteFile(ctx.FS(), "/input/app.spec", []byte("original"), fileperms.PublicFile)) + request := Request{ + SourceDir: "/input", WorkDir: "/scratch", SpecName: "app.spec", + ArchiveFile: "vendor.tar.bz2", ArchiveSuffix: ".tar.bz2", PrepSpec: []byte("prepared"), + } + ctx.CmdFactory.RunAndGetOutputHandler = func(command *exec.Cmd) (string, error) { + assert.Contains(t, command.Args, "--unpriv") + assert.Contains(t, command.Args, "--enable-network") + + for name, content := range map[string]string{ + request.ArchiveFile: "archive", "archive-name": "app-vendor.tar.bz2\n", + "license": "MIT\n", "sources-list": "Source0: app.spec\n", + } { + if name == invalid { + content = "%{unexpanded}" + } + + require.NoError(t, fileutils.WriteFile(ctx.FS(), + filepath.Join("/scratch/work", name), []byte(content), fileperms.PublicFile)) + } + + return "", nil + } + + artifact, err := Generate(ctx, ctx.FS(), mock.NewRunner(ctx, "/mock.cfg"), request) + if invalid != "" { + require.Error(t, err) + + return + } + + require.NoError(t, err) + assert.Equal(t, "/scratch/artifacts/vendor.tar.bz2", artifact.Path, + "published artifacts must come from the host-only snapshot") + assert.Equal(t, "app-vendor.tar.bz2", artifact.Name) + assert.Equal(t, "MIT", artifact.License) + digest, err := fileutils.ComputeFileHash(ctx.FS(), fileutils.HashTypeSHA512, artifact.Path) + require.NoError(t, err) + assert.Equal(t, digest, artifact.Digest) + + original, err := fileutils.ReadFile(ctx.FS(), "/input/app.spec") + require.NoError(t, err) + assert.Equal(t, "original", string(original)) + }) + } +} + +func TestGenerationCommand(t *testing.T) { + for _, expected := range []string{"", "MIT AND Apache-2.0"} { + command := generationCommand(Request{ + ArchiveFile: "app-vendor.tar.bz2", ModuleDir: "cmd/app", SpecName: "app.spec", ExpectedLicense: expected, + }, "aarch64") + assert.Equal(t, []string{"bash", "-euo", "pipefail", "-c"}, command[:4]) + script := command[4] + assert.Less(t, strings.Index(script, "rpmautospec"), strings.Index(script, "rpmbuild")) + assert.Less(t, strings.Index(script, "rpmbuild"), strings.Index(script, "go_vendor_archive")) + assert.Less(t, strings.Index(script, "go_vendor_archive"), strings.Index(script, "go_vendor_license")) + assert.Contains(t, script, "cd cmd/app") + assert.Contains(t, script, + "cd "+sourceMount+"\ngo_vendor_archive create --config "+outputMount+"/"+ConfigFilename+ + " --output "+outputMount+`/app-vendor.tar.bz2 "$module_root"`+"\n"+`cd "$module_root"`) + assert.Contains(t, script, "--target aarch64") + assert.Contains(t, script, "--no-prompt") + assert.Contains(t, script, "--autofill off") + assert.NotContains(t, script, "--idempotent") + assert.Contains(t, script, "cp "+topDir+"/azldev-go-archive-name "+outputMount+"/archive-name") + assert.Equal(t, expected != "", strings.Contains(script, "--verify")) + } +} + +func TestResolveModuleDir(t *testing.T) { + for _, testCase := range []struct { + name, config, module, want string + wantErr bool + }{ + {name: "default"}, + {name: "upstream", config: "[general]\ngo_mod_dir = 'cmd/app'\n[archive]\ntidy = false"}, + {name: "matching", config: "[general]\ngo_mod_dir = 'cmd/app'", module: "cmd/app"}, + {name: "conflicting", config: "[general]\ngo_mod_dir = 'upstream'", module: "cmd/app", wantErr: true}, + {name: "explicit", module: "cmd/app", want: "cmd/app"}, + {name: "root", config: "[general]\ngo_mod_dir = '.'"}, + {name: "absolute", config: "[general]\ngo_mod_dir = '/etc'", wantErr: true}, + {name: "traversal", config: "[general]\ngo_mod_dir = '../outside'", wantErr: true}, + {name: "macro", config: "[general]\ngo_mod_dir = '%{bad}'", wantErr: true}, + {name: "type", config: "[general]\ngo_mod_dir = 42", wantErr: true}, + {name: "toml", config: "[broken", wantErr: true}, + } { + t.Run(testCase.name, func(t *testing.T) { + original := Request{ModuleDir: testCase.module, ConfigContents: testCase.config} + + actual, err := resolveModuleDir(original) + if testCase.wantErr { + require.Error(t, err) + + return + } + + require.NoError(t, err) + assert.Equal(t, testCase.want, actual.ModuleDir) + assert.Equal(t, testCase.config, actual.ConfigContents) + assert.Equal(t, testCase.config, original.ConfigContents) + }) + } +} + +// Kernel-enforced O_NOFOLLOW cannot be exercised by the in-memory filesystem. +func TestGenerateRejectsSymlinkOutputs(t *testing.T) { + for _, name := range []string{"vendor.tar.bz2", "archive-name", "license", "sources-list"} { + t.Run(name, func(t *testing.T) { + ctx := testctx.NewCtx(testctx.WithHostFS()) + ctx.CmdFactory.RegisterCommandInSearchPath(mock.MockBinary) + + root := t.TempDir() + request := Request{ + SourceDir: filepath.Join(root, "input"), WorkDir: filepath.Join(root, "scratch"), SpecName: "app.spec", + ArchiveFile: "vendor.tar.bz2", ArchiveSuffix: ".tar.bz2", PrepSpec: []byte("prepared"), + } + require.NoError(t, fileutils.MkdirAll(ctx.FS(), request.SourceDir)) + require.NoError(t, fileutils.WriteFile(ctx.FS(), filepath.Join(request.SourceDir, "app.spec"), + []byte("original"), fileperms.PublicFile)) + target := filepath.Join(root, "host-only") + require.NoError(t, fileutils.WriteFile(ctx.FS(), target, []byte("host-only contents"), fileperms.PublicFile)) + ctx.CmdFactory.RunAndGetOutputHandler = func(_ *exec.Cmd) (string, error) { + for filename, content := range map[string]string{ + request.ArchiveFile: "archive", "archive-name": "app-vendor.tar.bz2\n", + "license": "MIT\n", "sources-list": "Source0: app.spec\n", + } { + path := filepath.Join(request.WorkDir, "work", filename) + if filename == name { + require.NoError(t, os.Symlink(target, path)) + } else { + require.NoError(t, fileutils.WriteFile(ctx.FS(), path, []byte(content), fileperms.PublicFile)) + } + } + + return "", nil + } + artifact, err := Generate(ctx, ctx.FS(), mock.NewRunner(ctx, "/mock.cfg"), request) + require.ErrorContains(t, err, "without following symlinks") + assert.Empty(t, artifact.Path) + }) + } +} + +func TestCollectOutputRejectsNonRegularFile(t *testing.T) { + ctx := testctx.NewCtx() + require.NoError(t, fileutils.MkdirAll(ctx.FS(), "/input/directory")) + err := fileutils.CopyFile(ctx, ctx.FS(), "/input/directory", "/output/file", + fileutils.CopyFileOptions{NoFollow: true}) + require.ErrorContains(t, err, "not a regular file") +} + +func TestGenerateUsesUpstreamModuleWithoutChangingInputs(t *testing.T) { + ctx := testctx.NewCtx() + ctx.CmdFactory.RegisterCommandInSearchPath(mock.MockBinary) + require.NoError(t, fileutils.WriteFile(ctx.FS(), "/input/app.spec", []byte("original"), fileperms.PublicFile)) + + settings := "[general]\ngo_mod_dir = 'cmd/app'\n" + ctx.CmdFactory.RunAndGetOutputHandler = func(command *exec.Cmd) (string, error) { + script := strings.Join(command.Args, " ") + assert.Contains(t, script, "test -f cmd/app/go.mod") + assert.NotContains(t, script, "cd cmd/app") + + config, err := fileutils.ReadFile(ctx.FS(), "/scratch/work/"+ConfigFilename) + require.NoError(t, err) + assert.Equal(t, settings, string(config)) + + for name, content := range map[string]string{ + "vendor.tar.bz2": "archive", "archive-name": "app-vendor.tar.bz2\n", + "license": "MIT\n", "sources-list": "Source0: app.spec\n", + } { + require.NoError(t, fileutils.WriteFile(ctx.FS(), filepath.Join("/scratch/work", name), + []byte(content), fileperms.PublicFile)) + } + + return "", nil + } + request := Request{ + SourceDir: "/input", WorkDir: "/scratch", SpecName: "app.spec", + ArchiveFile: "vendor.tar.bz2", ConfigContents: settings, + } + _, err := Generate(ctx, ctx.FS(), mock.NewRunner(ctx, "/mock.cfg"), request) + require.NoError(t, err) + assert.Equal(t, settings, request.ConfigContents) + assert.Empty(t, request.ModuleDir) +} diff --git a/internal/projectconfig/component.go b/internal/projectconfig/component.go index 6b56f9d8..be9e8c0e 100644 --- a/internal/projectconfig/component.go +++ b/internal/projectconfig/component.go @@ -374,6 +374,8 @@ type ComponentLockData struct { } // Defines a component. +// +//nolint:recvcheck // HashInclude must also support configs passed to hashstructure by value. type ComponentConfig struct { // The component's name; not actually present in serialized files. Name string `toml:"-" json:"name" table:",sortkey" fingerprint:"-"` @@ -432,6 +434,9 @@ type ComponentConfig struct { // Source file references for this component. SourceFiles []SourceFileReference `toml:"source-files,omitempty" json:"sourceFiles,omitempty" table:"-" jsonschema:"title=Source files,description=Source files to download for this component"` + // GoVendor converts supported Go application specs and generates their vendored sources. + GoVendor *GoVendorConfig `toml:"go-vendor,omitempty" json:"goVendor,omitempty" table:"-" jsonschema:"title=Go vendoring,description=Convert a Go application spec and generate its vendor archive during source preparation"` + // Per-package configuration overrides, keyed by exact binary package name. // Takes precedence over package-group defaults. Packages map[string]PackageConfig `toml:"packages,omitempty" json:"packages,omitempty" table:"-" validate:"dive" jsonschema:"title=Package overrides,description=Per-package configuration overrides keyed by exact binary package name"` @@ -450,6 +455,11 @@ type ComponentConfig struct { Tests *ComponentTestsConfig `toml:"tests,omitempty" json:"tests,omitempty" table:"-" jsonschema:"title=Tests,description=Per-component test or test-group references" fingerprint:"-"` } +// HashInclude preserves existing fingerprints when Go vendoring is not configured. +func (c ComponentConfig) HashInclude(field string, _ any) (bool, error) { + return field != "GoVendor" || c.GoVendor != nil, nil +} + // AllowedSourceFilesHashTypes defines the set of hash types that are supported // for use in [SourceFileReference] entries in component configs. // MD5 is excluded by design. @@ -480,9 +490,20 @@ func (c *ComponentConfig) MergeUpdatesFrom(other *ComponentConfig) error { c.OverlayFiles = otherOverlayFiles } + c.overrideGoVendorEnabled(other) + return nil } +func (c *ComponentConfig) overrideGoVendorEnabled(other *ComponentConfig) { + if other.GoVendor != nil && other.GoVendor.Enabled != nil { + config := *c.GoVendor + enabled := *other.GoVendor.Enabled + config.Enabled = &enabled + c.GoVendor = &config + } +} + func (c *ComponentConfig) resolveLocalCustomScriptPaths() { if c.Spec.SourceType != SpecSourceTypeLocal || c.Spec.Path == "" { return @@ -549,6 +570,8 @@ func (c *ComponentConfig) MergeOverridesFrom(other *ComponentConfig) error { c.OverlayFiles = otherOverlayFiles } + c.overrideGoVendorEnabled(other) + return nil } @@ -621,31 +644,18 @@ func ResolveComponentConfig( // Returns a copy of the component config with relative file paths converted to absolute // file paths (relative to referenceDir, not the current working directory). func (c *ComponentConfig) WithAbsolutePaths(referenceDir string) *ComponentConfig { - // Deep copy the input to avoid unexpected sharing. Make sure *not* to deep-copy - // the SourceConfigFile, as we *do* want to alias that pointer, sharing it across - // all configs that came from that source config file. - result := &ComponentConfig{ - Name: c.Name, - SourceConfigFile: c.SourceConfigFile, - upstreamCommitConfigFile: c.upstreamCommitConfigFile, - RenderedSpecDir: c.RenderedSpecDir, - Locked: deep.MustCopy(c.Locked), - Release: c.Release, - Spec: deep.MustCopy(c.Spec), - Build: deep.MustCopy(c.Build), - Render: c.Render, - SourceFiles: deep.MustCopy(c.SourceFiles), - Packages: deep.MustCopy(c.Packages), - Publish: deep.MustCopy(c.Publish), - // OverlayFiles is consumed after component config resolution; preserve it verbatim - // here so inherited patterns can be interpreted relative to the concrete component - // config file. - OverlayFiles: slices.Clone(c.OverlayFiles), - Tests: deep.MustCopy(c.Tests), - } + // Copy all configuration fields, excluding shared back-references which can + // point back into the component map and must retain their identity. + input := *c + input.SourceConfigFile, input.upstreamCommitConfigFile = nil, nil + result := deep.MustCopy(&input) + result.SourceConfigFile, result.upstreamCommitConfigFile = c.SourceConfigFile, c.upstreamCommitConfigFile // Fix up paths. result.Spec.Path = makeAbsolute(referenceDir, result.Spec.Path) + if result.GoVendor != nil { + result.GoVendor.ConfigFile = makeAbsolute(referenceDir, result.GoVendor.ConfigFile) + } for i := range result.SourceFiles { origin := &result.SourceFiles[i].Origin @@ -654,13 +664,8 @@ func (c *ComponentConfig) WithAbsolutePaths(referenceDir string) *ComponentConfi } } - // Copy and fix up overlays. - if c.Overlays != nil { - result.Overlays = make([]ComponentOverlay, len(c.Overlays)) - - for i := range result.Overlays { - result.Overlays[i] = *c.Overlays[i].WithAbsolutePaths(referenceDir) - } + for i := range result.Overlays { + result.Overlays[i] = *result.Overlays[i].WithAbsolutePaths(referenceDir) } return result diff --git a/internal/projectconfig/component_clone_internal_test.go b/internal/projectconfig/component_clone_internal_test.go new file mode 100644 index 00000000..2f6e4338 --- /dev/null +++ b/internal/projectconfig/component_clone_internal_test.go @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package projectconfig + +import ( + "testing" + + "github.com/samber/lo" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestComponentClonePreservesBackReferences(t *testing.T) { + source, pinSource := &ConfigFile{}, &ConfigFile{} + config := ComponentConfig{ + SourceConfigFile: source, upstreamCommitConfigFile: pinSource, + GoVendor: &GoVendorConfig{Enabled: lo.ToPtr(true), PrepPackages: []string{"git"}}, + Build: ComponentBuildConfig{Defines: map[string]string{"feature": "1"}}, + } + source.Components = map[string]ComponentConfig{"app": config} + pinSource.Components = source.Components + + resolved := config.WithAbsolutePaths("/project") + require.Same(t, source, resolved.SourceConfigFile) + require.Same(t, pinSource, resolved.upstreamCommitConfigFile) + require.NotSame(t, config.GoVendor, resolved.GoVendor) + require.Equal(t, config.GoVendor, resolved.GoVendor) + resolved.Build.Defines["feature"] = "2" + *resolved.GoVendor.Enabled = false + resolved.GoVendor.PrepPackages[0] = "changed" + assert.True(t, config.GoVendor.IsEnabled()) + + assert.Equal(t, "1", config.Build.Defines["feature"]) + assert.Equal(t, []string{"git"}, config.GoVendor.PrepPackages) +} diff --git a/internal/projectconfig/configfile.go b/internal/projectconfig/configfile.go index ee6f07c7..aee4be16 100644 --- a/internal/projectconfig/configfile.go +++ b/internal/projectconfig/configfile.go @@ -171,6 +171,10 @@ func validateComponentConfigs( } // Validate build configuration. + if err := component.GoVendor.Validate(); err != nil { + return fmt.Errorf("invalid 'go-vendor' config for component %#q:\n%w", componentName, err) + } + err := component.Build.Validate() if err != nil { return fmt.Errorf("invalid build config for component %#q:\n%w", componentName, err) diff --git a/internal/projectconfig/fingerprint_test.go b/internal/projectconfig/fingerprint_test.go index 836eee6f..881ea95f 100644 --- a/internal/projectconfig/fingerprint_test.go +++ b/internal/projectconfig/fingerprint_test.go @@ -36,6 +36,7 @@ func TestAllFingerprintedFieldsHaveDecision(t *testing.T) { reflect.TypeFor[projectconfig.Origin](), reflect.TypeFor[projectconfig.ReleaseConfig](), reflect.TypeFor[projectconfig.ComponentRenderConfig](), + reflect.TypeFor[projectconfig.GoVendorConfig](), } // Maps "StructName.FieldName" for every field that should carry a @@ -105,6 +106,9 @@ func TestAllFingerprintedFieldsHaveDecision(t *testing.T) { // SpecSource.Path — absolute path that varies by checkout location. // Spec content identity is captured separately via SourceIdentity. "SpecSource.Path": true, + + // GoVendorConfig.ConfigFile — absolute declaration path; contents are hashed separately. + "GoVendorConfig.ConfigFile": true, } // Collect all actual exclusions found via reflection, and flag invalid tag values. diff --git a/internal/projectconfig/govendor.go b/internal/projectconfig/govendor.go new file mode 100644 index 00000000..415a2a7c --- /dev/null +++ b/internal/projectconfig/govendor.go @@ -0,0 +1,102 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package projectconfig + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "strings" + + "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/pelletier/go-toml/v2" +) + +// GoVendorConfig describes a repeatable conversion, not a replacement spec. +type GoVendorConfig struct { + // Enabled is a pointer so an explicit false overrides inherited settings. + Enabled *bool `toml:"enabled,omitempty" json:"enabled,omitempty" jsonschema:"title=Enabled,description=Enable Go application vendoring"` + // ModuleDir is relative to the directory left by the spec's prep section. + ModuleDir string `toml:"module-dir,omitempty" json:"moduleDir,omitempty" jsonschema:"title=Module directory,description=Relative module directory within the prepared source tree; defaults to its root"` + // ConfigFile identifies a native go-vendor-tools TOML file, relative to the declaring + // config file. Resolved to an absolute path; its contents are fingerprinted separately. + ConfigFile string `toml:"config-file,omitempty" json:"configFile,omitempty" jsonschema:"title=Vendor configuration file,description=Path to a go-vendor-tools TOML file relative to the declaring component configuration. Replaces upstream settings when specified,example=go-vendor-tools.toml" fingerprint:"-"` + // PrepPackages supplies additional tools used by upstream prep commands. + PrepPackages []string `toml:"prep-packages,omitempty" json:"prepPackages,omitempty" jsonschema:"title=Prep packages,description=Additional RPM packages needed to execute the upstream prep section in mock"` +} + +// IsEnabled reports whether vendoring was explicitly enabled. +func (c *GoVendorConfig) IsEnabled() bool { + return c != nil && c.Enabled != nil && *c.Enabled +} + +// Validate checks configuration values before any source processing. +func (c *GoVendorConfig) Validate() error { + if c == nil { + return nil + } + + if c.ModuleDir != "" { + if err := fileutils.ValidateRelPath(c.ModuleDir); err != nil { + return fmt.Errorf("invalid 'module-dir':\n%w", err) + } + + if strings.ContainsAny(c.ModuleDir, "%\n\r") { + return errors.New("'module-dir' must not contain RPM macros or newlines") + } + } + + if strings.ContainsAny(c.ConfigFile, "\x00\r\n") { + return errors.New("'config-file' must be a local file path without NUL or newlines") + } + + for _, name := range c.PrepPackages { + if name == "" || strings.HasPrefix(name, "-") || strings.ContainsAny(name, "\r\n") { + return fmt.Errorf("invalid 'prep-packages' entry %#q", name) + } + } + + return nil +} + +// SourceContentIdentity hashes the selected configuration independently of its path. +// No configured file yields no identity; an explicitly empty file still has a hash. +func (c *GoVendorConfig) SourceContentIdentity(fs opctx.FS) (string, error) { + if c == nil || c.ConfigFile == "" { + return "", nil + } + + content, err := c.ReadConfigFile(fs) + if err != nil { + return "", err + } + + digest := sha256.Sum256([]byte(content)) + + return hex.EncodeToString(digest[:]), nil +} + +// ReadConfigFile reads and validates the explicitly selected native configuration. +// An empty file is a valid, intentional replacement; callers must not fall back to upstream settings. +func (c *GoVendorConfig) ReadConfigFile(fs opctx.FS) (string, error) { + data, err := fileutils.ReadFile(fs, c.ConfigFile) + if err != nil { + return "", fmt.Errorf("reading 'go-vendor.config-file' %#q:\n%w", c.ConfigFile, err) + } + + var settings map[string]any + if err := toml.Unmarshal(data, &settings); err != nil { + return "", fmt.Errorf("invalid TOML in 'go-vendor.config-file' %#q:\n%w", c.ConfigFile, err) + } + + if general, ok := settings["general"].(map[string]any); ok { + if _, exists := general["go_mod_dir"]; exists { + return "", fmt.Errorf("use 'module-dir' instead of 'general.go_mod_dir' in %#q", c.ConfigFile) + } + } + + return string(data), nil +} diff --git a/internal/projectconfig/govendor_loader_internal_test.go b/internal/projectconfig/govendor_loader_internal_test.go new file mode 100644 index 00000000..a0af07a3 --- /dev/null +++ b/internal/projectconfig/govendor_loader_internal_test.go @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package projectconfig + +import ( + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/global/testctx" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGoVendorConfigFileDeclarationAndInheritance(t *testing.T) { + ctx := testctx.NewCtx() + + files := map[string]string{ + "/project/azldev.toml": `includes = ["defaults/group.toml", "components/app.comp.toml"]`, + "/project/defaults/group.toml": ` +[component-groups.base] +components = ["app"] +[component-groups.base.default-component-config.go-vendor] +enabled = true +config-file = "go-vendor-tools.toml" +`, + "/project/components/app.comp.toml": `[components.app]`, + } + for path, content := range files { + require.NoError(t, fileutils.WriteFile(ctx.FS(), path, []byte(content), fileperms.PublicFile)) + } + + for _, withoutLockfile := range []bool{false, true} { + config, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{withoutLockfile: withoutLockfile}, + "/project/azldev.toml") + require.NoError(t, err) + resolved, err := ResolveComponentConfig(config.Components["app"], config.DefaultComponentConfig, + ComponentConfig{}, config.ComponentGroups, []string{"base"}) + require.NoError(t, err) + assert.Equal(t, "/project/defaults/go-vendor-tools.toml", resolved.GoVendor.ConfigFile) + assert.True(t, resolved.GoVendor.IsEnabled()) + } +} + +func TestGoVendorInlineConfigurationIsRejected(t *testing.T) { + for _, contents := range []string{ + "[components.app.go-vendor]\nenabled = true\ntool-config = '[archive]'", + "[components.app.go-vendor.tool-config.archive]\nuse_module_proxy = true", + } { + ctx := testctx.NewCtx() + require.NoError(t, fileutils.WriteFile(ctx.FS(), "/project/azldev.toml", []byte(contents), fileperms.PublicFile)) + _, err := loadAndResolveProjectConfig(ctx.FS(), loadOptions{}, "/project/azldev.toml") + require.Error(t, err) + } +} diff --git a/internal/projectconfig/govendor_test.go b/internal/projectconfig/govendor_test.go new file mode 100644 index 00000000..02feb694 --- /dev/null +++ b/internal/projectconfig/govendor_test.go @@ -0,0 +1,129 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package projectconfig_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/global/testctx" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/mitchellh/hashstructure/v2" + "github.com/pelletier/go-toml/v2" + "github.com/samber/lo" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGoVendorConfig(t *testing.T) { + var config projectconfig.ComponentConfig + require.NoError(t, toml.Unmarshal([]byte(` +[go-vendor] +enabled = true +module-dir = "cmd/app" +prep-packages = ["git"] +config-file = "go-vendor-tools.toml" +`), &config)) + require.True(t, config.GoVendor.IsEnabled()) + require.NoError(t, config.GoVendor.Validate()) + resolved := config.WithAbsolutePaths("/project") + assert.Equal(t, filepath.Join("/project", "go-vendor-tools.toml"), resolved.GoVendor.ConfigFile) + assert.Equal(t, "go-vendor-tools.toml", config.GoVendor.ConfigFile) + assert.Equal(t, resolved.GoVendor, resolved.WithAbsolutePaths("/elsewhere").GoVendor) + require.NoError(t, config.MergeUpdatesFrom(&projectconfig.ComponentConfig{ + GoVendor: &projectconfig.GoVendorConfig{Enabled: lo.ToPtr(false)}, + })) + assert.False(t, config.GoVendor.IsEnabled(), "explicit false must override an inherited true") +} + +func TestGoVendorConfigValidation(t *testing.T) { + for _, config := range []*projectconfig.GoVendorConfig{ + {ModuleDir: "../outside"}, + {ModuleDir: "/absolute"}, + {ModuleDir: "%{macro}"}, + {ConfigFile: "bad\x00path"}, + {ConfigFile: "bad\npath"}, + {PrepPackages: []string{"--bad"}}, + {PrepPackages: []string{""}}, + } { + require.Error(t, config.Validate()) + } + + var absent *projectconfig.GoVendorConfig + assert.False(t, absent.IsEnabled()) + require.NoError(t, absent.Validate()) +} + +func TestGoVendorFingerprint(t *testing.T) { + empty := projectconfig.ComponentConfig{} + included, err := empty.HashInclude("GoVendor", nil) + require.NoError(t, err) + assert.False(t, included, "absent vendoring must not invalidate existing component fingerprints") + included, err = empty.HashInclude("Build", nil) + require.NoError(t, err) + assert.True(t, included) + + base := projectconfig.ComponentConfig{GoVendor: &projectconfig.GoVendorConfig{Enabled: lo.ToPtr(true)}} + original, err := hashstructure.Hash(base, hashstructure.FormatV2, &hashstructure.HashOptions{TagName: "fingerprint"}) + require.NoError(t, err) + + for _, config := range []*projectconfig.GoVendorConfig{ + {Enabled: lo.ToPtr(false)}, + {Enabled: lo.ToPtr(true), ModuleDir: "sub"}, + {Enabled: lo.ToPtr(true), PrepPackages: []string{"git"}}, + } { + changed := base + changed.GoVendor = config + hash, err := hashstructure.Hash(changed, hashstructure.FormatV2, &hashstructure.HashOptions{TagName: "fingerprint"}) + require.NoError(t, err) + assert.NotEqual(t, original, hash) + } +} + +func TestGoVendorConfigFileContentIdentity(t *testing.T) { + ctx := testctx.NewCtx() + + var absent *projectconfig.GoVendorConfig + + identity, err := absent.SourceContentIdentity(ctx.FS()) + require.NoError(t, err) + assert.Empty(t, identity) + + config := &projectconfig.GoVendorConfig{} + identity, err = config.SourceContentIdentity(ctx.FS()) + require.NoError(t, err) + assert.Empty(t, identity) + + config.ConfigFile = "/project/vendor.toml" + _, err = config.SourceContentIdentity(ctx.FS()) + require.ErrorIs(t, err, os.ErrNotExist) + + for _, contents := range []string{"", "# reviewed\n[archive]\nuse_module_proxy = true\n"} { + require.NoError(t, fileutils.WriteFile(ctx.FS(), config.ConfigFile, []byte(contents), fileperms.PublicFile)) + actual, err := config.ReadConfigFile(ctx.FS()) + require.NoError(t, err) + assert.Equal(t, contents, actual) + + identity, err = config.SourceContentIdentity(ctx.FS()) + require.NoError(t, err) + expected, err := fileutils.ComputeFileHash(ctx.FS(), fileutils.HashTypeSHA256, config.ConfigFile) + require.NoError(t, err) + assert.Equal(t, expected, identity) + } +} + +func TestGoVendorConfigFileValidation(t *testing.T) { + for _, contents := range []string{"[", "[general]\ngo_mod_dir = 'subdir'"} { + ctx := testctx.NewCtx() + require.NoError(t, fileutils.WriteFile(ctx.FS(), "/project/vendor.toml", []byte(contents), fileperms.PublicFile)) + + config := projectconfig.GoVendorConfig{ConfigFile: "/project/vendor.toml"} + _, err := config.ReadConfigFile(ctx.FS()) + require.Error(t, err) + assert.Contains(t, err.Error(), config.ConfigFile) + } +} diff --git a/internal/rpm/spec/govendor.go b/internal/rpm/spec/govendor.go new file mode 100644 index 00000000..2422df37 --- /dev/null +++ b/internal/rpm/spec/govendor.go @@ -0,0 +1,379 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package spec + +import ( + "errors" + "fmt" + "regexp" + "slices" + "strconv" + "strings" + + "github.com/kballard/go-shellquote" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" +) + +// GoVendorOptions names generated artifacts. All paths except ModuleDir are basenames. +type GoVendorOptions struct { + ArchiveName string + ConfigName string + ModuleDir string +} + +// GoVendorSpec is the common plan for converting or refreshing Go vendoring. +type GoVendorSpec struct { + Spec []byte + PrepSpec []byte + ArchiveFile string // Filename used inside generation staging. + ArchiveSuffix string // Required suffix of the RPM-expanded destination name, if any. + ConfigName string + ArchiveSource string // Existing Source tag to pin to the generated filename. + ExpectedLicense string // Empty: detect and update. Otherwise: verify and preserve. + AllowReplacement bool // Permit replacing the existing vendor payload and configuration. +} + +const goVendorEnvironment = `export GO111MODULE=on GOPROXY=off GOSUMDB=off GOTOOLCHAIN=local +export GOFLAGS="${GOFLAGS:+$GOFLAGS }-mod=vendor"` + +var goBuildRequirement = regexp.MustCompile(`^golang\([^)]+\)(\s*(>=|<=|=|>|<)\s*[^\s,]+)?$`) + +var ( + goVendorUnsupportedMacro = regexp.MustCompile( + `%(?:\{[!?]*)?(?:gopkg\w*|goinstall|gofiles?\w*|go_vendor_\w*|include)\b`) + goVendorRequiresMacro = regexp.MustCompile(`%(?:\{[!?]*)?go_generate_buildrequires\b`) + goVendorCheckMacro = regexp.MustCompile(`%(?:\{[!?]*)?gocheck\b`) +) + +// ConvertGoVendor changes only recognized Go application packaging constructs. +// It always uses the lossless tree, independently of the selected overlay editor. +// Unsupported Go macros and ambiguous section ownership fail before returning output. +func ConvertGoVendor(data []byte, options GoVendorOptions) (*GoVendorSpec, error) { + if err := options.validate(); err != nil { + return nil, err + } + + root, err := parseTree(strings.Split(strings.TrimSuffix(string(data), "\n"), "\n")) + if err != nil { + return nil, err + } + + tree := &specTree{root: root} + + configSource, err := existingVendorConfigSource(tree) + if err != nil { + return nil, err + } + + if configSource != "" { + return prepareExistingGoVendor(tree, configSource) + } + + if err := validateGoVendorTree(tree); err != nil { + return nil, err + } + + if err := convertGoVendorLines(tree, options.ModuleDir); err != nil { + return nil, err + } + + // Do not replace upstream prep or build scripts: capture the directory after + // their existing prep commands, including patches and module-file edits. + prep := tree.Section("%prep", "") + prep.PrependLines([]string{"export GOTOOLCHAIN=local"}) + prep.AppendLines([]string{"", `pwd -P > "%{_topdir}/azldev-go-module-root"`}) + + prepSpec := goVendorPrepSpec(tree, options.ArchiveName) + + sourceNumber, err := freeGoVendorSources(tree, options) + if err != nil { + return nil, err + } + + configRef := fmt.Sprintf("%%{SOURCE%d}", sourceNumber+1) + + extract := fmt.Sprintf("tar -xf %%{SOURCE%d}", sourceNumber) + if options.ModuleDir != "" { + extract += " -C " + shellquote.Join(options.ModuleDir) + } + + prep.AppendLines([]string{"", extract}) + tree.Section("", "").AppendLines([]string{ + fmt.Sprintf("Source%d: %s", sourceNumber, options.ArchiveName), + fmt.Sprintf("Source%d: %s", sourceNumber+1, options.ConfigName), + "BuildRequires: go-vendor-tools", + "BuildRequires: go-rpm-macros", + "BuildRequires: golang", + "%global gomodulesmode GO111MODULE=on", + "%global go_vendor_license_filelist() %{_topdir}/azldev-go-vendor-licenses.list", + "", + }) + + addGoVendorBuildSections(tree, options, configRef) + + return &GoVendorSpec{ + Spec: serializeGoVendorTree(tree), PrepSpec: prepSpec, + ArchiveFile: options.ArchiveName, ConfigName: options.ConfigName, + }, nil +} + +func addGoVendorBuildSections(tree *specTree, options GoVendorOptions, configRef string) { + requires := tree.Section("%generate_buildrequires", "") + if requires == nil { + blk := &block{Kind: sectionBlock, Name: "%generate_buildrequires", Header: "%generate_buildrequires"} + buildIndex := slices.Index(tree.root.Children, tree.Section("%build", "").block) + tree.root.Children = slices.Insert(tree.root.Children, buildIndex, blk) + requires = §ionHandle{block: blk, tree: tree} + } + + requires.PrependLines([]string{"%go_vendor_license_buildrequires -c " + configRef}) + + addGoVendorEnvironment(tree) + + tree.Section("%install", "").AppendLines([]string{ + "", goVendorInModule(options.ModuleDir, "%go_vendor_license_install -c "+configRef), + }) + // Licensing is a build invariant, even when the component disables %check. + tree.Section("%build", "").PrependLines([]string{ + goVendorInModule(options.ModuleDir, "%go_vendor_license_check -c "+configRef), + }) + files := tree.Section("%files", "") + files.block.Header += " -f %{go_vendor_license_filelist}" +} + +func (o GoVendorOptions) validate() error { + for _, name := range []string{o.ArchiveName, o.ConfigName} { + if err := fileutils.ValidateFilename(name); err != nil { + return fmt.Errorf("invalid Go vendor artifact name:\n%w", err) + } + + if strings.ContainsAny(name, "%\r\n") { + return errors.New("go vendor artifact name must not contain macros or newlines") + } + } + + if o.ModuleDir != "" { + if err := fileutils.ValidateRelPath(o.ModuleDir); err != nil { + return fmt.Errorf("invalid Go module directory:\n%w", err) + } + + if strings.ContainsAny(o.ModuleDir, "%\r\n") { + return errors.New("go module directory must not contain macros or newlines") + } + } + + return nil +} + +func serializeGoVendorTree(tree *specTree) []byte { + return []byte(strings.Join(serializeTree(tree.root), "\n") + "\n") +} + +func addGoVendorEnvironment(tree *specTree) { + for _, name := range []string{"%build", "%install", "%check", "%generate_buildrequires"} { + if section := tree.Section(name, ""); section != nil { + section.PrependLines([]string{goVendorEnvironment}) + } + } +} + +func goVendorPrepSpec(tree *specTree, archiveValue string) []byte { + prep := tree.Section("%prep", "") + prep.AppendLines([]string{ + "printf '%%s\\n' %{shescape:" + archiveValue + `} > "%{_topdir}/azldev-go-archive-name"`, + }) + + result := serializeGoVendorTree(tree) + prep.block.Children = prep.block.Children[:len(prep.block.Children)-1] + + return result +} + +// Finalize pins an existing archive reference before release/history processing, +// and updates the license only for newly converted specs. It does not mutate the plan. +func (p *GoVendorSpec) Finalize(archiveName, expression string) ([]byte, error) { + editor, err := openStructuralSpec(strings.NewReader(string(p.Spec))) + if err != nil { + return nil, err + } + + if p.ArchiveSource != "" { + if err := editor.UpdateExistingTag("", p.ArchiveSource, archiveName); err != nil { + return nil, err + } + } + + if p.ExpectedLicense == "" { + if err := editor.UpdateExistingTag("", "License", expression); err != nil { + return nil, err + } + } + + return []byte(strings.Join(editor.rawLines, "\n") + "\n"), nil +} + +func goVendorInModule(moduleDir, command string) string { + lines := []string{`( cd "$(cat "%{_topdir}/azldev-go-module-root")"`} + if moduleDir != "" { + lines = append(lines, "cd "+shellquote.Join(moduleDir)) + } + + lines = append(lines, command, ")") + + return strings.Join(lines, "\n") +} + +func validateGoVendorTree(tree *specTree) error { + if tree.HasSection("%sourcelist") { + return errors.New("go vendoring requires explicit Source tags instead of '%sourcelist'") + } + + for _, name := range []string{"", "%prep", "%build", "%install", "%files"} { + if err := validateGoVendorSection(tree, name, true); err != nil { + return err + } + } + + for _, name := range []string{"%check", "%generate_buildrequires"} { + if err := validateGoVendorSection(tree, name, false); err != nil { + return err + } + } + + var invalid error + + walkBlocks(tree.root, func(blk *block) bool { + if invalid != nil { + return false + } + + invalid = validateGoVendorBlock(blk) + + return true + }) + + return invalid +} + +func validateGoVendorSection(tree *specTree, name string, required bool) error { + sections := tree.Sections(name, "") + if len(sections) == 0 && !required { + return nil + } + + if len(sections) != 1 { + return fmt.Errorf("go vendoring requires one main-package %#q section; use overlays to disambiguate", name) + } + + if !slices.Contains(tree.root.Children, sections[0].block) { + return fmt.Errorf("go vendoring does not support conditional %#q sections; use an overlay", name) + } + + return nil +} + +func validateGoVendorBlock(blk *block) error { + if blk.Kind == macroDefBlock && (blk.Name == "gomodulesmode" || strings.HasPrefix(blk.Name, "go_vendor_")) { + return fmt.Errorf("explicit %#q conflicts with Go vendoring; remove it with an overlay", blk.Name) + } + + for _, line := range append([]string{blk.Header}, blk.Lines...) { + if strings.HasPrefix(strings.TrimSpace(line), "#") { + continue + } + + if macro := goVendorUnsupportedMacro.FindString(line); macro != "" { + return fmt.Errorf("unsupported Go vendoring macro %#q in %#q; adapt it with an overlay", macro, line) + } + + if blk.Kind == macroDefBlock && + (goVendorRequiresMacro.MatchString(line) || goVendorCheckMacro.MatchString(line) || + strings.Contains(line, "golang(")) { + return fmt.Errorf("go dependency or check logic inside a macro definition requires an explicit overlay: %#q", line) + } + + for _, macro := range []string{"-mod=mod", "GO111MODULE=off"} { + if strings.Contains(line, macro) { + return fmt.Errorf("unsupported Go vendoring construct %#q in %#q; adapt it with an overlay", macro, line) + } + } + } + + return nil +} + +func convertGoVendorLines(tree *specTree, moduleDir string) error { + return tree.VisitAllLines(func(section, _ string, line *lineHandle) error { + text := strings.TrimSpace(line.Text) + if strings.HasPrefix(text, "#") { + return nil + } + + if tag, value, ok := parseTagLine(text); ok && isTagBearingSection(section) && + strings.EqualFold(tag, "BuildRequires") && strings.Contains(value, "golang(") { + if !goBuildRequirement.MatchString(value) { + return fmt.Errorf("ambiguous Go BuildRequires %#q; split it into individual tags with an overlay", line.Text) + } + + line.Remove() + } + + switch { + case goVendorRequiresMacro.MatchString(text): + if text != "%go_generate_buildrequires" || section != "%generate_buildrequires" { + return fmt.Errorf("unsupported Go dependency generator %#q; adapt it with an overlay", text) + } + + line.Remove() + case goVendorCheckMacro.MatchString(text): + if text != "%gocheck" || section != "%check" { + return fmt.Errorf("unsupported Go check command %#q; use a module-aware check overlay", text) + } + + line.Replace(goVendorInModule(moduleDir, "%gotest ./...")) + } + + return nil + }) +} + +func freeGoVendorSources(tree *specTree, options GoVendorOptions) (int, error) { + used := make(map[int]bool) + + err := tree.VisitAllLines(func(section, _ string, line *lineHandle) error { + tag, value, ok := parseTagLine(line.Text) + if !ok || !isTagBearingSection(section) || !strings.HasPrefix(strings.ToLower(tag), "source") { + return nil + } + + suffix := tag[len("source"):] + number := 0 + + if suffix != "" { + var err error + + number, err = strconv.Atoi(suffix) + if err != nil || number < 0 { + return fmt.Errorf("cannot allocate vendor Source tags alongside %#q", tag) + } + } + + used[number] = true + + if value == options.ArchiveName || value == options.ConfigName { + return fmt.Errorf("generated Go vendor source collides with existing Source tag %#q", line.Text) + } + + return nil + }) + if err != nil { + return 0, err + } + + for number := 0; ; number++ { + if !used[number] && !used[number+1] { + return number, nil + } + } +} diff --git a/internal/rpm/spec/govendor_existing.go b/internal/rpm/spec/govendor_existing.go new file mode 100644 index 00000000..3013eacb --- /dev/null +++ b/internal/rpm/spec/govendor_existing.go @@ -0,0 +1,232 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package spec + +import ( + "errors" + "fmt" + "path/filepath" + "regexp" + "slices" + "strings" + + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" +) + +var ( + existingVendorLicense = regexp.MustCompile( + `^%go_vendor_license_(?:buildrequires|install|check)\s+-c\s+%\{(?:S:|SOURCE)([0-9]+)\}\s*$`) + existingVendorExtract = regexp.MustCompile(`^tar\s+-xf\s+%\{(?:S:|SOURCE)([0-9]+)\}\s*$`) + existingVendorSetup = regexp.MustCompile(`%(?:\{[!?]*)?(?:(?:auto)?setup|goprep)\b.*\s-[a-zA-Z]*[ab]`) + existingVendorChdir = regexp.MustCompile(`^(?:cd|pushd|popd|%(?:auto)?setup|%goprep)\b`) +) + +func prepareExistingGoVendor(tree *specTree, configSource string) (*GoVendorSpec, error) { + if err := validateGoVendorSection(tree, "%prep", true); err != nil { + return nil, err + } + + configName, err := existingVendorTagValue(tree, "Source"+configSource) + if err != nil { + return nil, err + } + + if err := fileutils.ValidateFilename(configName); err != nil || strings.Contains(configName, "%") { + return nil, fmt.Errorf("existing Go vendor config Source%s must name a plain file, got %#q", configSource, configName) + } + + extraction, sourceNumber, err := takeExistingVendorExtraction(tree) + if err != nil { + return nil, err + } + + if sourceNumber == configSource { + return nil, errors.New("go vendor archive and tool configuration cannot use the same Source tag") + } + + archiveValue, err := existingVendorTagValue(tree, "Source"+sourceNumber) + if err != nil { + return nil, err + } + + suffix, err := existingVendorArchiveSuffix(archiveValue) + if err != nil { + return nil, err + } + + license, err := existingVendorLicenseExpression(tree) + if err != nil { + return nil, err + } + + // Keep upstream's Source numbers, macro settings, file lists, and custom commands. + for _, sectionName := range []string{"%build", "%install", "%check", "%generate_buildrequires"} { + if err := validateGoVendorSection(tree, sectionName, false); err != nil { + return nil, err + } + } + + // Regeneration already includes prep's module edits. Extract the resulting + // module files after those edits so the RPM build cannot apply them twice. + tree.Section("%prep", "").AppendLines([]string{extraction}) + addGoVendorEnvironment(tree) + finalSpec := serializeGoVendorTree(tree) + + if err := removeExistingVendorInput(tree, sourceNumber, extraction); err != nil { + return nil, err + } + + tree.Section("%prep", "").AppendLines([]string{ + `pwd -P > "%{_topdir}/azldev-go-module-root"`, + }) + + return &GoVendorSpec{ + Spec: finalSpec, PrepSpec: goVendorPrepSpec(tree, archiveValue), + ArchiveFile: "azldev-generated-vendor" + suffix, ArchiveSuffix: suffix, ConfigName: configName, + ArchiveSource: "Source" + sourceNumber, ExpectedLicense: license, AllowReplacement: true, + }, nil +} + +func existingVendorLicenseExpression(tree *specTree) (string, error) { + license, err := existingVendorTagValue(tree, "License") + if err != nil { + return "", err + } + + if strings.ContainsAny(license, "%\r\n") || license == "" { + return "", errors.New("regenerating an existing Go vendor archive requires an explicit main-package 'License' tag") + } + + return license, nil +} + +// An empty result means the spec has no existing vendor license commands. +func existingVendorConfigSource(tree *specTree) (string, error) { + source := "" + + err := tree.VisitAllLines(func(_, _ string, line *lineHandle) error { + text := strings.TrimSpace(line.Text) + if !strings.HasPrefix(text, "%go_vendor_license_") { + return nil + } + + matches := existingVendorLicense.FindStringSubmatch(text) + if len(matches) == 0 { + return fmt.Errorf("unsupported existing Go vendor license command %#q", text) + } + + if source != "" && source != matches[1] { + return errors.New("existing Go vendor license commands reference different configuration sources") + } + + source = matches[1] + + return nil + }) + if err != nil { + return "", err + } + + return source, nil +} + +func takeExistingVendorExtraction(tree *specTree) (line, source string, err error) { + count := 0 + + err = tree.Section("%prep", "").VisitLines(func(candidate *lineHandle) error { + text := strings.TrimSpace(candidate.Text) + if existingVendorSetup.MatchString(text) { + return errors.New("secondary-source extraction in '%setup' or '%autosetup' makes Go vendoring ambiguous") + } + + if count > 0 && existingVendorChdir.MatchString(text) { + return errors.New("directory changes after Go vendor extraction require an explicit prep overlay") + } + + matches := existingVendorExtract.FindStringSubmatch(text) + if strings.HasPrefix(text, "tar ") && strings.Contains(text, "%{S") && len(matches) == 0 { + return fmt.Errorf("unsupported secondary-source extraction %#q", text) + } + + if len(matches) != 0 { + if !slices.Contains(tree.Section("%prep", "").block.Children, candidate.block) { + return errors.New("conditional Go vendor extraction requires an explicit preparation strategy") + } + + count++ + line = text + source = matches[1] + + candidate.Remove() + } + + return nil + }) + if err != nil { + return "", "", err + } + + if count != 1 { + return "", "", errors.New("existing Go vendoring requires one unambiguous 'tar -xf %{S:N}' prep command") + } + + return line, source, nil +} + +func existingVendorTagValue(tree *specTree, name string) (string, error) { + var values []string + + err := tree.VisitAllLines(func(section, pkg string, line *lineHandle) error { + tag, value, ok := parseTagLine(line.Text) + if ok && section == "" && pkg == "" && strings.EqualFold(tag, name) { + if !slices.Contains(tree.Section("", "").block.Children, line.block) { + return fmt.Errorf("existing Go vendoring requires an unconditional %#q tag", name) + } + + values = append(values, value) + } + + return nil + }) + if err != nil { + return "", err + } + + if len(values) != 1 { + return "", fmt.Errorf("expected exactly one %#q declaration for existing Go vendoring", name) + } + + return values[0], nil +} + +func existingVendorArchiveSuffix(value string) (string, error) { + if strings.Contains(value, "://") { + return "", errors.New("existing Go vendor archive must use a local source filename") + } + + extension := filepath.Ext(value) + switch extension { + case ".bz2", ".gz", ".xz", ".zst", ".zstd": + return ".tar" + extension, nil + case ".tar": + return ".tar", nil + default: + return "", fmt.Errorf("unsupported existing Go vendor archive format %#q", value) + } +} + +func removeExistingVendorInput(tree *specTree, sourceNumber, extraction string) error { + return tree.VisitAllLines(func(section, pkg string, line *lineHandle) error { + tag, _, ok := parseTagLine(line.Text) + if ok && section == "" && pkg == "" && strings.EqualFold(tag, "Source"+sourceNumber) { + line.Remove() + } + + if section == "%prep" && strings.TrimSpace(line.Text) == extraction { + line.Remove() + } + + return nil + }) +} diff --git a/internal/rpm/spec/govendor_existing_test.go b/internal/rpm/spec/govendor_existing_test.go new file mode 100644 index 00000000..280c9b4c --- /dev/null +++ b/internal/rpm/spec/govendor_existing_test.go @@ -0,0 +1,129 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package spec_test + +import ( + "strings" + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/rpm/spec" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func existingVendorSpec() string { + return strings.Join([]string{ + "Name: app", "Version: 1.2", "Release: %autorelease", "Summary: Application", + "License: MIT AND Apache-2.0", + "Source0: app-%{version}.tar.gz", + "Source5: app-%{version}-vendor.tar.bz2", + "Source8: go-vendor-tools.toml", + "Source9: app.conf", + "BuildRequires: go-vendor-tools", "Requires: runtime-helper", + "%description", "Application", "%prep", "%goprep -p1", + "tar -xf %{S:5}", "sed -i 's/old/new/' go.mod", "cp %{S:9} .", + "%generate_buildrequires", "%go_vendor_license_buildrequires -c %{S:8}", + "%build", "%global gomodulesmode GO111MODULE=on", "%gobuild -o app .", + "%install", "%go_vendor_license_install -c %{S:8}", + "install -D app %{buildroot}%{_bindir}/app", + "%check", "%go_vendor_license_check -c %{S:8}", "%gocheck2 -t integration", + "%files -f %{go_vendor_license_filelist}", "%{_bindir}/app", "%changelog", "%autochangelog", "", + }, "\n") +} + +func TestExistingGoVendoringPreservesPackaging(t *testing.T) { + for _, version := range []string{"1.2", "2.4"} { + input := strings.ReplaceAll(existingVendorSpec(), "Version: 1.2", "Version: "+version) + result, err := spec.ConvertGoVendor([]byte(input), spec.GoVendorOptions{ + ArchiveName: "app-vendor.tar.bz2", ConfigName: "azldev-go-vendor.toml", + }) + require.NoError(t, err) + assert.True(t, result.AllowReplacement) + assert.Equal(t, "go-vendor-tools.toml", result.ConfigName) + assert.Equal(t, ".tar.bz2", result.ArchiveSuffix) + assert.Equal(t, "MIT AND Apache-2.0", result.ExpectedLicense) + + for _, line := range strings.Split(input, "\n") { + assert.Contains(t, string(result.Spec), line) + } + + assert.NotContains(t, string(result.Spec), "azldev-go-vendor.toml") + assert.NotContains(t, string(result.Spec), "azldev-go-module-root") + assert.Contains(t, string(result.Spec), "GOPROXY=off") + assert.NotContains(t, string(result.PrepSpec), "Source5:") + assert.NotContains(t, string(result.PrepSpec), "tar -xf %{S:5}") + assert.Contains(t, string(result.PrepSpec), "sed -i 's/old/new/' go.mod") + assert.Contains(t, string(result.PrepSpec), "%{shescape:app-%{version}-vendor.tar.bz2}") + assert.Contains(t, string(result.PrepSpec), `pwd -P > "%{_topdir}/azldev-go-module-root"`) + } +} + +func TestExistingGoVendoringRejectsAmbiguousInputs(t *testing.T) { + for name, input := range map[string]string{ + "multiple extractions": strings.ReplaceAll(existingVendorSpec(), "%goprep -p1", "%goprep -p1\ntar -xf %{S:9}"), + "autosetup secondary": strings.ReplaceAll(existingVendorSpec(), "%goprep -p1", "%autosetup -a 1"), + "setup secondary": strings.ReplaceAll(existingVendorSpec(), "%goprep -p1", "%setup -qb1"), + "braced autosetup secondary": strings.ReplaceAll(existingVendorSpec(), "%goprep -p1", "%{autosetup -a 1}"), + "other tar flags": strings.ReplaceAll(existingVendorSpec(), "%goprep -p1", "%goprep -p1\ntar -xjf %{S:9}"), + "directory after extraction": strings.ReplaceAll(existingVendorSpec(), "sed -i 's/old/new/' go.mod", "cd submodule"), + "no extraction": strings.ReplaceAll(existingVendorSpec(), "tar -xf %{S:5}", ""), + "mismatched config": strings.ReplaceAll(existingVendorSpec(), + "%go_vendor_license_check -c %{S:8}", "%go_vendor_license_check -c %{S:9}"), + "dynamic config": strings.ReplaceAll(existingVendorSpec(), "go-vendor-tools.toml", "%{name}.toml"), + "unknown compression": strings.ReplaceAll(existingVendorSpec(), "-vendor.tar.bz2", "-vendor.zip"), + "macro license": strings.ReplaceAll(existingVendorSpec(), "License: MIT AND Apache-2.0", "License: %{license}"), + "missing source": strings.ReplaceAll(existingVendorSpec(), "Source5:", "Source6:"), + "conditional extraction": strings.ReplaceAll(existingVendorSpec(), + "tar -xf %{S:5}", "%if 1\ntar -xf %{S:5}\n%endif"), + "conditional source": strings.ReplaceAll(existingVendorSpec(), + "Source5: app-%{version}-vendor.tar.bz2", "%if 1\nSource5: app-%{version}-vendor.tar.bz2\n%endif"), + "conditional license": strings.ReplaceAll(existingVendorSpec(), + "License: MIT AND Apache-2.0", "%if 1\nLicense: MIT AND Apache-2.0\n%endif"), + } { + t.Run(name, func(t *testing.T) { + result, err := spec.ConvertGoVendor([]byte(input), spec.GoVendorOptions{ + ArchiveName: "app-vendor.tar.bz2", ConfigName: "azldev-go-vendor.toml", + }) + require.Error(t, err) + assert.Nil(t, result) + }) + } +} + +func TestExistingGoVendorExtractsAfterModulePatches(t *testing.T) { + input := strings.ReplaceAll(existingVendorSpec(), "%goprep -p1", "%setup -q") + input = strings.ReplaceAll(input, "sed -i 's/old/new/' go.mod", "%autopatch -p1\nsed -i 's/old/new/' go.mod") + result, err := spec.ConvertGoVendor([]byte(input), spec.GoVendorOptions{ + ArchiveName: "app-vendor.tar.bz2", ConfigName: "vendor.toml", + }) + require.NoError(t, err) + + output := string(result.Spec) + assert.Less(t, strings.Index(output, "%autopatch -p1"), strings.Index(output, "tar -xf %{S:5}")) + assert.Less(t, strings.Index(output, "sed -i"), strings.Index(output, "tar -xf %{S:5}")) + assert.Less(t, strings.Index(output, "cp %{S:9} ."), strings.Index(output, "tar -xf %{S:5}")) + assert.Contains(t, string(result.PrepSpec), "%autopatch -p1") + assert.NotContains(t, string(result.PrepSpec), "tar -xf %{S:5}") +} + +func TestExistingGoVendorPinsExpandedArchiveBeforeReleaseChanges(t *testing.T) { + input := strings.ReplaceAll(existingVendorSpec(), "app-%{version}-vendor", "app-%{version}-%{release}-vendor") + result, err := spec.ConvertGoVendor([]byte(input), spec.GoVendorOptions{ + ArchiveName: "app-vendor.tar.bz2", ConfigName: "vendor.toml", + }) + require.NoError(t, err) + final, err := result.Finalize("app-1.2-7-vendor.tar.bz2", "MIT AND Apache-2.0") + require.NoError(t, err) + assert.Contains(t, string(final), "Source5: app-1.2-7-vendor.tar.bz2") + assert.NotContains(t, string(final), "app-%{version}-%{release}-vendor") + assert.Contains(t, string(result.Spec), "Source5: app-%{version}-%{release}-vendor.tar.bz2", + "finalization must not mutate the plan") + editor, err := spec.OpenSpec(strings.NewReader(string(final)), spec.WithEditor(spec.EditorStructural)) + require.NoError(t, err) + require.NoError(t, editor.UpdateExistingTag("", "Release", "8")) + + var output strings.Builder + require.NoError(t, editor.Serialize(&output)) + assert.Contains(t, output.String(), "Source5: app-1.2-7-vendor.tar.bz2") +} diff --git a/internal/rpm/spec/govendor_test.go b/internal/rpm/spec/govendor_test.go new file mode 100644 index 00000000..a29a44a3 --- /dev/null +++ b/internal/rpm/spec/govendor_test.go @@ -0,0 +1,203 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package spec_test + +import ( + "strings" + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/rpm/spec" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func vendorInput() string { + return strings.Join([]string{ + "# Upstream customization", + "%global goipath example.com/app", + "Name: app", + "Version: 1.2", + "Release: %autorelease", + "Summary: App", + "License: MIT", + "Source0: app-%{version}.tar.gz", + "Source1: app.service", + "Source9999: app.macros", + "Patch0: fix.patch", + "BuildRequires: golang(example.com/dep) >= 1.0", + "BuildRequires: gcc", + "Requires: systemd", + "", + "%description", + "App description", + "%package tools", + "Summary: Tools", + "Requires: app = %{version}-%{release}", + "%description tools", + "Tool description", + "%prep", + "%goprep -p1", + "sed -i 's/old/new/' go.mod", + "%generate_buildrequires", + "%go_generate_buildrequires", + "echo custom-build-dependency", + "%build", + "%gobuild -o app %{goipath}", + "%install", + "install -D app %{buildroot}%{_bindir}/app", + "%check", + "%if %{with check}", + "%gocheck", + "./custom-test", + "%endif", + "%files", + "%{_bindir}/app", + "%files tools", + "%{_bindir}/tool", + "%changelog", + "%autochangelog", + "", + }, "\n") +} + +func TestConvertGoVendorPreservesUpstream(t *testing.T) { + for _, version := range []string{"1.2", "2.7"} { + t.Run(version, func(t *testing.T) { + input := strings.ReplaceAll(vendorInput(), "Version: 1.2", "Version: "+version) + result, err := spec.ConvertGoVendor([]byte(input), spec.GoVendorOptions{ + ArchiveName: "app-vendor.tar.bz2", ConfigName: "vendor.toml", + }) + require.NoError(t, err) + + output := string(result.Spec) + for _, line := range []string{ + "# Upstream customization", "Version: " + version, "Release: %autorelease", + "Source0: app-%{version}.tar.gz", "Source1: app.service", "Source9999: app.macros", + "Patch0: fix.patch", "BuildRequires: gcc", "Requires: systemd", + "%package tools", "Requires: app = %{version}-%{release}", + "%goprep -p1", "sed -i 's/old/new/' go.mod", "echo custom-build-dependency", + "%gobuild -o app %{goipath}", "install -D app %{buildroot}%{_bindir}/app", + "%if %{with check}\n" + + `( cd "$(cat "%{_topdir}/azldev-go-module-root")"` + "\n%gotest ./...\n)\n./custom-test\n%endif", + "%files tools\n%{_bindir}/tool", "%autochangelog", + "Source2: app-vendor.tar.bz2", "Source3: vendor.toml", "tar -xf %{SOURCE2}", + "%go_vendor_license_buildrequires -c %{SOURCE3}", + "%go_vendor_license_check -c %{SOURCE3}", + "%go_vendor_license_install -c %{SOURCE3}", + "%files -f %{go_vendor_license_filelist}", + "GOPROXY=off", "-mod=vendor", "%global gomodulesmode GO111MODULE=on", + } { + assert.Contains(t, output, line) + } + + assert.NotContains(t, output, "BuildRequires: golang(") + assert.NotContains(t, output, "%go_generate_buildrequires") + assert.Contains(t, output, `pwd -P > "%{_topdir}/azldev-go-module-root"`) + assert.Contains(t, output, `%global go_vendor_license_filelist() %{_topdir}/azldev-go-vendor-licenses.list`) + assert.Contains(t, string(result.PrepSpec), "Release: %autorelease") + assert.Contains(t, string(result.PrepSpec), `pwd -P > "%{_topdir}/azldev-go-module-root"`) + assert.Contains(t, string(result.PrepSpec), "sed -i 's/old/new/' go.mod") + assert.NotContains(t, string(result.PrepSpec), "Source2: app-vendor.tar.bz2") + assert.Contains(t, string(result.PrepSpec), "%{shescape:app-vendor.tar.bz2}") + assert.False(t, result.AllowReplacement) + assert.Empty(t, result.ExpectedLicense) + assert.Contains(t, string(result.PrepSpec), "%autochangelog") + + _, err = spec.OpenSpec(strings.NewReader(output), spec.WithEditor(spec.EditorStructural)) + require.NoError(t, err) + }) + } +} + +func TestConvertGoVendorRejectsAmbiguity(t *testing.T) { + for name, input := range map[string]string{ + "development subpackages": strings.ReplaceAll(vendorInput(), "%package tools", "%gopkg\n%package tools"), + "mixed requirements": strings.ReplaceAll(vendorInput(), + "golang(example.com/dep) >= 1.0", "gcc, golang(example.com/dep)"), + "check flags": strings.ReplaceAll(vendorInput(), "%gocheck", "%gocheck -z foo"), + "generator shell": strings.ReplaceAll(vendorInput(), + "%go_generate_buildrequires", "%go_generate_buildrequires | cat"), + "conditional prep": strings.ReplaceAll(vendorInput(), + "%prep\n%goprep -p1", "%if 1\n%prep\n%goprep -p1\n%endif"), + "module mode conflict": "%global gomodulesmode GO111MODULE=off\n" + vendorInput(), + "module mode override": "%global gomodulesmode GO111MODULE=on\n" + vendorInput(), + "license check override": "%global go_vendor_license_check_disable 1\n" + vendorInput(), + "existing vendor config": strings.ReplaceAll(vendorInput(), "app.service", "vendor.toml"), + "missing install": strings.ReplaceAll(vendorInput(), "%install", "# install"), + "macro source index": strings.ReplaceAll(vendorInput(), "Source1:", "Source%{number}:"), + "already vendored": strings.ReplaceAll(vendorInput(), "%gocheck", "%go_vendor_license_check"), + "braced generator": strings.ReplaceAll(vendorInput(), + "%go_generate_buildrequires", "%{go_generate_buildrequires}"), + "braced devel macro": strings.ReplaceAll(vendorInput(), "%package tools", "%{gopkg}\n%package tools"), + "hidden generator": "%global dependencies %{go_generate_buildrequires}\n" + vendorInput(), + "go filelist": strings.ReplaceAll(vendorInput(), "%files\n", "%files -f %{gofilelist}\n"), + "unsupported after comment": "# unrelated comment\n%gopkg\n" + vendorInput(), + } { + t.Run(name, func(t *testing.T) { + result, err := spec.ConvertGoVendor([]byte(input), spec.GoVendorOptions{ + ArchiveName: "app-vendor.tar.bz2", ConfigName: "vendor.toml", + }) + require.Error(t, err) + assert.Nil(t, result) + }) + } +} + +func TestConvertGoVendorModuleDirAndStaticRequires(t *testing.T) { + input := strings.ReplaceAll(vendorInput(), + "%generate_buildrequires\n%go_generate_buildrequires\necho custom-build-dependency\n", "") + input = strings.ReplaceAll(input, "%changelog\n%autochangelog\n", "") + result, err := spec.ConvertGoVendor([]byte(input), spec.GoVendorOptions{ + ArchiveName: "app-vendor.tar.bz2", ConfigName: "vendor.toml", ModuleDir: "cmd/app", + }) + require.NoError(t, err) + assert.Contains(t, string(result.Spec), "tar -xf %{SOURCE2} -C cmd/app") + assert.Contains(t, string(result.Spec), "cd cmd/app\n%go_vendor_license_install") + assert.Contains(t, string(result.Spec), + `( cd "$(cat "%{_topdir}/azldev-go-module-root")"`+"\ncd cmd/app\n%gotest ./...\n)") + assert.Contains(t, string(result.Spec), `( cd "$(cat "%{_topdir}/azldev-go-module-root")"`) + assert.Contains(t, string(result.Spec), "%generate_buildrequires\n") +} + +func TestConvertGoVendorRejectsUnsafeOptions(t *testing.T) { + for _, options := range []spec.GoVendorOptions{ + {ArchiveName: "../vendor.tar.bz2", ConfigName: "config"}, + {ArchiveName: "vendor", ConfigName: "%{name}"}, + {ArchiveName: "vendor", ConfigName: "config", ModuleDir: "../escape"}, + {ArchiveName: "vendor", ConfigName: "config", ModuleDir: "%{bad}"}, + } { + _, err := spec.ConvertGoVendor([]byte(vendorInput()), options) + require.Error(t, err) + } +} + +func TestConvertGoVendorPreservesStaticReleaseAndCustomDirectories(t *testing.T) { + input := strings.ReplaceAll(vendorInput(), "Release: %autorelease", "Release: 42%{?dist}") + input = strings.ReplaceAll(input, "%check", "cd assets\n%check") + result, err := spec.ConvertGoVendor([]byte(input), spec.GoVendorOptions{ + ArchiveName: "app-vendor.tar.bz2", ConfigName: "vendor.toml", + }) + require.NoError(t, err) + assert.Contains(t, string(result.PrepSpec), "Release: 42%{?dist}") + assert.Contains(t, string(result.Spec), "cd assets") + assert.Contains(t, string(result.Spec), + `( cd "$(cat "%{_topdir}/azldev-go-module-root")"`+"\n%go_vendor_license_install") + assert.Contains(t, string(result.Spec), + "%global go_vendor_license_filelist() %{_topdir}/azldev-go-vendor-licenses.list") +} + +func TestConvertGoVendorPreservesNonTagsAndComments(t *testing.T) { + script := "cat <<'EOF' > metadata\nBuildRequires: golang(not-a-dependency)\nSource2: not-a-source\nEOF\n" + input := strings.ReplaceAll(vendorInput(), "%gobuild", script+"%gobuild") + input = "# %gopkg is intentionally not invoked\n" + input + input = strings.ReplaceAll(input, "%files\n", "%files\n# app files\n") + result, err := spec.ConvertGoVendor([]byte(input), spec.GoVendorOptions{ + ArchiveName: "app-vendor.tar.bz2", ConfigName: "vendor.toml", + }) + require.NoError(t, err) + assert.Contains(t, string(result.Spec), script) + assert.Contains(t, string(result.Spec), "Source2: app-vendor.tar.bz2") + assert.Contains(t, string(result.Spec), "# %gopkg is intentionally not invoked") + assert.Contains(t, string(result.Spec), "%files -f %{go_vendor_license_filelist}\n# app files") +} diff --git a/internal/utils/fileutils/copy.go b/internal/utils/fileutils/copy.go index 79082297..d0002e7c 100644 --- a/internal/utils/fileutils/copy.go +++ b/internal/utils/fileutils/copy.go @@ -18,11 +18,14 @@ import ( type CopyFileOptions struct { // Whether or not to preserve POSIX mode bits on files. Does *not* apply to directories. PreserveFileMode bool + // NoFollow requires a regular source file and rejects final-component symlinks. + NoFollow bool } // Copies the contents of an existing file to a new file at the given destination path. // May preserve mode bits if "options" says so but does *not* preserve ownership, timestamps, -// or other metadata. *Does* follow symlinks. Is enlightened to skip copying in dry-run mode. +// or other metadata. Follows source symlinks unless [CopyFileOptions.NoFollow] is set. +// Is enlightened to skip copying in dry-run mode. func CopyFile(dryRunnable opctx.DryRunnable, fs opctx.FS, sourcePath, destPath string, options CopyFileOptions) error { return CopyFileCrossFS(dryRunnable, fs, sourcePath, fs, destPath, options) } @@ -30,7 +33,8 @@ func CopyFile(dryRunnable opctx.DryRunnable, fs opctx.FS, sourcePath, destPath s // Copies the contents of an existing file to a new file at the given destination path, // potentially across two [opctx.FS] filesystem instances. May preserve mode bits if // "options" says so but does *not* preserve ownership, timestamps, or other metadata. -// *Does* follow symlinks. Is enlightened to skip copying in dry-run mode. +// Follows source symlinks unless [CopyFileOptions.NoFollow] is set. +// Is enlightened to skip copying in dry-run mode. func CopyFileCrossFS( dryRunnable opctx.DryRunnable, sourceFS opctx.FS, sourcePath string, @@ -67,7 +71,14 @@ func copyFileCrossFS( destFileMode = stat.Mode() } - sourceFile, err := sourceFS.Open(sourcePath) + var sourceFile opctx.File + + if options.NoFollow { + sourceFile, err = OpenRegularFileNoFollow(sourceFS, sourcePath) + } else { + sourceFile, err = sourceFS.Open(sourcePath) + } + if err != nil { err = fmt.Errorf("failed to open copy source '%s':\n%w", sourcePath, err) diff --git a/internal/utils/fileutils/file.go b/internal/utils/fileutils/file.go index 52fbcbd3..50e9d5dd 100644 --- a/internal/utils/fileutils/file.go +++ b/internal/utils/fileutils/file.go @@ -14,6 +14,7 @@ import ( "github.com/bmatcuk/doublestar/v4" "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx" "github.com/spf13/afero" + "golang.org/x/sys/unix" ) // Adaptation of [os.WriteFile] that works with [opctx.FS]. @@ -28,6 +29,31 @@ func ReadFile(fs opctx.FS, filename string) ([]byte, error) { return afero.ReadFile(fs, filename) } +// OpenRegularFileNoFollow atomically rejects final-component symlinks and checks +// the opened descriptor rather than a path that could change before reading. +// The parent directory must be trusted. Nonblocking mode avoids opening a FIFO indefinitely. +func OpenRegularFileNoFollow(fs opctx.FS, filename string) (opctx.File, error) { + file, err := fs.OpenFile(filename, os.O_RDONLY|unix.O_NOFOLLOW|unix.O_NONBLOCK, 0) + if err != nil { + return nil, fmt.Errorf("opening regular file %#q without following symlinks:\n%w", filename, err) + } + + info, err := file.Stat() + if err != nil { + file.Close() + + return nil, fmt.Errorf("checking opened file %#q:\n%w", filename, err) + } + + if !info.Mode().IsRegular() { + file.Close() + + return nil, fmt.Errorf("%#q is not a regular file", filename) + } + + return file, nil +} + // Adaptation of [filepath.Glob] that works with [opctx.FS] and also uses the // [doublestar] package to supports standard double-star globs ('**'). func Glob(fs opctx.FS, pattern string, opts ...doublestar.GlobOption) ([]string, error) { diff --git a/internal/utils/fileutils/nofollow_test.go b/internal/utils/fileutils/nofollow_test.go new file mode 100644 index 00000000..9e87e3af --- /dev/null +++ b/internal/utils/fileutils/nofollow_test.go @@ -0,0 +1,74 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package fileutils_test + +import ( + "io" + "os" + "path/filepath" + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx/opctx_test" + "github.com/microsoft/azure-linux-dev-tools/internal/global/testctx" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + "golang.org/x/sys/unix" +) + +func TestOpenRegularFileNoFollow(t *testing.T) { + ctx := testctx.NewCtx() + require.NoError(t, fileutils.WriteFile(ctx.FS(), "/regular", []byte("contents"), fileperms.PublicFile)) + file, err := fileutils.OpenRegularFileNoFollow(ctx.FS(), "/regular") + require.NoError(t, err) + + defer file.Close() + + data, err := io.ReadAll(file) + require.NoError(t, err) + assert.Equal(t, "contents", string(data)) + + missing, err := fileutils.OpenRegularFileNoFollow(ctx.FS(), "/missing") + require.ErrorIs(t, err, os.ErrNotExist) + assert.Nil(t, missing) + require.NoError(t, fileutils.MkdirAll(ctx.FS(), "/directory")) + directory, err := fileutils.OpenRegularFileNoFollow(ctx.FS(), "/directory") + require.ErrorContains(t, err, "not a regular file") + assert.Nil(t, directory) +} + +func TestOpenRegularFileNoFollowFlagsAndStatErrors(t *testing.T) { + ctrl := gomock.NewController(t) + mockFS := opctx_test.NewMockFS(ctrl) + flags := os.O_RDONLY | unix.O_NOFOLLOW | unix.O_NONBLOCK + mockFS.EXPECT().OpenFile("/denied", flags, os.FileMode(0)).Return(nil, os.ErrPermission) + file, err := fileutils.OpenRegularFileNoFollow(mockFS, "/denied") + require.ErrorIs(t, err, os.ErrPermission) + assert.Nil(t, file) + + opened := opctx_test.NewMockFile(ctrl) + mockFS.EXPECT().OpenFile("/bad-stat", flags, os.FileMode(0)).Return(opened, nil) + opened.EXPECT().Stat().Return(nil, os.ErrPermission) + opened.EXPECT().Close().Return(nil) + + file, err = fileutils.OpenRegularFileNoFollow(mockFS, "/bad-stat") + require.ErrorIs(t, err, os.ErrPermission) + assert.Nil(t, file) +} + +// Kernel-enforced symlink rejection requires a real filesystem. +func TestCopyFileNoFollowRejectsSymlinks(t *testing.T) { + ctx := testctx.NewCtx(testctx.WithHostFS()) + root := t.TempDir() + target, link, output := filepath.Join(root, "target"), filepath.Join(root, "link"), filepath.Join(root, "output") + require.NoError(t, fileutils.WriteFile(ctx.FS(), target, []byte("host-only contents"), fileperms.PublicFile)) + require.NoError(t, os.Symlink(target, link)) + err := fileutils.CopyFile(ctx, ctx.FS(), link, output, fileutils.CopyFileOptions{NoFollow: true}) + require.ErrorContains(t, err, "without following symlinks") + exists, err := fileutils.Exists(ctx.FS(), output) + require.NoError(t, err) + assert.False(t, exists) +} diff --git a/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap b/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap index 9cf47bd3..9259ce68 100755 --- a/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshotsContainer_config_generate-schema_stdout_1.snap @@ -150,6 +150,11 @@ "title": "Source files", "description": "Source files to download for this component" }, + "go-vendor": { + "$ref": "#/$defs/GoVendorConfig", + "title": "Go vendoring", + "description": "Convert a Go application spec and generate its vendor archive during source preparation" + }, "packages": { "additionalProperties": { "$ref": "#/$defs/PackageConfig" @@ -682,6 +687,38 @@ "additionalProperties": false, "type": "object" }, + "GoVendorConfig": { + "properties": { + "enabled": { + "type": "boolean", + "title": "Enabled", + "description": "Enable Go application vendoring" + }, + "module-dir": { + "type": "string", + "title": "Module directory", + "description": "Relative module directory within the prepared source tree; defaults to its root" + }, + "config-file": { + "type": "string", + "title": "Vendor configuration file", + "description": "Path to a go-vendor-tools TOML file relative to the declaring component configuration. Replaces upstream settings when specified", + "examples": [ + "go-vendor-tools.toml" + ] + }, + "prep-packages": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Prep packages", + "description": "Additional RPM packages needed to execute the upstream prep section in mock" + } + }, + "additionalProperties": false, + "type": "object" + }, "ImageCapabilities": { "properties": { "machine-bootable": { diff --git a/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap b/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap index 9cf47bd3..9259ce68 100755 --- a/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap +++ b/scenario/__snapshots__/TestSnapshots_config_generate-schema_stdout_1.snap @@ -150,6 +150,11 @@ "title": "Source files", "description": "Source files to download for this component" }, + "go-vendor": { + "$ref": "#/$defs/GoVendorConfig", + "title": "Go vendoring", + "description": "Convert a Go application spec and generate its vendor archive during source preparation" + }, "packages": { "additionalProperties": { "$ref": "#/$defs/PackageConfig" @@ -682,6 +687,38 @@ "additionalProperties": false, "type": "object" }, + "GoVendorConfig": { + "properties": { + "enabled": { + "type": "boolean", + "title": "Enabled", + "description": "Enable Go application vendoring" + }, + "module-dir": { + "type": "string", + "title": "Module directory", + "description": "Relative module directory within the prepared source tree; defaults to its root" + }, + "config-file": { + "type": "string", + "title": "Vendor configuration file", + "description": "Path to a go-vendor-tools TOML file relative to the declaring component configuration. Replaces upstream settings when specified", + "examples": [ + "go-vendor-tools.toml" + ] + }, + "prep-packages": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Prep packages", + "description": "Additional RPM packages needed to execute the upstream prep section in mock" + } + }, + "additionalProperties": false, + "type": "object" + }, "ImageCapabilities": { "properties": { "machine-bootable": { diff --git a/schemas/azldev.schema.json b/schemas/azldev.schema.json index 9cf47bd3..9259ce68 100644 --- a/schemas/azldev.schema.json +++ b/schemas/azldev.schema.json @@ -150,6 +150,11 @@ "title": "Source files", "description": "Source files to download for this component" }, + "go-vendor": { + "$ref": "#/$defs/GoVendorConfig", + "title": "Go vendoring", + "description": "Convert a Go application spec and generate its vendor archive during source preparation" + }, "packages": { "additionalProperties": { "$ref": "#/$defs/PackageConfig" @@ -682,6 +687,38 @@ "additionalProperties": false, "type": "object" }, + "GoVendorConfig": { + "properties": { + "enabled": { + "type": "boolean", + "title": "Enabled", + "description": "Enable Go application vendoring" + }, + "module-dir": { + "type": "string", + "title": "Module directory", + "description": "Relative module directory within the prepared source tree; defaults to its root" + }, + "config-file": { + "type": "string", + "title": "Vendor configuration file", + "description": "Path to a go-vendor-tools TOML file relative to the declaring component configuration. Replaces upstream settings when specified", + "examples": [ + "go-vendor-tools.toml" + ] + }, + "prep-packages": { + "items": { + "type": "string" + }, + "type": "array", + "title": "Prep packages", + "description": "Additional RPM packages needed to execute the upstream prep section in mock" + } + }, + "additionalProperties": false, + "type": "object" + }, "ImageCapabilities": { "properties": { "machine-bootable": {