diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 049f0ef54..a6c3f14e7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -74,8 +74,8 @@ jobs: fail-fast: false matrix: host: [ubuntu-latest] - # Add 'tmt' once the quarantined readiness regression is fixed. - target: [component-idempotency, image, full-render] + # The tmt target runs local coverage; virtual provisioning remains quarantined. + target: [component-idempotency, image, full-render, tmt] runs-on: ${{ matrix.host }} permissions: contents: read diff --git a/docs/developer/how-to/testing.md b/docs/developer/how-to/testing.md index 987d18c29..671043914 100644 --- a/docs/developer/how-to/testing.md +++ b/docs/developer/how-to/testing.md @@ -46,16 +46,16 @@ general end-to-end suite with: mage e2e ``` -The GitHub Actions workflow runs three E2E shards in parallel: +The GitHub Actions workflow runs four E2E shards in parallel: - `mage e2e -target=component-idempotency` runs update idempotency and subset render idempotency. - `mage e2e -target=image` runs Azure Linux image construction. - `mage e2e -target=full-render` runs the full Azure Linux render idempotency check, which dominates the suite runtime. +- `mage e2e -target=tmt` runs local TMT reporting; virtual provisioning stays + quarantined unless explicitly enabled. -`mage e2e -target=tmt` runs the temporarily quarantined virtual TMT -regression; add it to the CI matrix when the readiness fix lands. ### Azure Linux image build @@ -70,20 +70,26 @@ container. Only the resolved source revision and checked qcow2 are written to a host artifact mount. The privileged container is restricted to this image-construction test; ordinary scenario and e2e containers remain unchanged. -### Virtual TMT provisioning regression +### TMT tests -`mage e2e -target=tmt` builds the current Azure Linux image, then runs the real -`azldev component test` flow with the pinned TMT/testcloud runtime and a tiny -test-only probe RPM. The non-privileged TMT runner maps `/dev/kvm`, uses private -session-libvirt state, and retains diagnostics under `out/tmt-virtual-e2e/`. -Production and test dependency pins share -`internal/tmt/requirements.txt`, which Dependabot tracks. +```bash +mage e2e -target=tmt +``` + +Local reporting runs in a disposable, unprivileged AZL4 container without KVM. +Do not use the local provisioner directly on your development host: it installs +packages and executes tests there. Diagnostics remain under `out/tmt-local-e2e/`. + +Virtual provisioning is quarantined pending the testcloud readiness fix. Set +`TESTING_ENABLE_TMT_VIRTUAL_E2E=1` with the same command to reproduce it. Once +enabled, it skips if `/dev/kvm` is missing and checks access before building an +image; access errors fail the test. Image construction uses a **privileged** +container. Virtual diagnostics remain under `out/tmt-virtual-e2e/`. -This regression is temporarily quarantined while the known testcloud readiness -failure is fixed. The normal `mage e2e` suite compiles the test but skips it; -`mage e2e -target=tmt` explicitly enables and executes the known-red path. Remove the -temporary environment guard once the provisioning fixes land so the standard -E2E job enforces the regression. +Remove the temporary environment guard when the virtual test is fixed. Both +tests share the production pins in +`internal/tmt/requirements.txt`, which Dependabot tracks. Broad `mage e2e` also runs +the privileged image-builder test independently of the virtual quarantine. ## Mutation Testing diff --git a/docs/user/explanation/tmt-tests.md b/docs/user/explanation/tmt-tests.md index e16a35562..29aa6ffdc 100644 --- a/docs/user/explanation/tmt-tests.md +++ b/docs/user/explanation/tmt-tests.md @@ -143,6 +143,59 @@ The same technique works locally: `createrepo_c` the built RPMs, then (dnf accepts a local directory as the `--repofrompath` baseurl) resolves conflicts for you instead of a hand-rolled `--rpm` filter. +## Structured reports and diagnostic files + +Use JSON output to locate retained reports and logs: + +```bash +azldev component test example --provision virtual \ + --image-path ./image.qcow2 --rpm ./example.rpm --plan /plans/smoke \ + --work-dir ./test-artifacts -O json >artifacts.json +status=$? +``` + +The output lists attempted catalog entries; execution stops at the first failure. +JSON and CSV keep diagnostics on stderr. Example (paths are illustrative): + +```json +[ + { + "test": "smoke", + "exitCode": 0, + "artifactDir": "/work/test-artifacts/azldev-test-123", + "junit": ["001-smoke/native/plans/smoke/report/azldev-junit/junit.xml"] + } +] +``` + +`artifactDir` is a fresh directory containing `setup.log`, each invocation's +`runner.log`, and its native TMT run directory. JUnit paths are relative to it; +copy the whole directory to retain diagnostics outside a container. Test-created +payloads may retain restrictive permissions; report discovery does not change them. +Discovery is best-effort: unreadable or invalid artifact paths produce warnings +and are omitted, without changing the command's execution result. +It matches the `report/azldev-junit/junit.xml` path suffix, not report provenance: +test data or copied sources containing the same layout can also appear in `junit`. +Consumers publishing these files should account for that limitation; discovery +does not override TMT's exit status. + +TMT's native JUnit reporter is added without replacing existing reporters. +Its default mapping is lossy (`info` becomes skipped; `warn` and `pending` +become errors); native results and detailed logs remain in the artifact directory. +Early failures or interruption can leave no JUnit. Available artifacts are +returned alongside the command error, not reconstructed after execution. + +azldev exits **0 on success, 1 on failure**, including TMT's all-skipped and +no-results exits. `exitCode` records the native TMT value, or `null` before +startup/on signal termination. Capture/output failures can fail azldev even if +TMT exited zero. Validation can fail before any artifact row exists. + +`--dry-run` previews commands without execution or artifact creation; rows have +an empty directory/report list and null exit code. Cancellation bounds waiting +for inherited output pipes, not guest teardown. SIGKILL cannot guarantee output. +Artifacts persist until the caller removes them. Logs and stderr can contain +sensitive test output or credentials; restrict access before sharing. + ## Where plans and tests can live Because `source` pins the **plan** and `discover` resolves the **tests**, the two diff --git a/docs/user/reference/cli/azldev_component_test.md b/docs/user/reference/cli/azldev_component_test.md index e316666e3..75babd4c6 100644 --- a/docs/user/reference/cli/azldev_component_test.md +++ b/docs/user/reference/cli/azldev_component_test.md @@ -60,6 +60,8 @@ on the host; git is required for azldev's own catalog clone unless '--source-dir' is used, though the selected plan's 'discover' step may still invoke git itself regardless of '--source-dir'. +Reports and logs stay under --work-dir. Use -O json for artifact paths and TMT exit codes, including on failure. + ``` azldev component test COMPONENT [flags] ``` @@ -90,7 +92,7 @@ azldev component test COMPONENT [flags] -r, --rpm strings Local RPM to install before testing (required; may be repeated). Typically include the main package and -tests package. --source-dir string Run tmt straight from this local fmf tree instead of cloning the catalog 'source'. Accepts the output of either 'azldev component render' (with 'render.skip-file-filter = true') or 'azldev component prepare-sources'. Local inner-loop convenience; not used by cloud (TEE) runs. -t, --test strings Mapped TMT test name to run (may be repeated; defaults to all) - --work-dir string Directory for cloned metadata and TMT artifacts (default: current directory) + --work-dir string Parent for the reusable TMT venv and fresh retained test artifacts (default: current directory) ``` ### Options inherited from parent commands diff --git a/internal/app/azldev/agentskill/agentskill_test.go b/internal/app/azldev/agentskill/agentskill_test.go index a263a100f..5dbaa2a74 100644 --- a/internal/app/azldev/agentskill/agentskill_test.go +++ b/internal/app/azldev/agentskill/agentskill_test.go @@ -153,6 +153,17 @@ func TestBuildComponentSkillDocumentsRPMDevBumpspecPreview(t *testing.T) { assert.NotContains(t, doc, "rpmdev-bumpspec` or RPM macros") } +func TestBuildComponentSkillDocumentsTMTArtifactsInBothModes(t *testing.T) { + for _, withoutLockfile := range []bool{false, true} { + doc, err := agentskill.NewCatalog(withoutLockfile).SkillDocument("azldev-build-component", testParams()) + require.NoError(t, err) + assert.Contains(t, doc, "-O json") + assert.Contains(t, doc, "relative JUnit paths") + assert.Contains(t, doc, "early failures may have logs but no JUnit") + assert.Contains(t, doc, "disposable Azure Linux 4 test container") + } +} + func TestComponentTomlSkillDocumentsRPMDevBumpspecPreview(t *testing.T) { doc, err := agentskill.SkillDocument("azldev-comp-toml", testParams()) require.NoError(t, err) diff --git a/internal/app/azldev/agentskill/content/build-component.md.tmpl b/internal/app/azldev/agentskill/content/build-component.md.tmpl index 9375d5e22..0b9c805a1 100644 --- a/internal/app/azldev/agentskill/content/build-component.md.tmpl +++ b/internal/app/azldev/agentskill/content/build-component.md.tmpl @@ -38,6 +38,15 @@ Prefer `comp render` for quick overlay verification; use `comp diff-sources` to exact overlay effect (it fetches sources once, applies overlays to a copy, and diffs the two trees). Builds can be slow — set generous timeouts. +For components with TMT tests, `azldev component test --image-path +--rpm --plan /plans/smoke --work-dir -O json` retains +logs and available native JUnit reports. JSON rows give the test, TMT exit code, +artifact directory, and relative JUnit paths. Keep the command's nonzero exit +status; early failures may have logs but no JUnit. Artifacts persist until removed. +Do not use +`--provision local` on a development host: it installs packages and executes +tests on that machine; use a disposable Azure Linux 4 test container instead. + Finalize with `azldev comp update -p ` before opening a PR (see the `azldev-update-component` skill). diff --git a/internal/app/azldev/agentskill/content/withoutlockfile/build-component.md.tmpl b/internal/app/azldev/agentskill/content/withoutlockfile/build-component.md.tmpl index cac327d99..01b7976b8 100644 --- a/internal/app/azldev/agentskill/content/withoutlockfile/build-component.md.tmpl +++ b/internal/app/azldev/agentskill/content/withoutlockfile/build-component.md.tmpl @@ -38,6 +38,15 @@ Prefer `comp render` for quick overlay verification; use `comp diff-sources` to exact overlay effect (it fetches sources once, applies overlays to a copy, and diffs the two trees). Builds can be slow — set generous timeouts. +For components with TMT tests, `azldev component test --image-path +--rpm --plan /plans/smoke --work-dir -O json` retains +logs and available native JUnit reports. JSON rows give the test, TMT exit code, +artifact directory, and relative JUnit paths. Keep the command's nonzero exit +status; early failures may have logs but no JUnit. Artifacts persist until removed. +Do not use +`--provision local` on a development host: it installs packages and executes +tests on that machine; use a disposable Azure Linux 4 test container instead. + Finalize with `azldev comp refresh-upstream-commit -p ` before opening a PR (see the `azldev-refresh-upstream-commit` skill). diff --git a/internal/app/azldev/cmds/component/test.go b/internal/app/azldev/cmds/component/test.go index 62fe5a361..3fa02cd48 100644 --- a/internal/app/azldev/cmds/component/test.go +++ b/internal/app/azldev/cmds/component/test.go @@ -72,7 +72,9 @@ azldev creates or reuses an isolated Python environment under --work-dir and installs TMT with virtual-provisioner support there. python3 must be available on the host; git is required for azldev's own catalog clone unless '--source-dir' is used, though the selected plan's 'discover' step may still -invoke git itself regardless of '--source-dir'.`, +invoke git itself regardless of '--source-dir'. + +Reports and logs stay under --work-dir. Use -O json for artifact paths and TMT exit codes, including on failure.`, Example: ` # Build the component first azldev component build buildah @@ -86,7 +88,7 @@ invoke git itself regardless of '--source-dir'.`, --rpm ./base/out/rpms/rpm-base/buildah-tests-1.29.0-5.x86_64.rpm`, Args: cobra.ExactArgs(1), RunE: azldev.RunFuncWithExtraArgs(func(env *azldev.Env, args []string) (interface{}, error) { - return nil, tmt.RunComponentTests(env, args[0], options) + return tmt.RunComponentTests(env, args[0], options) }), } @@ -107,7 +109,8 @@ func registerComponentTestFlags(cmd *cobra.Command, options *tmt.Options) { &options.Tests, "test", "t", nil, "Mapped TMT test name to run (may be repeated; defaults to all)", ) cmd.Flags().StringVar( - &options.WorkDir, "work-dir", "", "Directory for cloned metadata and TMT artifacts (default: current directory)", + &options.WorkDir, "work-dir", "", "Parent for the reusable TMT venv and fresh retained test artifacts "+ + "(default: current directory)", ) _ = cmd.MarkFlagDirname("work-dir") cmd.Flags().StringVar(&options.Provision, "provision", tmt.ProvisionVirtual, diff --git a/internal/app/azldev/cmds/component/tmtrunner_internal_test.go b/internal/app/azldev/cmds/component/tmtrunner_internal_test.go new file mode 100644 index 000000000..491e600f5 --- /dev/null +++ b/internal/app/azldev/cmds/component/tmtrunner_internal_test.go @@ -0,0 +1,344 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package component_test + +import ( + "encoding/csv" + "encoding/json" + "errors" + "io" + "os" + "os/exec" + "path/filepath" + "slices" + "strings" + "testing" + "time" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/cmds/component" + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/testutils" + "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + testProjectDir = "/project" + tmtProgram = "tmt" +) + +func reportTestEnv(t *testing.T) *testutils.TestEnv { + t.Helper() + env := newTMTTestEnv(t, nil) + env.Env.SetDefaultReportFormat(azldev.ReportFormatJSON) + env.CmdFactory.RegisterCommandInSearchPath("python3") + + for path, content := range map[string]string{ + testImagePath: "image", testRPMPath: "rpm", "/source/.fmf/version": "1\n", + } { + require.NoError(t, fileutils.WriteFile(env.TestFS, path, []byte(content), fileperms.PrivateFile)) + } + + env.CmdFactory.RunAndGetOutputHandler = func(*exec.Cmd) (string, error) { + return `[{"name":"/plans/all"}]`, nil + } + + return env +} + +func executeReportTest(t *testing.T, env *testutils.TestEnv) ([]azldev.TestRunResult, error) { + t.Helper() + + var output strings.Builder + env.Env.SetReportFile(&output) + + cmd := component.NewComponentTestCmd() + cmd.SetArgs([]string{ + testTMTComponentName, "--image-path", testImagePath, "--rpm", testRPMPath, + "--source-dir", "/source", "--work-dir", "/work", "--plan", "/plans/all", + }) + err := cmd.ExecuteContext(env.Env) + + var results []azldev.TestRunResult + + decoder := json.NewDecoder(strings.NewReader(output.String())) + require.NoError(t, decoder.Decode(&results), "stdout: %s", output.String()) + require.ErrorIs(t, decoder.Decode(new(any)), io.EOF) + + return results, err +} + +func TestComponentTestArtifacts(t *testing.T) { + for _, failure := range []string{"", "test", "setup", "no-report"} { + t.Run(failure, func(t *testing.T) { + env := reportTestEnv(t) + env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + require.Equal(t, 2*time.Second, cmd.WaitDelay) + _, err := io.WriteString(cmd.Stdout, "noisy stdout\n") + require.NoError(t, err) + _, err = io.WriteString(cmd.Stderr, "noisy stderr\n") + require.NoError(t, err) + + if failure == "setup" { + return errors.New("setup failed") + } + + if filepath.Base(cmd.Args[0]) != tmtProgram { + return nil + } + + native := cmd.Args[slices.Index(cmd.Args, "--id")+1] + store := cmd.Args[slices.Index(cmd.Args, "--workdir-root")+1] + assert.Equal(t, filepath.Join(filepath.Dir(native), "workdir-root"), store) + assert.NotEqual(t, filepath.Dir(filepath.Dir(native)), store, "do not chmod the private artifact root") + assert.Less(t, slices.Index(cmd.Args, "--workdir-root"), slices.Index(cmd.Args, "plan")) + + if failure != "no-report" { + for _, plan := range []string{"", "plans/all"} { + path := filepath.Join(native, plan, "report", "azldev-junit", "junit.xml") + require.NoError(t, fileutils.WriteFile(env.TestFS, path, []byte(""), fileperms.PrivateFile)) + } + + require.NoError(t, fileutils.WriteFile(env.TestFS, + filepath.Join(native, "report/supplied-junit/junit.xml"), []byte("not ours"), fileperms.PrivateFile)) + } + + if failure != "" { + return errors.New("runner failed") + } + + return nil + } + + results, err := executeReportTest(t, env) + if failure == "" { + require.NoError(t, err) + } else { + require.Error(t, err) + } + + require.Len(t, results, 1) + result := results[0] + assert.Nil(t, result.ExitCode, "mock did not observe an exit code") + assert.Equal(t, testTMTComponentName, result.Test) + + expectedReports := 2 + if failure == "setup" || failure == "no-report" { + expectedReports = 0 + + assert.NotContains(t, err.Error(), "run.yaml", "do not invent missing-state failures") + } + + assert.Len(t, result.JUnit, expectedReports) + + for _, path := range result.JUnit { + require.True(t, filepath.IsLocal(path)) + assert.NotContains(t, path, "supplied-junit") + _, err := env.TestFS.Stat(filepath.Join(result.ArtifactDir, path)) + require.NoError(t, err) + } + + log, err := fileutils.ReadFile(env.TestFS, filepath.Join(result.ArtifactDir, "setup.log")) + require.NoError(t, err) + assert.Contains(t, string(log), "noisy stdout") + assert.Contains(t, string(log), "noisy stderr") + }) + } +} + +func TestComponentTestCSVWithNoisyCommands(t *testing.T) { + env := reportTestEnv(t) + env.Env.SetDefaultReportFormat(azldev.ReportFormatCSV) + + var output strings.Builder + env.Env.SetReportFile(&output) + env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + _, err := io.WriteString(cmd.Stdout, "not a CSV header\n") + + return err + } + cmd := component.NewComponentTestCmd() + cmd.SetArgs([]string{ + testTMTComponentName, "--image-path", testImagePath, "--rpm", testRPMPath, + "--source-dir", "/source", "--work-dir", "/work", "--plan", "/plans/all", + }) + require.NoError(t, cmd.ExecuteContext(env.Env)) + + rows, err := csv.NewReader(strings.NewReader(output.String())).ReadAll() + require.NoError(t, err) + require.Len(t, rows, 2) + assert.Equal(t, []string{"test", "artifactDir"}, rows[0]) + assert.Equal(t, testTMTComponentName, rows[1][0]) + assert.NotContains(t, output.String(), "") +} + +func TestComponentTestFreshArtifactsAndFailFast(t *testing.T) { + env := reportTestEnv(t) + comp := env.Config.Components[testTMTComponentName] + comp.Tests = &projectconfig.ComponentTestsConfig{Tests: []projectconfig.TestRef{{Name: "first"}, {Name: "second"}}} + + env.Config.Components[testTMTComponentName] = comp + for _, name := range []string{"first", "second"} { + env.Config.Tests[name] = projectconfig.TestDefinition{Type: "tmt", Tmt: map[string]any{}} + } + + var roots []string + + env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + if filepath.Base(cmd.Args[0]) == tmtProgram { + roots = append(roots, cmd.Args[slices.Index(cmd.Args, "--workdir-root")+1]) + + return errors.New("runner failed") + } + + return nil + } + first, err := executeReportTest(t, env) + require.Error(t, err) + second, err := executeReportTest(t, env) + require.Error(t, err) + require.Len(t, first, 1) + require.Len(t, second, 1) + require.Len(t, roots, 2) + assert.NotEqual(t, roots[0], roots[1]) + assert.NotEqual(t, first[0].ArtifactDir, second[0].ArtifactDir) + assert.Equal(t, "first", first[0].Test, "no rows for unattempted entries") +} + +type unreadableArtifactFS struct { + opctx.FS + blocked string + err error +} + +func (fs unreadableArtifactFS) Open(path string) (afero.File, error) { + if path == fs.blocked { + return nil, &os.PathError{Op: "open", Path: path, Err: fs.err} + } + + return fs.FS.Open(path) +} + +func (fs unreadableArtifactFS) Stat(path string) (os.FileInfo, error) { + if strings.HasPrefix(path, fs.blocked+string(filepath.Separator)) { + return nil, &os.PathError{Op: "stat", Path: path, Err: fs.err} + } + + return fs.FS.Stat(path) +} + +func TestComponentTestDiscoveryPreservesRunnerError(t *testing.T) { + runnerErr := errors.New("test runner failed") + for _, runErr := range []error{nil, runnerErr} { + env := reportTestEnv(t) + filesystem := &unreadableArtifactFS{FS: env.TestFS, blocked: "/unused", err: os.ErrPermission} + configureReportTestEnv(t, env, filesystem, false) + env.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + if filepath.Base(cmd.Args[0]) != tmtProgram { + return nil + } + + native := cmd.Args[slices.Index(cmd.Args, "--id")+1] + require.NoError(t, env.TestFS.MkdirAll(native, fileperms.PrivateDir)) + filesystem.blocked = native + + return runErr + } + + results, err := executeReportTest(t, env) + if runErr == nil { + require.NoError(t, err, "discovery must not turn a successful run into a failure") + } else { + require.ErrorIs(t, err, runnerErr, "discovery must not replace the runner error") + } + + require.Len(t, results, 1) + assert.NotEmpty(t, results[0].ArtifactDir) + assert.Empty(t, results[0].JUnit) + } +} + +type reportFSFactory struct{ filesystem opctx.FS } + +func (factory reportFSFactory) FS() opctx.FS { return factory.filesystem } + +func configureReportTestEnv(t *testing.T, env *testutils.TestEnv, filesystem opctx.FS, dryRun bool) { + t.Helper() + + options := azldev.NewEnvOptions() + options.ProjectDir, options.Config = testProjectDir, env.Config + options.Interfaces = env.TestInterfaces + options.Interfaces.FileSystemFactory = reportFSFactory{filesystem} + options.EventListener = env.EventListener + options.DryRunnable = azldev.NewAppDryRunnable(dryRun) + env.Env = azldev.NewEnv(t.Context(), options) + env.Env.SetDefaultReportFormat(azldev.ReportFormatJSON) +} + +func TestComponentTestDryRunDoesNotCreateArtifacts(t *testing.T) { + env := reportTestEnv(t) + configureReportTestEnv(t, env, env.TestFS, true) + env.CmdFactory.RunHandler = func(*exec.Cmd) error { + t.Fatal("dry-run executed command") + + return nil + } + results, err := executeReportTest(t, env) + require.NoError(t, err) + require.Len(t, results, 1) + assert.Empty(t, results[0].ArtifactDir) + assert.Empty(t, results[0].JUnit) + assert.Nil(t, results[0].ExitCode) + + _, err = env.TestFS.Stat("/work") + require.ErrorIs(t, err, os.ErrNotExist) +} + +type transcriptFailureFS struct { + opctx.FS + closeFailure bool +} + +func (fs transcriptFailureFS) OpenFile(name string, flag int, perm os.FileMode) (afero.File, error) { + file, err := fs.FS.OpenFile(name, flag, perm) + if err != nil { + return nil, err + } + + return transcriptFailureFile{File: file, closeFailure: fs.closeFailure}, nil +} + +type transcriptFailureFile struct { + afero.File + closeFailure bool +} + +func (file transcriptFailureFile) Write(data []byte) (int, error) { + if !file.closeFailure { + return 0, io.ErrShortWrite + } + + return file.File.Write(data) +} + +func (file transcriptFailureFile) Close() error { + return errors.Join(file.File.Close(), io.ErrClosedPipe) +} + +func TestComponentTestCaptureFailures(t *testing.T) { + for _, closeFailure := range []bool{false, true} { + env := reportTestEnv(t) + configureReportTestEnv(t, env, transcriptFailureFS{FS: env.TestFS, closeFailure: closeFailure}, false) + results, err := executeReportTest(t, env) + require.Error(t, err) + require.Len(t, results, 1) + assert.Empty(t, results[0].JUnit) + assert.NotEmpty(t, results[0].ArtifactDir) + } +} diff --git a/internal/app/azldev/command.go b/internal/app/azldev/command.go index 54c326c8d..703e667f9 100644 --- a/internal/app/azldev/command.go +++ b/internal/app/azldev/command.go @@ -7,6 +7,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "os" "reflect" @@ -359,7 +360,14 @@ func reportResultsAsJSON(env *Env, results interface{}) error { } if len(jsonBytes) > 0 { - fmt.Fprintf(env.ReportFile(), "%s\n", jsonBytes) + count, err := fmt.Fprintf(env.ReportFile(), "%s\n", jsonBytes) + if err == nil && count != len(jsonBytes)+1 { + err = io.ErrShortWrite + } + + if err != nil { + return fmt.Errorf("failed to write JSON command results:\n%w", err) + } } return nil diff --git a/internal/app/azldev/command_test.go b/internal/app/azldev/command_test.go index ea46a9525..67f9de8d5 100644 --- a/internal/app/azldev/command_test.go +++ b/internal/app/azldev/command_test.go @@ -5,6 +5,7 @@ package azldev_test import ( "errors" + "io" "strings" "testing" @@ -14,6 +15,38 @@ import ( "github.com/stretchr/testify/assert" ) +type failingReportWriter struct{} + +func (failingReportWriter) Write([]byte) (int, error) { + return 0, io.ErrClosedPipe +} + +type shortReportWriter struct{} + +func (shortReportWriter) Write([]byte) (int, error) { + return 0, nil +} + +func TestRunFuncJSONShortWriteFailure(t *testing.T) { + env := testutils.NewTestEnv(t) + env.Env.SetDefaultReportFormat(azldev.ReportFormatJSON) + env.Env.SetReportFile(shortReportWriter{}) + run := azldev.RunFunc(func(*azldev.Env) (interface{}, error) { + return []string{"result"}, nil + }) + assert.ErrorIs(t, run(testCmd(env.Env), nil), io.ErrShortWrite) +} + +func TestRunFuncJSONWriteFailure(t *testing.T) { + env := testutils.NewTestEnv(t) + env.Env.SetDefaultReportFormat(azldev.ReportFormatJSON) + env.Env.SetReportFile(failingReportWriter{}) + run := azldev.RunFunc(func(*azldev.Env) (interface{}, error) { + return []string{"result"}, nil + }) + assert.ErrorIs(t, run(testCmd(env.Env), nil), io.ErrClosedPipe) +} + func testCmd(env *azldev.Env) (cmd *cobra.Command) { cmd = &cobra.Command{} cmd.SetContext(env) diff --git a/internal/app/azldev/testresult.go b/internal/app/azldev/testresult.go new file mode 100644 index 000000000..500b74e3d --- /dev/null +++ b/internal/app/azldev/testresult.go @@ -0,0 +1,14 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package azldev + +// TestRunResult identifies a test runner's retained artifacts. +// JUnit paths are relative to [TestRunResult.ArtifactDir]. +// [TestRunResult.ExitCode] is nil if no normal process exit was observed. +type TestRunResult struct { + Test string `json:"test"` + ExitCode *int `json:"exitCode"` + ArtifactDir string `json:"artifactDir"` + JUnit []string `json:"junit"` +} diff --git a/internal/tmt/component.go b/internal/tmt/component.go index e84584777..9604e10d3 100644 --- a/internal/tmt/component.go +++ b/internal/tmt/component.go @@ -4,12 +4,10 @@ package tmt import ( - "bytes" _ "embed" "encoding/json" "errors" "fmt" - "io" "io/fs" "log/slog" "os" @@ -28,7 +26,7 @@ import ( "github.com/microsoft/azure-linux-dev-tools/internal/utils/prereqs" ) -// Options holds options for running component TMT tests locally. +// Options holds options for running component TMT tests. type Options struct { ImagePath string RPMPaths []string @@ -78,6 +76,7 @@ type tmtRunSettings struct { // SourceDir, when set, runs tmt straight from this local fmf tree instead of // cloning the catalog 'source' (see Options.SourceDir). SourceDir string + SetupLog string } const ( @@ -118,35 +117,37 @@ def _azldev_pull(self, source=None, destination=None, options=None): GuestSsh.pull = _azldev_pull ` -// RunComponentTests runs the component's selected TMT tests with the supplied inputs. -func RunComponentTests(env *azldev.Env, componentName string, options *Options) error { - if err := validateComponentTestOptions(env, options); err != nil { - return err +// RunComponentTests runs the component's selected TMT tests with retained artifacts. +func RunComponentTests( + env *azldev.Env, componentName string, options *Options, +) ([]azldev.TestRunResult, error) { + if err := validateOptions(env, options); err != nil { + return nil, err } imagePath, err := resolveTMTImagePath(env, options.Provision, options.ImagePath) if err != nil { - return err + return nil, err } rpms, err := absoluteRegularFiles(env, options.RPMPaths, "rpm") if err != nil { - return err + return nil, err } sourceDir, err := effectiveSourceDir(env, componentName, options) if err != nil { - return err + return nil, err } if sourceDir != "" { preflightWorkDir, err := componentTMTWorkDir(env, options.WorkDir) if err != nil { - return fmt.Errorf("resolve work directory:\n%w", err) + return nil, fmt.Errorf("resolve work directory:\n%w", err) } if dirsOverlap(sourceDir, preflightWorkDir) { - return fmt.Errorf( + return nil, fmt.Errorf( "'--source-dir' (%#q) and the work directory (%#q) must not overlap: azldev creates "+ "per-test directories and TMT writes run artifacts under the work directory, which "+ "would otherwise modify the supplied source tree", sourceDir, preflightWorkDir) @@ -155,7 +156,7 @@ func RunComponentTests(env *azldev.Env, componentName string, options *Options) resolved, err := resolveComponentTMTTests(env, componentName, options.Tests, sourceDir, options.Plan) if err != nil { - return err + return nil, err } // Preflight every selected test's decoded config against sourceDir and the @@ -163,7 +164,7 @@ func RunComponentTests(env *azldev.Env, componentName string, options *Options) // install, cloning), so these deterministic input errors surface // immediately instead of after network work and on-disk artifacts. if err := preflightTMTTests(resolved, sourceDir, options.Provision); err != nil { - return err + return nil, err } // Validate the source dir up front so '--source-dir'/'--from-spec' fail fast, @@ -171,34 +172,21 @@ func RunComponentTests(env *azldev.Env, componentName string, options *Options) // --work-dir. if sourceDir != "" { if _, err := resolveSourceRunDir(env, sourceDir); err != nil { - return err + return nil, err } } - workDir, tmtProgramPath, err := prepareTMTEnvironment(env, options.WorkDir, options.Provision, sourceDir != "") - if err != nil { - return err - } - settings := tmtRunSettings{ - ImagePath: imagePath, - RPMs: rpms, - WorkDir: workDir, - TMTProgramPath: tmtProgramPath, - Provision: options.Provision, - SourceDir: sourceDir, - } - - for _, test := range resolved { - if err := runOneTMTTest(env, test, settings); err != nil { - return fmt.Errorf("TMT test %#q:\n%w", test.Name, err) - } + ImagePath: imagePath, + RPMs: rpms, + Provision: options.Provision, + SourceDir: sourceDir, } - return nil + return executeComponentTMTTests(env, options.WorkDir, resolved, settings) } -func validateComponentTestOptions(env *azldev.Env, options *Options) error { +func validateOptions(env *azldev.Env, options *Options) error { if err := validateTMTProvisionOptions(options.Provision, options.ImagePath); err != nil { return err } @@ -375,7 +363,7 @@ func overrideTMTPlan(tests []projectconfig.ResolvedTest, plan string) []projectc } func prepareTMTEnvironment( - env *azldev.Env, configuredWorkDir string, provision string, fromSourceDir bool, + env *azldev.Env, configuredWorkDir string, provision string, fromSourceDir bool, setupLog string, ) (string, string, error) { workDir, err := componentTMTWorkDir(env, configuredWorkDir) if err != nil { @@ -391,7 +379,7 @@ func prepareTMTEnvironment( return "", "", fmt.Errorf("create work directory:\n%w", err) } - tmtProgramPath, err = ensureTMTVenv(env, workDir, provision, fromSourceDir) + tmtProgramPath, err = ensureTMTVenv(env, workDir, provision, fromSourceDir, setupLog) if err != nil { return "", "", err } @@ -404,9 +392,10 @@ func prepareTMTEnvironment( // cloning) are explicit host prerequisites, while the test framework itself is // installed in a venv under the selected work directory rather than assumed to // be packaged by the host distribution. For virtual provisioning, the testcloud -// plugin supplies provisioner support. For local provisioning, only base TMT is -// required. -func ensureTMTVenv(env *azldev.Env, workDir string, provision string, fromSourceDir bool) (string, error) { +// plugin supplies provisioner support. Both modes include native JUnit export. +func ensureTMTVenv( + env *azldev.Env, workDir string, provision string, fromSourceDir bool, setupLog string, +) (string, error) { if err := prereqs.RequireExecutable(env, tmtPythonProgram, nil); err != nil { return "", fmt.Errorf("python3 is required to run TMT tests:\n%w", err) } @@ -431,7 +420,7 @@ func ensureTMTVenv(env *azldev.Env, workDir string, provision string, fromSource if !venvExists { slog.Info("Creating TMT Python venv", slog.String("path", venvDir)) - if err := runHostCommand(env, "", tmtPythonProgram, "-m", "venv", venvDir); err != nil { + if err := runHostCommand(env, setupLog, "", tmtPythonProgram, "-m", "venv", venvDir); err != nil { return "", fmt.Errorf("create TMT Python venv at %#q:\n%w", venvDir, err) } } else { @@ -453,7 +442,7 @@ func ensureTMTVenv(env *azldev.Env, workDir string, provision string, fromSource ) if err := runHostCommand( - env, "", venvPython, "-m", "pip", "install", "--quiet", + env, setupLog, "", venvPython, "-m", "pip", "install", "--quiet", "--constraint", requirementsPath, requirement, ); err != nil { return "", fmt.Errorf("install TMT:\n%w", err) @@ -464,10 +453,10 @@ func ensureTMTVenv(env *azldev.Env, workDir string, provision string, fromSource func tmtInstallRequirement(provision string) string { if provision == ProvisionVirtual { - return "tmt[provision-virtual]" + return "tmt[provision-virtual,report-junit]" } - return "tmt" + return "tmt[report-junit]" } func componentTMTWorkDir(env *azldev.Env, configuredWorkDir string) (string, error) { @@ -546,14 +535,6 @@ func absoluteRegularFiles(env *azldev.Env, paths []string, kind string) ([]strin return result, nil } -func removePreviousTMTRepository(env *azldev.Env, repoDir string) error { - if err := env.FS().RemoveAll(repoDir); err != nil { - return fmt.Errorf("remove test metadata repository %#q:\n%w", repoDir, err) - } - - return nil -} - // preflightTMTTests validates every selected test's decoded tmt config against // sourceDir and the provisioner before RunComponentTests prepares the TMT // environment (venv creation, pip install) or runOneTMTTest clones metadata. @@ -589,7 +570,7 @@ func preflightTMTTests(resolved []projectconfig.ResolvedTest, sourceDir string, func selectTMTTests(tests []projectconfig.ResolvedTest, selectors []string) []projectconfig.ResolvedTest { result := make([]projectconfig.ResolvedTest, 0, len(tests)) for _, test := range tests { - if test.Definition.Type != "tmt" { + if test.Definition.Type != tmtProgram { continue } @@ -601,7 +582,9 @@ func selectTMTTests(tests []projectconfig.ResolvedTest, selectors []string) []pr return result } -func runOneTMTTest(env *azldev.Env, test projectconfig.ResolvedTest, settings tmtRunSettings) error { +func runOneTMTTest( + env *azldev.Env, test projectconfig.ResolvedTest, settings tmtRunSettings, result *azldev.TestRunResult, +) error { config, err := decodeTMTConfig(test.Definition.Tmt) if err != nil { return err @@ -615,11 +598,11 @@ func runOneTMTTest(env *azldev.Env, test projectconfig.ResolvedTest, settings tm return fmt.Errorf("test %#q: 'tmt.source' (git-url, ref) is required unless '--source-dir' is set", test.Name) } - testDir := filepath.Join(settings.WorkDir, test.Name) + testDir := settings.WorkDir repoDir := filepath.Join(testDir, "repo") - tmtWorkDir := filepath.Join(testDir, "tmt") + tmtWorkDir := filepath.Join(testDir, "native") - pluginDir, err := prepareTMTTestDir(env, testDir, repoDir, settings) + pluginDir, err := prepareTMTTestDir(env, testDir, settings) if err != nil { return err } @@ -633,11 +616,15 @@ func runOneTMTTest(env *azldev.Env, test projectconfig.ResolvedTest, settings tm return err } } else { - if err := runHostCommand(env, testDir, "git", "clone", "--no-checkout", config.Source.GitURL, repoDir); err != nil { + if err := runHostCommand( + env, settings.SetupLog, testDir, "git", "clone", "--no-checkout", config.Source.GitURL, repoDir, + ); err != nil { return fmt.Errorf("clone test metadata:\n%w", err) } - if err := runHostCommand(env, repoDir, "git", "checkout", "--detach", config.Source.Ref); err != nil { + if err := runHostCommand( + env, settings.SetupLog, repoDir, "git", "checkout", "--detach", config.Source.Ref, + ); err != nil { return fmt.Errorf("checkout test metadata:\n%w", err) } } @@ -652,7 +639,7 @@ func runOneTMTTest(env *azldev.Env, test projectconfig.ResolvedTest, settings tm } hardwareArgs, err = resolvedPlanHardwareArgs( - env, runDir, settings.TMTProgramPath, config.Plan, + env, runDir, settings.TMTProgramPath, config.Plan, settings.SetupLog, ) if err != nil { return fmt.Errorf("resolve hardware for TMT plan %#q:\n%w", config.Plan, err) @@ -661,21 +648,11 @@ func runOneTMTTest(env *azldev.Env, test projectconfig.ResolvedTest, settings tm args := componentTMTArgs(config, tmtWorkDir, settings.Provision, settings.ImagePath, hardwareArgs, settings.RPMs) - if err := runTMTCommand(env, runDir, pluginDir, settings.TMTProgramPath, settings.Provision, args...); err != nil { - return fmt.Errorf("run TMT plan %#q (artifacts: %#q):\n%w", tmtPlanDescription(config.Plan), tmtWorkDir, err) - } - - return nil -} - -// tmtPlanDescription returns a human-readable plan selection for error -// messages, since config.Plan may be empty (every enabled plan runs). -func tmtPlanDescription(plan string) string { - if plan == "" { - return "" - } + result.ExitCode, err = runTMTCommand( + env, filepath.Join(testDir, "runner.log"), runDir, pluginDir, settings.TMTProgramPath, settings.Provision, args..., + ) - return plan + return err } // resolveSourceRunDir validates that the given directory carries an fmf root, @@ -714,7 +691,7 @@ func resolveSourceRunDir(env *azldev.Env, sourceDir string) (string, error) { // prepareTMTTestDir creates the per-test directory and, for virtual runs, the // testcloud plugin. It returns the plugin directory, which is empty when no // plugin is needed. -func prepareTMTTestDir(env *azldev.Env, testDir string, repoDir string, settings tmtRunSettings) (string, error) { +func prepareTMTTestDir(env *azldev.Env, testDir string, settings tmtRunSettings) (string, error) { if env.DryRun() { return "", nil } @@ -723,19 +700,6 @@ func prepareTMTTestDir(env *azldev.Env, testDir string, repoDir string, settings return "", fmt.Errorf("create test directory:\n%w", err) } - // A caller may intentionally reuse --work-dir after a failed or completed - // run. Metadata is always cloned at the pinned ref, so remove only the - // previous checkout while retaining TMT artifacts for diagnosis. repoDir is - // only ever populated by our own clone step, so this is skipped entirely for - // '--source-dir': that flow never clones into repoDir, and the caller's - // supplied tree must never be touched, even if it happens to coincide with - // (or contain) the managed repoDir path. - if settings.SourceDir == "" { - if err := removePreviousTMTRepository(env, repoDir); err != nil { - return "", fmt.Errorf("remove previous test metadata checkout:\n%w", err) - } - } - if settings.Provision != ProvisionVirtual { return "", nil } @@ -759,8 +723,7 @@ func validateTMTTestName(name string) error { // componentTMTArgs builds a run which changes only the provisioner and adds the // candidate RPM installation. In particular, it deliberately leaves discover, -// execute, and report to the selected plan: upstream Fedora plans commonly use -// non-default plugins or scripts for those steps. +// and execute to the selected plan, and retains its existing reporters. func componentTMTArgs( config tmtConfig, tmtWorkDir string, @@ -774,7 +737,8 @@ func componentTMTArgs( // Both provisioners need --become: candidate RPM installation and most plans' // execute steps require root, and neither the testcloud guest user nor the // invoking local user is root. TMT escalates with passwordless sudo. - args = append(args, "run", "--all", "--keep", "--workdir-root", tmtWorkDir) + args = append(args, "run", "--all", "--keep", "--id", tmtWorkDir, + "--workdir-root", filepath.Join(filepath.Dir(tmtWorkDir), "workdir-root")) // Omitting '--name' entirely (plan empty) makes tmt discover and run every // enabled plan it finds — the same default Fedora's own Testing Farm uses, @@ -806,6 +770,8 @@ func componentTMTArgs( args = append(args, "--package", rpm) } + args = append(args, "report", "--insert", "--how", "junit", "--name", tmtJUnitPhase, "--no-include-output-log") + return args } @@ -815,7 +781,7 @@ func componentTMTArgs( // resolved constraints retains requirements such as memory, CPU topology, and // disk size while adding UEFI firmware. func resolvedPlanHardwareArgs( - env *azldev.Env, repoDir string, tmtProgramPath string, plan string, + env *azldev.Env, repoDir string, tmtProgramPath string, plan string, setupLog string, ) ([]string, error) { // The plan export command produces no output in dry-run mode. In that case, // emit only boot.method so the printed invocation represents what would run. @@ -823,7 +789,7 @@ func resolvedPlanHardwareArgs( return []string{"boot.method = uefi"}, nil } - hardware, err := exportedPlanHardware(env, repoDir, tmtProgramPath, plan) + hardware, err := exportedPlanHardware(env, repoDir, tmtProgramPath, plan, setupLog) if err != nil { return nil, err } @@ -831,8 +797,10 @@ func resolvedPlanHardwareArgs( return mergePlanHardwareArgs(hardware) } -func exportedPlanHardware(env *azldev.Env, repoDir string, tmtProgramPath string, plan string) (map[string]any, error) { - output, err := runHostCommandOutput(env, repoDir, tmtProgramPath, "plan", "export", plan, "--how", "json") +func exportedPlanHardware( + env *azldev.Env, repoDir string, tmtProgramPath string, plan string, setupLog string, +) (map[string]any, error) { + output, err := runHostCommandOutput(env, setupLog, repoDir, tmtProgramPath, "plan", "export", plan, "--how", "json") if err != nil { return nil, err } @@ -1019,78 +987,35 @@ func validateTMTProvisionOptions(provision string, imagePath string) error { return nil } -func runHostCommand(env *azldev.Env, dir string, program string, args ...string) error { +func runHostCommand(env *azldev.Env, logPath string, dir string, program string, args ...string) error { command := exec.CommandContext(env, program, args...) command.Dir = dir - command.Stdout = os.Stdout - - var stderr bytes.Buffer + _, _, err := runCapturedTMTCommand(env, command, logPath, false) - command.Stderr = io.MultiWriter(os.Stderr, &stderr) - - wrapped, err := env.Command(command) - if err != nil { - return fmt.Errorf("wrap host command %#q:\n%w", program, err) - } - - if err := wrapped.Run(env); err != nil { - if trimmedStderr := strings.TrimSpace(stderr.String()); trimmedStderr != "" { - return fmt.Errorf("run host command %#q:\n%s\n%w", program, trimmedStderr, err) - } - - return fmt.Errorf("run host command %#q:\n%w", program, err) - } - - return nil + return err } -func runHostCommandOutput(env *azldev.Env, dir string, program string, args ...string) ([]byte, error) { +func runHostCommandOutput(env *azldev.Env, logPath string, dir string, program string, args ...string) ([]byte, error) { command := exec.CommandContext(env, program, args...) command.Dir = dir + output, _, err := runCapturedTMTCommand(env, command, logPath, true) - var stderr bytes.Buffer - - command.Stderr = &stderr - - wrapped, err := env.Command(command) - if err != nil { - return nil, fmt.Errorf("wrap host command %#q:\n%w", program, err) - } - - output, err := wrapped.RunAndGetOutput(env) - if err != nil { - if trimmedStderr := strings.TrimSpace(stderr.String()); trimmedStderr != "" { - return nil, fmt.Errorf("run host command %#q:\n%s\n%w", program, trimmedStderr, err) - } - - return nil, fmt.Errorf("run host command %#q:\n%w", program, err) - } - - return []byte(output), nil + return []byte(output), err } // runTMTCommand runs the TMT CLI with the environment produced by -// [tmtCommandEnv] layered on top of the inherited process environment. +// [tmtCommandEnv] layered on top of the inherited process environment. The exit +// code is absent if the process did not start or was terminated by a signal. func runTMTCommand( - env *azldev.Env, dir string, pluginDir string, tmtProgramPath string, provision string, args ...string, -) error { + env *azldev.Env, logPath string, dir string, pluginDir string, tmtProgramPath string, provision string, args ...string, +) (*int, error) { command := exec.CommandContext(env, tmtProgramPath, args...) command.Dir = dir - command.Stdout = os.Stdout - command.Stderr = os.Stderr command.Env = append(os.Environ(), tmtCommandEnv(env, pluginDir, provision)...) + _, exitCode, err := runCapturedTMTCommand(env, command, logPath, false) - wrapped, err := env.Command(command) - if err != nil { - return fmt.Errorf("wrap tmt command:\n%w", err) - } - - if err := wrapped.Run(env); err != nil { - return fmt.Errorf("run tmt command:\n%w", err) - } - - return nil + return exitCode, err } // tmtCommandEnv returns the environment variables azldev adds on top of the diff --git a/internal/tmt/component_internal_test.go b/internal/tmt/component_internal_test.go index 36fc5ce1c..e774d9584 100644 --- a/internal/tmt/component_internal_test.go +++ b/internal/tmt/component_internal_test.go @@ -24,6 +24,12 @@ import ( const testRenderedSpecDir = "/project/specs/u/util-linux" +const ( + testProjectDir = "/project" + testImagePath = "/project/image.qcow2" + testRPMPath = "/project/component.rpm" +) + func TestDecodeTMTConfig(t *testing.T) { config, err := decodeTMTConfig(map[string]any{ "source": map[string]any{ @@ -87,8 +93,8 @@ func TestValidateTMTProvisionOptions(t *testing.T) { } func TestTMTInstallRequirementUsesProvisionerExtra(t *testing.T) { - assert.Equal(t, "tmt", tmtInstallRequirement(tmtProvisionLocal)) - assert.Equal(t, "tmt[provision-virtual]", tmtInstallRequirement(ProvisionVirtual)) + assert.Equal(t, "tmt[report-junit]", tmtInstallRequirement(tmtProvisionLocal)) + assert.Equal(t, "tmt[provision-virtual,report-junit]", tmtInstallRequirement(ProvisionVirtual)) } func TestSelectTMTTests(t *testing.T) { @@ -148,7 +154,7 @@ func TestFlattenHardwareConstraintsRejectsBooleanGroups(t *testing.T) { func TestResolvedPlanHardwareArgsDryRunSkipsPlanExport(t *testing.T) { testEnv := testutils.NewTestEnv(t) dryRunOptions := azldev.NewEnvOptions() - dryRunOptions.ProjectDir = "/project" + dryRunOptions.ProjectDir = testProjectDir dryRunOptions.Config = testEnv.Config dryRunOptions.DryRunnable = azldev.NewAppDryRunnable(true) dryRunOptions.EventListener = testEnv.EventListener @@ -163,7 +169,7 @@ func TestResolvedPlanHardwareArgsDryRunSkipsPlanExport(t *testing.T) { } args, err := resolvedPlanHardwareArgs( - dryRunEnv, "/work/repo", "/work/tmt", "/plans/smoke", + dryRunEnv, "/work/repo", "/work/tmt", "/plans/smoke", "", ) require.NoError(t, err) @@ -245,14 +251,14 @@ func TestComponentTMTWorkDirDefaultsToCurrentDirectory(t *testing.T) { func TestPrepareTMTEnvironmentDryRunAvoidsFilesystemChanges(t *testing.T) { testEnv := testutils.NewTestEnv(t) dryRunOptions := azldev.NewEnvOptions() - dryRunOptions.ProjectDir = "/project" + dryRunOptions.ProjectDir = testProjectDir dryRunOptions.Config = testEnv.Config dryRunOptions.DryRunnable = azldev.NewAppDryRunnable(true) dryRunOptions.EventListener = testEnv.EventListener dryRunOptions.Interfaces = testEnv.TestInterfaces dryRunEnv := azldev.NewEnv(t.Context(), dryRunOptions) - workDir, tmtProgramPath, err := prepareTMTEnvironment(dryRunEnv, "artifacts", ProvisionVirtual, false) + workDir, tmtProgramPath, err := prepareTMTEnvironment(dryRunEnv, "artifacts", ProvisionVirtual, false, "") require.NoError(t, err) assert.Equal(t, "/project/artifacts", workDir) @@ -323,7 +329,7 @@ func TestRunOneTMTTestFromSourceDirDoesNotClone(t *testing.T) { SourceDir: sourceDir, } - require.NoError(t, runOneTMTTest(testEnv.Env, test, settings)) + require.NoError(t, runOneTMTTest(testEnv.Env, test, settings, &azldev.TestRunResult{})) ranTMT := false @@ -368,7 +374,7 @@ func TestRunOneTMTTestFromSourceDirWithNoCatalogEntryAutoDiscoversPlan(t *testin SourceDir: sourceDir, } - require.NoError(t, runOneTMTTest(testEnv.Env, test, settings)) + require.NoError(t, runOneTMTTest(testEnv.Env, test, settings, &azldev.TestRunResult{})) var tmtArgs []string @@ -609,7 +615,7 @@ func TestDirsOverlap(t *testing.T) { "identical paths": {"/project/source", "/project/source", true}, "b nested under a": {"/project/source", "/project/source/nested", true}, "a nested under b": {"/project/source/nested", "/project/source", true}, - "project root contains work": {"/project", "/project/work", true}, + "project root contains work": {testProjectDir, "/project/work", true}, "unrelated sibling paths": {"/project/source", "/project/work", false}, "similar but distinct prefix": {"/project/source", "/project/source-other", false}, } { @@ -619,23 +625,56 @@ func TestDirsOverlap(t *testing.T) { } } -func TestRemovePreviousTMTRepositoryRemovesOnlyRepository(t *testing.T) { +func TestExecuteComponentTMTTestsUsesFreshMetadataRepositories(t *testing.T) { testEnv := testutils.NewTestEnv(t) - testDir := "/project/tmt-test" - repoDir := filepath.Join(testDir, "repo") - artifactPath := filepath.Join(testDir, "tmt", "results.yaml") - staleMetadataPath := filepath.Join(repoDir, "stale-metadata") + testEnv.CmdFactory.RegisterCommandInSearchPath("python3") + testEnv.CmdFactory.RegisterCommandInSearchPath("git") + + var repositories []string + + testEnv.CmdFactory.RunHandler = func(cmd *exec.Cmd) error { + if filepath.Base(cmd.Args[0]) == "git" && len(cmd.Args) > 1 && cmd.Args[1] == "clone" { + repoDir := cmd.Args[len(cmd.Args)-1] + _, err := testEnv.TestFS.Stat(repoDir) + require.ErrorIs(t, err, fs.ErrNotExist) + require.NoError(t, testEnv.TestFS.MkdirAll(repoDir, fileperms.PublicDir)) + require.NoError(t, fileutils.WriteFile( + testEnv.TestFS, filepath.Join(repoDir, "metadata"), []byte("retained"), fileperms.PrivateFile, + )) + repositories = append(repositories, repoDir) + } + + return nil + } + + tests := []projectconfig.ResolvedTest{{ + Name: "ci", + Definition: projectconfig.TestDefinition{Type: "tmt", Tmt: map[string]any{ + "source": map[string]any{ + "git-url": "https://example.test/tests.git", + "ref": "0123456789012345678901234567890123456789", + }, + }}, + }} - require.NoError(t, testEnv.TestFS.MkdirAll(repoDir, fileperms.PublicDir)) - require.NoError(t, fileutils.WriteFile(testEnv.TestFS, staleMetadataPath, []byte("stale"), fileperms.PrivateFile)) - require.NoError(t, testEnv.TestFS.MkdirAll(filepath.Dir(artifactPath), fileperms.PublicDir)) - require.NoError(t, fileutils.WriteFile(testEnv.TestFS, artifactPath, []byte("preserve"), fileperms.PrivateFile)) + roots := make([]string, 0, 2) - require.NoError(t, removePreviousTMTRepository(testEnv.Env, repoDir)) - _, err := testEnv.TestFS.Stat(repoDir) - require.ErrorIs(t, err, fs.ErrNotExist) - _, err = testEnv.TestFS.Stat(artifactPath) - assert.NoError(t, err) + for range 2 { + results, err := executeComponentTMTTests(testEnv.Env, "/work", tests, tmtRunSettings{Provision: tmtProvisionLocal}) + require.NoError(t, err) + require.Len(t, results, 1) + roots = append(roots, results[0].ArtifactDir) + } + + require.Len(t, repositories, 2) + assert.NotEqual(t, roots[0], roots[1]) + + for i, root := range roots { + assert.Equal(t, filepath.Join(root, "001-ci", "repo"), repositories[i]) + data, err := fileutils.ReadFile(testEnv.TestFS, filepath.Join(repositories[i], "metadata")) + require.NoError(t, err) + assert.Equal(t, "retained", string(data)) + } } func TestRunHostCommandOutputIncludesStderr(t *testing.T) { @@ -648,10 +687,13 @@ func TestRunHostCommandOutputIncludesStderr(t *testing.T) { return "", errors.New("command failed") } - _, err := runHostCommandOutput(testEnv.Env, "/project", "tmt", "plan", "export") + _, err := runHostCommandOutput(testEnv.Env, "/project/setup.log", testProjectDir, tmtProgram, "plan", "export") require.Error(t, err) - require.ErrorContains(t, err, "tmt export diagnostic") + + log, readErr := fileutils.ReadFile(testEnv.TestFS, "/project/setup.log") + require.NoError(t, readErr) + assert.Contains(t, string(log), "tmt export diagnostic") require.ErrorContains(t, err, "command failed") } @@ -661,9 +703,9 @@ func TestRunHostCommandOutputOmitsEmptyStderr(t *testing.T) { return "", errors.New("command failed") } - _, err := runHostCommandOutput(testEnv.Env, "/project", "tmt", "plan", "export") + _, err := runHostCommandOutput(testEnv.Env, "", testProjectDir, tmtProgram, "plan", "export") - require.EqualError(t, err, "run host command `tmt`:\ncommand failed") + require.ErrorContains(t, err, "command failed") } func TestComponentTMTArgsPreservesNativePlanSteps(t *testing.T) { @@ -677,17 +719,17 @@ func TestComponentTMTArgsPreservesNativePlanSteps(t *testing.T) { ) assert.Equal(t, []string{ - "run", "--all", "--keep", "--workdir-root", "/work/tmt", + "run", "--all", "--keep", "--id", "/work/tmt", "--workdir-root", "/work/workdir-root", "plan", "--name", "/plans/smoke", "provision", "--how", "virtual", "--become", "--image", "/path/to/image.qcow2", "--hardware", "memory >= 4 GB", "--hardware", "boot.method = uefi", "prepare", "--insert", "--how", "install", "--name", tmtCandidateRPMPrepareStepName, "--package", "/rpms/component.rpm", "--package", "/rpms/component-tests.rpm", + "report", "--insert", "--how", "junit", "--name", tmtJUnitPhase, "--no-include-output-log", }, args) assert.NotContains(t, args, "discover") assert.NotContains(t, args, "execute") - assert.NotContains(t, args, "report") assert.NotContains(t, args, "finish") } @@ -702,12 +744,13 @@ func TestComponentTMTArgsVirtualProvisioner(t *testing.T) { ) assert.Equal(t, []string{ - "run", "--all", "--keep", "--workdir-root", "/work/tmt", + "run", "--all", "--keep", "--id", "/work/tmt", "--workdir-root", "/work/workdir-root", "plan", "--name", "/plans/smoke", "provision", "--how", "virtual", "--become", "--image", "/path/to/image.qcow2", "--hardware", "boot.method = uefi", "prepare", "--insert", "--how", "install", "--name", tmtCandidateRPMPrepareStepName, "--package", "/rpms/component.rpm", + "report", "--insert", "--how", "junit", "--name", tmtJUnitPhase, "--no-include-output-log", }, args) } @@ -725,11 +768,12 @@ func TestComponentTMTArgsLocalProvisioner(t *testing.T) { ) assert.Equal(t, []string{ - "run", "--all", "--keep", "--workdir-root", "/work/tmt", + "run", "--all", "--keep", "--id", "/work/tmt", "--workdir-root", "/work/workdir-root", "plan", "--name", "/plans/smoke", "provision", "--how", "local", "--become", "prepare", "--insert", "--how", "install", "--name", tmtCandidateRPMPrepareStepName, "--package", "/rpms/component.rpm", + "report", "--insert", "--how", "junit", "--name", tmtJUnitPhase, "--no-include-output-log", }, args) assert.NotContains(t, args, "--image") assert.NotContains(t, args, "--hardware") @@ -746,11 +790,12 @@ func TestComponentTMTArgsOmitsNameWhenPlanIsEmpty(t *testing.T) { ) assert.Equal(t, []string{ - "run", "--all", "--keep", "--workdir-root", "/work/tmt", + "run", "--all", "--keep", "--id", "/work/tmt", "--workdir-root", "/work/workdir-root", "plan", "provision", "--how", "local", "--become", "prepare", "--insert", "--how", "install", "--name", tmtCandidateRPMPrepareStepName, "--package", "/rpms/component.rpm", + "report", "--insert", "--how", "junit", "--name", tmtJUnitPhase, "--no-include-output-log", }, args) } diff --git a/internal/tmt/runner.go b/internal/tmt/runner.go new file mode 100644 index 000000000..7f21a3a1e --- /dev/null +++ b/internal/tmt/runner.go @@ -0,0 +1,262 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package tmt + +import ( + "errors" + "fmt" + "io" + iofs "io/fs" + "log/slog" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev" + "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx" + "github.com/microsoft/azure-linux-dev-tools/internal/projectconfig" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/spf13/afero" +) + +const ( + tmtJUnitPhase = "azldev-junit" + tmtPipeWaitDelay = 2 * time.Second +) + +func executeComponentTMTTests( + env *azldev.Env, workDir string, tests []projectconfig.ResolvedTest, settings tmtRunSettings, +) ([]azldev.TestRunResult, error) { + workDir, err := componentTMTWorkDir(env, workDir) + if err != nil { + return nil, err + } + + root := filepath.Join(workDir, "") + if !env.DryRun() { + if err := env.FS().MkdirAll(workDir, fileperms.PrivateDir); err != nil { + return nil, fmt.Errorf("create TMT work directory:\n%w", err) + } + + root, err = fileutils.MkdirTemp(env.FS(), workDir, "azldev-test-") + if err != nil { + return nil, fmt.Errorf("create test artifact directory:\n%w", err) + } + + slog.Info("Retaining test artifacts", "path", root) + settings.SetupLog = filepath.Join(root, "setup.log") + } + + results := make([]azldev.TestRunResult, 0, len(tests)) + for index, test := range tests { + result := azldev.TestRunResult{Test: test.Name, JUnit: []string{}} + if !env.DryRun() { + result.ArtifactDir = root + } + + results = append(results, result) + + if index == 0 { + _, settings.TMTProgramPath, err = prepareTMTEnvironment( + env, workDir, settings.Provision, settings.SourceDir != "", settings.SetupLog, + ) + if err != nil { + return results, err + } + } + + settings.WorkDir = filepath.Join(root, fmt.Sprintf("%03d-%s", index+1, test.Name)) + + runErr := runOneTMTTest(env, test, settings, &results[index]) + if !env.DryRun() { + results[index].JUnit = collectTMTJUnit(env.FS(), root, filepath.Join(settings.WorkDir, "native")) + } + + if runErr != nil { + return results, fmt.Errorf("TMT test %#q:\n%w", test.Name, runErr) + } + } + + return results, nil +} + +func collectTMTJUnit(fs opctx.FS, root string, native string) []string { + reports := []string{} + + info, err := lstatIfPossible(fs, native) + if errors.Is(err, os.ErrNotExist) { + return reports + } + + if err != nil { + slog.Warn("Unable to inspect TMT artifacts", "path", native, "error", err) + + return reports + } + + if !info.IsDir() { + slog.Warn("Skipping TMT artifacts: not a directory", "path", native) + + return reports + } + + nativeFS := afero.NewIOFS(afero.NewBasePathFs(fs, native)) + + err = iofs.WalkDir(nativeFS, ".", func(path string, entry iofs.DirEntry, err error) error { + if err != nil { + slog.Warn("Skipping unreadable TMT artifacts", "path", filepath.Join(native, path), "error", err) + + return nil + } + + phaseDir := filepath.Dir(path) + if entry.Name() == "junit.xml" && filepath.Base(phaseDir) == tmtJUnitPhase && + filepath.Base(filepath.Dir(phaseDir)) == "report" { + rel, err := retainedTMTReport(fs, root, filepath.Join(native, path)) + if err != nil { + slog.Warn("Skipping invalid TMT report", "path", filepath.Join(native, path), "error", err) + } else { + reports = append(reports, rel) + } + } + + return nil + }) + if err != nil { + slog.Warn("Unable to finish TMT report discovery", "path", native, "error", err) + } + + return reports +} + +func lstatIfPossible(fs opctx.FS, path string) (os.FileInfo, error) { + if lstater, ok := fs.(afero.Lstater); ok { + info, _, err := lstater.LstatIfPossible(path) + + return info, err //nolint:wrapcheck // The caller adds report context. + } + + return fs.Stat(path) //nolint:wrapcheck // The caller adds report context. +} + +func retainedTMTReport(fs opctx.FS, root string, path string) (string, error) { + rel, err := filepath.Rel(root, path) + if err != nil || !filepath.IsLocal(rel) || rel == "." { + return "", fmt.Errorf("report %#q escapes artifact directory %#q", path, root) + } + + current := root + for _, part := range append([]string{"."}, strings.Split(rel, string(filepath.Separator))...) { + current = filepath.Join(current, part) + + info, err := lstatIfPossible(fs, current) + if err != nil { + return "", fmt.Errorf("inspect report %#q:\n%w", current, err) + } + + if info.Mode()&os.ModeSymlink != 0 { + return "", fmt.Errorf("report %#q contains a symlink", path) + } + + if current == path && (!info.Mode().IsRegular() || info.Size() == 0) { + return "", fmt.Errorf("report %#q is not a nonempty regular file", path) + } + } + + return filepath.ToSlash(rel), nil +} + +// stdout and stderr may copy concurrently into the same transcript. +type testTranscript struct { + mu sync.Mutex + writer io.Writer + err error +} + +func (w *testTranscript) Write(data []byte) (int, error) { + w.mu.Lock() + defer w.mu.Unlock() + + count, err := w.writer.Write(data) + if err == nil && count != len(data) { + err = io.ErrShortWrite + } + + w.err = errors.Join(w.err, err) + + return count, err +} + +//nolint:wrapcheck // The deferred wrapper adds command/transcript context. +func runCapturedTMTCommand( + env *azldev.Env, command *exec.Cmd, logPath string, captureOutput bool, +) (output string, exitCode *int, err error) { + if env.DryRun() { + slog.Info("Would run", "command", command.Args) + + return "", nil, nil + } + + // Descendants may retain the pipes after cancellation kills their parent. + command.WaitDelay = tmtPipeWaitDelay + transcript := io.Discard + + if logPath != "" { + file, openErr := env.FS().OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, fileperms.PrivateFile) + if openErr != nil { + return "", nil, fmt.Errorf("open command transcript %#q:\n%w", logPath, openErr) + } + + defer func() { err = errors.Join(err, file.Close()) }() + + transcript = file + } + + log := &testTranscript{writer: transcript} + + defer func() { + err = errors.Join(err, log.err) + if err != nil { + err = fmt.Errorf("command %q (transcript %#q):\n%w", command.Args, logPath, err) + } + }() + + if _, err = fmt.Fprintf(log, "$ %q\n", command.Args); err != nil { + return "", nil, err + } + + stdout := env.ReportFile() + if env.DefaultReportFormat() == azldev.ReportFormatJSON || env.DefaultReportFormat() == azldev.ReportFormatCSV { + stdout = os.Stderr + } + + command.Stderr = io.MultiWriter(log, os.Stderr) + if !captureOutput { + command.Stdout = io.MultiWriter(log, stdout) + } + + wrapped, err := env.Command(command) + if err != nil { + return "", nil, err + } + + if captureOutput { + output, err = wrapped.RunAndGetOutput(env) + _, writeErr := io.WriteString(log, output) + err = errors.Join(err, writeErr) + } else { + err = wrapped.Run(env) + } + + if command.ProcessState != nil && command.ProcessState.Exited() { + code := command.ProcessState.ExitCode() + exitCode = &code + } + + return output, exitCode, err +} diff --git a/internal/tmt/runner_internal_test.go b/internal/tmt/runner_internal_test.go new file mode 100644 index 000000000..0b6aa2980 --- /dev/null +++ b/internal/tmt/runner_internal_test.go @@ -0,0 +1,135 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package tmt + +import ( + "io" + "log/slog" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/microsoft/azure-linux-dev-tools/internal/app/azldev/core/testutils" + "github.com/microsoft/azure-linux-dev-tools/internal/global/opctx" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileutils" + "github.com/spf13/afero" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type unreadableArtifactFS struct { + opctx.FS + blocked string + err error +} + +func (fs unreadableArtifactFS) Open(path string) (afero.File, error) { + if path == fs.blocked { + return nil, &os.PathError{Op: "open", Path: path, Err: fs.err} + } + + return fs.FS.Open(path) +} + +func (fs unreadableArtifactFS) Stat(path string) (os.FileInfo, error) { + if strings.HasPrefix(path, fs.blocked+string(filepath.Separator)) { + return nil, &os.PathError{Op: "stat", Path: path, Err: fs.err} + } + + return fs.FS.Stat(path) +} + +func TestCollectTMTJUnitUnreadableDirectories(t *testing.T) { + const ( + root = "/owned" + native = "/owned/native" + ) + // Directory names here are legitimate plan hierarchy, not lifecycle steps. + goodReport := "native/execute/data/report/nested/report/azldev-junit/junit.xml" + otherReport := "native/other/report/azldev-junit/junit.xml" + + for _, testCase := range []struct { + name, blocked string + readErr error + want []string + }{ + {"private test data", "other/execute/data/private", os.ErrPermission, []string{goodReport, otherReport}}, + { + "private payload named report", "other/execute/data/report", os.ErrPermission, + []string{goodReport, otherReport}, + }, + {"private report directory", "other/report", os.ErrPermission, []string{goodReport}}, + {"private reporter directory", "other/report/azldev-junit", os.ErrPermission, []string{goodReport}}, + {"private run directory", "", os.ErrPermission, []string{}}, + {"payload IO failure", "other/execute/data/private", io.ErrUnexpectedEOF, []string{goodReport, otherReport}}, + } { + t.Run(testCase.name, func(t *testing.T) { + env := testutils.NewTestEnv(t) + + var warnings strings.Builder + + prev := slog.Default() + slog.SetDefault(slog.New(slog.NewTextHandler(&warnings, nil))) + t.Cleanup(func() { slog.SetDefault(prev) }) + + for _, path := range []string{goodReport, otherReport} { + require.NoError(t, fileutils.WriteFile(env.TestFS, filepath.Join(root, path), + []byte(""), fileperms.PrivateFile)) + } + + blocked := filepath.Join(native, testCase.blocked) + require.NoError(t, env.TestFS.MkdirAll(blocked, fileperms.PrivateDir)) + + reports := collectTMTJUnit(unreadableArtifactFS{ + FS: env.TestFS, blocked: blocked, err: testCase.readErr, + }, root, native) + + assert.Equal(t, testCase.want, reports) + assert.Contains(t, warnings.String(), "level=WARN") + assert.Contains(t, warnings.String(), blocked) + assert.Contains(t, warnings.String(), testCase.readErr.Error()) + }) + } +} + +type reportSymlinkFS struct { + opctx.FS + link string +} + +type reportSymlinkInfo struct{ os.FileInfo } + +func (reportSymlinkInfo) Mode() os.FileMode { return os.ModeSymlink | fileperms.PublicFile } + +func (fs reportSymlinkFS) LstatIfPossible(path string) (os.FileInfo, bool, error) { + info, err := fs.Stat(path) + if err == nil && path == fs.link { + info = reportSymlinkInfo{info} + } + + return info, true, err +} + +func TestRetainedTMTReportBoundaries(t *testing.T) { + env := testutils.NewTestEnv(t) + require.NoError(t, fileutils.WriteFile( + env.TestFS, "/owned/native/junit.xml", []byte(""), fileperms.PrivateFile, + )) + + for _, link := range []string{"/owned", "/owned/native", "/owned/native/junit.xml"} { + _, err := retainedTMTReport(reportSymlinkFS{FS: env.TestFS, link: link}, "/owned", "/owned/native/junit.xml") + require.ErrorContains(t, err, "symlink") + } + + for _, path := range []string{"/outside", "/owned/../outside", "/owned/native", "/owned"} { + _, err := retainedTMTReport(env.TestFS, "/owned", path) + require.Error(t, err) + } + + rel, err := retainedTMTReport(env.TestFS, "/owned", "/owned/native/junit.xml") + require.NoError(t, err) + assert.Equal(t, "native/junit.xml", rel) +} diff --git a/magefiles/magescenario/magescenario.go b/magefiles/magescenario/magescenario.go index 892a38625..2f2e120e8 100644 --- a/magefiles/magescenario/magescenario.go +++ b/magefiles/magescenario/magescenario.go @@ -96,12 +96,9 @@ func E2E(target *string) error { options.testSelector = "^(TestAzureLinuxComponentUpdateIsIdempotent|" + "TestAzureLinuxComponentRenderSubsetIsIdempotent)$" case e2eTargetTMT: - options.label = "virtual TMT end-to-end" + options.label = "TMT end-to-end" options.timeout = "45m" - options.testSelector = "^TestTMTVirtualProvisioning$" - options.additionalEnv = map[string]string{ - "TESTING_ENABLE_TMT_VIRTUAL_E2E": "1", - } + options.testSelector = "^TestTMT(LocalReporting|VirtualProvisioning)$" default: return fmt.Errorf( "%w: unknown end-to-end target %#q; available targets: %q", @@ -172,8 +169,6 @@ type scenarioGoTestOptions struct { // testSelector, when set, is passed to 'go test -run' to execute one // focused test without running the rest of the selected tier. testSelector string - // additionalEnv adds target-specific environment variables to the test process. - additionalEnv map[string]string } func runScenarioGoTest(options scenarioGoTestOptions) error { @@ -192,10 +187,6 @@ func runScenarioGoTest(options scenarioGoTestOptions) error { env := map[string]string{ buildtestenv.TestingAzldevBinPathEnvVar: path.Join(mageutil.BinDir(), "azldev"), } - for name, value := range options.additionalEnv { - env[name] = value - } - if options.doSnapshotUpdate { env[buildtestenv.TestingUpdateSnapshotsEnvVar] = buildtestenv.TestingUpdateSnapshotsEnvValue } diff --git a/scenario/testdata/tmt-local/.fmf/version b/scenario/testdata/tmt-local/.fmf/version new file mode 100644 index 000000000..d00491fd7 --- /dev/null +++ b/scenario/testdata/tmt-local/.fmf/version @@ -0,0 +1 @@ +1 diff --git a/scenario/testdata/tmt-local/plans.fmf b/scenario/testdata/tmt-local/plans.fmf new file mode 100644 index 000000000..83eac25ff --- /dev/null +++ b/scenario/testdata/tmt-local/plans.fmf @@ -0,0 +1,51 @@ +discover: + how: fmf +execute: + how: tmt +report: + - how: display + - how: junit + name: supplied-junit + include-output-log: false +/pass: + discover+: + test: ^/tests/probe$ +/private-data: + discover+: + test: ^/tests/private-data$ +/mixed: + discover+: + test: ^/tests/(probe|fail|skip|info|warn|error)$ +/skip: + discover+: + test: ^/tests/skip$ +/fail: + discover+: + test: ^/tests/fail$ +/warn: + discover+: + test: ^/tests/warn$ +/with.dots: + discover+: + test: ^/tests/probe$ +/empty: + discover+: + test: ^/tests/does-not-exist$ +/prepare-failure: + discover+: + test: ^/tests/probe$ + prepare: + how: shell + script: echo controlled-prepare-failure >&2; exit 1 +/finish-failure: + discover+: + test: ^/tests/probe$ + finish: + how: shell + script: echo controlled-finish-failure >&2; exit 1 +/report-failure: + discover+: + test: ^/tests/probe$ + report: + how: junit + file: /nonexistent-parent/junit.xml diff --git a/scenario/testdata/tmt-local/run.sh b/scenario/testdata/tmt-local/run.sh new file mode 100644 index 000000000..b5b03b371 --- /dev/null +++ b/scenario/testdata/tmt-local/run.sh @@ -0,0 +1,103 @@ +#!/bin/bash +set -euo pipefail + +cd /workdir/project +find /workdir/fixture /workdir/root-fixture -type f -print0 | sort -z | xargs -0 sha256sum >/artifacts/source-before +if rpm -q jq >/dev/null 2>&1; then + echo "jq must be absent before TMT installs the declared dependency" >&2 + exit 1 +fi + +for format in json table; do + azldev --no-default-config --dry-run component test azldev-tmt-probe \ + --provision local --source-dir /workdir/fixture --work-dir /artifacts/dry-run \ + --plan /plans/pass --rpm /opt/azldev-tmt-fixtures/candidate.rpm -O "${format}" \ + >"/artifacts/dry-${format}.out" 2>"/artifacts/dry-${format}.stderr" + grep -q 'Would run' "/artifacts/dry-${format}.stderr" + test ! -e /artifacts/dry-run +done + +for case in pass private-data mixed skip fail warn with.dots empty prepare-failure finish-failure report-failure multi pass-empty root; do + source=/workdir/fixture + plan="/plans/${case}$" + case "${case}" in + multi) plan='/plans/(pass|skip)$' ;; + pass-empty) plan='/plans/(pass|empty)$' ;; + root) source=/workdir/root-fixture; plan='/$' ;; + esac + set +e + TMT_STATE_FORMAT=json azldev --no-default-config component test azldev-tmt-probe \ + --provision local --source-dir "${source}" --work-dir /artifacts/work \ + --plan "${plan}" --rpm /opt/azldev-tmt-fixtures/candidate.rpm -O json \ + >"/artifacts/${case}.json" 2>"/artifacts/${case}.stderr" + status=$? + set -e + printf '%s\n' "${status}" >"/artifacts/${case}.exit" +done + +# A surviving descendant holds both capture pipes after its parent exits or is killed. +python=/artifacts/work/tmt/venv/bin/python3 +mv /artifacts/work/tmt/venv/bin/tmt /artifacts/work/tmt/venv/bin/tmt-original +cat >/artifacts/work/tmt/venv/bin/tmt <<'SH' +#!/bin/bash +set -eu +while [[ "$1" != --id ]]; do shift; done +exec /artifacts/work/tmt/venv/bin/python3 - "$2" <<'PY' +import os +import subprocess +import sys +import time +from pathlib import Path + +native = Path(sys.argv[1]) +native.mkdir(parents=True) +(native / "diagnostic.txt").write_text("persisted evidence") +print("runner stdout", flush=True) +print("runner stderr", file=sys.stderr, flush=True) +child = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(60)"]) +Path(os.environ["AZL_TMT_READY"]).write_text(str(child.pid)) +if not os.environ.get("AZL_TMT_EXIT"): + time.sleep(60) +PY +SH +chmod 0755 /artifacts/work/tmt/venv/bin/tmt +"${python}" - <<'PY' +import os +import signal +import subprocess +import time +from pathlib import Path + +for name, sig in (("sigint", signal.SIGINT), ("sigterm", signal.SIGTERM), ("pipe-timeout", None)): + ready = Path(f"/artifacts/{name}.ready") + with Path(f"/artifacts/{name}.json").open("w") as stdout, Path(f"/artifacts/{name}.stderr").open("w") as stderr: + process = subprocess.Popen( + ["azldev", "--no-default-config", "component", "test", "azldev-tmt-probe", + "--provision", "local", "--source-dir", "/workdir/fixture", + "--work-dir", "/artifacts/work", "--plan", "/plans/pass", + "--rpm", "/opt/azldev-tmt-fixtures/candidate.rpm", "-O", "json"], + stdout=stdout, stderr=stderr, + env={**os.environ, "AZL_TMT_READY": str(ready), "AZL_TMT_EXIT": "1" if sig is None else ""}, + ) + try: + deadline = time.monotonic() + 30 + while not ready.exists(): + if process.poll() is not None or time.monotonic() >= deadline: + raise RuntimeError(f"{name}: runner did not become ready") + time.sleep(0.05) + if sig is not None: + process.send_signal(sig) + status = process.wait(timeout=10) + Path(f"/artifacts/{name}.exit").write_text(f"{status}\n") + finally: + if process.poll() is None: + process.kill() + process.wait(timeout=10) + if ready.exists(): + os.kill(int(ready.read_text()), signal.SIGKILL) +PY +mv /artifacts/work/tmt/venv/bin/tmt-original /artifacts/work/tmt/venv/bin/tmt + +find /workdir/fixture /workdir/root-fixture -type f -print0 | sort -z | xargs -0 sha256sum >/artifacts/source-after +cmp /artifacts/source-before /artifacts/source-after +rpm -q jq diff --git a/scenario/testdata/tmt-local/tests.fmf b/scenario/testdata/tmt-local/tests.fmf new file mode 100644 index 000000000..d8a6cf17a --- /dev/null +++ b/scenario/testdata/tmt-local/tests.fmf @@ -0,0 +1,39 @@ +/probe: + require: + - jq + test: | + set -eu + echo noisy-probe-stdout + echo noisy-probe-stderr >&2 + test "$(id -u)" = 0 + expected="$(rpm -qp --qf '%{NAME}-%{VERSION}-%{RELEASE}.%{ARCH}' /opt/azldev-tmt-fixtures/candidate.rpm)" + test "$(rpm -q azldev-tmt-probe)" = "${expected}" + test "$(rpm -qf /usr/bin/azldev-tmt-probe)" = "${expected}" + jq -ne '1 + 1 == 2' + printf 'probe round trip\n' >"${TMT_TEST_DATA}/input" + azldev-tmt-probe "${TMT_TEST_DATA}/input" "${TMT_TEST_DATA}/output" + grep -qx 'PROBE ROUND TRIP' "${TMT_TEST_DATA}/output" +/fail: + test: echo controlled-assertion-failure; exit 1 +/private-data: + test: | + set -eu + test "$(id -u)" = 0 + mkdir -m 0700 "${TMT_TEST_DATA}/private" "${TMT_TEST_DATA}/report" + echo private-payload >"${TMT_TEST_DATA}/private/output.txt" +/skip: + result: custom + test: | + printf '[{"name": "/", "result": "skip"}]\n' >"${TMT_TEST_DATA}/results.json" +/info: + result: custom + test: | + printf '[{"name": "/", "result": "info"}]\n' >"${TMT_TEST_DATA}/results.json" +/warn: + result: custom + test: | + printf '[{"name": "/", "result": "warn"}]\n' >"${TMT_TEST_DATA}/results.json" +/error: + result: custom + test: | + printf '[{"name": "/", "result": "error"}]\n' >"${TMT_TEST_DATA}/results.json" diff --git a/scenario/testdata/tmt-root/.fmf/version b/scenario/testdata/tmt-root/.fmf/version new file mode 100644 index 000000000..d00491fd7 --- /dev/null +++ b/scenario/testdata/tmt-root/.fmf/version @@ -0,0 +1 @@ +1 diff --git a/scenario/testdata/tmt-root/main.fmf b/scenario/testdata/tmt-root/main.fmf new file mode 100644 index 000000000..4d373b375 --- /dev/null +++ b/scenario/testdata/tmt-root/main.fmf @@ -0,0 +1,7 @@ +discover: + how: shell + tests: + - name: smoke + test: echo root-plan-passed +execute: + how: tmt diff --git a/scenario/tmt_local_e2e_test.go b/scenario/tmt_local_e2e_test.go new file mode 100644 index 000000000..608d6711f --- /dev/null +++ b/scenario/tmt_local_e2e_test.go @@ -0,0 +1,190 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//go:build e2e && linux + +package scenario_tests + +import ( + "bytes" + "encoding/json" + "encoding/xml" + "io" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + "github.com/microsoft/azure-linux-dev-tools/internal/utils/fileperms" + "github.com/microsoft/azure-linux-dev-tools/scenario/internal/cmdtest" + "github.com/microsoft/azure-linux-dev-tools/scenario/internal/containertest" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type tmtArtifactRow struct { + Test string `json:"test"` + ExitCode *int `json:"exitCode"` + ArtifactDir string `json:"artifactDir"` + JUnit []string `json:"junit"` +} + +func TestTMTLocalReporting(t *testing.T) { + if testing.Short() { + t.Skip("skipping long test") + } + base, err := filepath.Abs(filepath.Join("..", "out", "tmt-local-e2e")) + require.NoError(t, err) + require.NoError(t, os.MkdirAll(base, fileperms.PrivateDir)) + // Keep retained private payloads out of Go's recursive package discovery. + artifacts, err := os.MkdirTemp(base, "_run-") + require.NoError(t, err) + t.Logf("Retained local TMT artifacts: %s", artifacts) + + test := cmdtest.NewScenarioTest("fixture/run.sh"). + WithCustomCmd("bash"). + WithTimeout(20 * time.Minute). + InContainer(). + WithDockerfile("Dockerfile.tmt"). + WithInit(). + WithNetwork(). + WithExtraMounts([]containertest.ContainerMount{ + containertest.NewContainerMount(artifacts, "/artifacts", nil), + }) + test.AddDirRecursive(t, "fixture", filepath.Join("testdata", "tmt-local")) + test.AddDirRecursive(t, "root-fixture", filepath.Join("testdata", "tmt-root")) + test.AddSingleFile("project/azldev.toml", filepath.Join("testdata", "tmt-virtual", "azldev.toml")) + test.AddSingleFile("project/azldev-tmt-probe.spec", filepath.Join("testdata", "tmt-virtual", "azldev-tmt-probe.spec")) + results, err := test.Run(t) + require.NoError(t, os.WriteFile(filepath.Join(artifacts, "container.stdout"), []byte(results.Stdout), fileperms.PrivateFile)) + require.NoError(t, os.WriteFile(filepath.Join(artifacts, "container.stderr"), []byte(results.Stderr), fileperms.PrivateFile)) + require.NoError(t, err) + results.AssertZeroExitCode(t) + + roots := map[string]bool{} + for _, tc := range []struct { + name string + azldevExit, tmtExit, reports int + pass, fail, skip, errors int + }{ + {"pass", 0, 0, 1, 1, 0, 0, 0}, + {"private-data", 0, 0, 1, 1, 0, 0, 0}, + {"mixed", 1, 2, 1, 1, 1, 2, 2}, + {"skip", 1, 4, 1, 0, 0, 1, 0}, + {"fail", 1, 1, 1, 0, 1, 0, 0}, + {"warn", 1, 1, 1, 0, 0, 0, 1}, + {"with.dots", 0, 0, 1, 1, 0, 0, 0}, + {"root", 0, 0, 1, 1, 0, 0, 0}, + {"empty", 1, 3, 0, 0, 0, 0, 0}, + {"prepare-failure", 1, 2, 1, 0, 0, 0, 1}, + {"finish-failure", 1, 2, 1, 1, 0, 0, 0}, + {"report-failure", 1, 2, 0, 0, 0, 0, 0}, + {"multi", 0, 0, 2, 1, 0, 1, 0}, + {"pass-empty", 0, 0, 1, 1, 0, 0, 0}, + {"sigint", 1, -1, 0, 0, 0, 0, 0}, + {"sigterm", 1, -1, 0, 0, 0, 0, 0}, + {"pipe-timeout", 1, 0, 0, 0, 0, 0, 0}, + } { + t.Run(tc.name, func(t *testing.T) { + data, err := os.ReadFile(filepath.Join(artifacts, tc.name+".json")) + require.NoError(t, err) + var rows []tmtArtifactRow + decoder := json.NewDecoder(bytes.NewReader(data)) + require.NoError(t, decoder.Decode(&rows), "stdout: %s", data) + require.ErrorIs(t, decoder.Decode(new(any)), io.EOF) + require.Len(t, rows, 1) + row := rows[0] + require.Equal(t, "azldev-tmt-probe", row.Test) + exit, err := os.ReadFile(filepath.Join(artifacts, tc.name+".exit")) + require.NoError(t, err) + assert.Equal(t, strconv.Itoa(tc.azldevExit)+"\n", string(exit)) + if tc.tmtExit < 0 { + assert.Nil(t, row.ExitCode) + } else { + require.NotNil(t, row.ExitCode) + assert.Equal(t, tc.tmtExit, *row.ExitCode) + } + assert.Len(t, row.JUnit, tc.reports, "%s", data) + assert.False(t, roots[row.ArtifactDir], "work-directory reuse must not reuse a run") + roots[row.ArtifactDir] = true + rel, err := filepath.Rel("/artifacts", row.ArtifactDir) + require.NoError(t, err) + require.True(t, filepath.IsLocal(rel)) + root := filepath.Join(artifacts, rel) + counts := [4]int{} + for _, report := range row.JUnit { + require.True(t, filepath.IsLocal(report)) + assert.Contains(t, report, "/report/azldev-junit/junit.xml") + testCounts := localTMTJUnitCounts(t, filepath.Join(root, report)) + for index, count := range testCounts { + counts[index] += count + } + } + assert.Equal(t, [4]int{tc.pass, tc.fail, tc.skip, tc.errors}, counts) + runnerLog := filepath.Join(root, "001-azldev-tmt-probe", "runner.log") + log, err := os.ReadFile(runnerLog) + require.NoError(t, err) + if tc.name == "private-data" { + privateDirs, globErr := filepath.Glob(filepath.Join(root, + "*/native/plans/private-data/execute/data/guest/*/tests/private-data-*/data/private")) + require.NoError(t, globErr) + require.Len(t, privateDirs, 1) + _, readErr := os.ReadDir(privateDirs[0]) + require.ErrorIs(t, readErr, os.ErrPermission, "private payload must remain unreadable") + } + if tc.name == "pass" { + assert.Contains(t, string(log), "azldev-junit") + testLogs, globErr := filepath.Glob(filepath.Join(root, + "*/native/plans/pass/execute/data/guest/*/tests/probe-*/output.txt")) + require.NoError(t, globErr) + require.Len(t, testLogs, 1) + testOutput, readErr := os.ReadFile(testLogs[0]) + require.NoError(t, readErr) + assert.Contains(t, string(testOutput), "noisy-probe-stdout") + assert.Contains(t, string(testOutput), "noisy-probe-stderr") + copied := t.TempDir() + require.NoError(t, os.CopyFS(copied, os.DirFS(root))) + assert.Equal(t, counts, localTMTJUnitCounts(t, filepath.Join(copied, row.JUnit[0]))) + supplied, globErr := filepath.Glob(filepath.Join(root, "*/native/plans/pass/report/supplied-junit/junit.xml")) + require.NoError(t, globErr) + require.Len(t, supplied, 1, "existing reporter must remain") + } + if strings.HasPrefix(tc.name, "sig") || tc.name == "pipe-timeout" { + assert.Contains(t, string(log), "runner stderr") + require.FileExists(t, filepath.Join(root, "001-azldev-tmt-probe/native/diagnostic.txt")) + } + }) + } +} + +func localTMTJUnitCounts(t *testing.T, path string) [4]int { + t.Helper() + data, err := os.ReadFile(path) + require.NoError(t, err) + var document struct { + XMLName xml.Name `xml:"testsuites"` + Cases []struct { + Failure *struct{} `xml:"failure"` + Error *struct{} `xml:"error"` + Skipped *struct{} `xml:"skipped"` + } `xml:"testsuite>testcase"` + } + require.NoError(t, xml.Unmarshal(data, &document)) + counts := [4]int{} + for _, result := range document.Cases { + switch { + case result.Failure != nil: + counts[1]++ + case result.Error != nil: + counts[3]++ + case result.Skipped != nil: + counts[2]++ + default: + counts[0]++ + } + } + + return counts +} diff --git a/scenario/tmt_virtual_e2e_test.go b/scenario/tmt_virtual_e2e_test.go index 4aab109c6..1dcd76776 100644 --- a/scenario/tmt_virtual_e2e_test.go +++ b/scenario/tmt_virtual_e2e_test.go @@ -25,9 +25,21 @@ func TestTMTVirtualProvisioning(t *testing.T) { t.Skip("skipping long test") } if os.Getenv(tmtEnableEnv) == "" { - t.Skip("temporarily disabled; run 'mage e2e -target=tmt' to execute the virtual TMT regression") + t.Skip("quarantined; set TESTING_ENABLE_TMT_VIRTUAL_E2E=1 to reproduce the virtual TMT regression") } + kvm, err := os.OpenFile("/dev/kvm", os.O_RDWR, 0) + if os.IsNotExist(err) { + t.Skip("virtual TMT requires /dev/kvm") + } + require.NoError(t, err, "open /dev/kvm read-write as the scenario runner user") + require.NoError(t, kvm.Close()) + + kvmInfo, err := os.Stat("/dev/kvm") + require.NoError(t, err) + kvmStat, ok := kvmInfo.Sys().(*syscall.Stat_t) + require.True(t, ok, "read /dev/kvm ownership") + inputs, externalInputs, err := loadTMTVirtualInputs(os.Getenv) require.NoError(t, err) if !externalInputs { @@ -41,15 +53,6 @@ func TestTMTVirtualProvisioning(t *testing.T) { rpmContainerPath = "/inputs/candidate.rpm" } - kvm, err := os.OpenFile("/dev/kvm", os.O_RDWR, 0) - require.NoError(t, err, "open /dev/kvm read-write as the scenario runner user") - require.NoError(t, kvm.Close()) - - kvmInfo, err := os.Stat("/dev/kvm") - require.NoError(t, err) - kvmStat, ok := kvmInfo.Sys().(*syscall.Stat_t) - require.True(t, ok, "read /dev/kvm ownership") - runID := fmt.Sprintf("%d", time.Now().UTC().UnixNano()) artifactBase, err := filepath.Abs(filepath.Join("..", "out", "tmt-virtual-e2e")) require.NoError(t, err)