diff --git a/api/kptfile/v1/validation.go b/api/kptfile/v1/validation.go index dbbf0410af..17006e1ae8 100644 --- a/api/kptfile/v1/validation.go +++ b/api/kptfile/v1/validation.go @@ -413,8 +413,11 @@ func AreKRM(nodes []*yaml.RNode) error { // IsKRM validates if given resource is a valid KRM resource by ensuring // that resource has a valid apiVersion, kind and metadata.name field. // It excludes kustomization resource from KRM check. +// It validates function configs, which always need a name. The CLI's +// pipeline-input classifier (pkg/lib/krmvalidate.IsKRM) is deliberately +// more lenient about names and must not be merged with this one. func IsKRM(n *yaml.RNode) error { - if isKustomization(n) { + if IsKustomization(n) { // exclude kustomization files from KRM check // https://github.com/kptdev/kpt/issues/2388 return nil @@ -435,8 +438,8 @@ func IsKRM(n *yaml.RNode) error { return nil } -// isKustomization determines if given YAML is a kustomization file or resource. -func isKustomization(n *yaml.RNode) bool { +// IsKustomization determines if given YAML is a kustomization file or resource. +func IsKustomization(n *yaml.RNode) bool { resourcePath, _, err := kioutil.GetFileAnnotations(n) if err == nil { // perform the check only if we are able to reliably diff --git a/api/kptfile/v1/validation_test.go b/api/kptfile/v1/validation_test.go index c6856e141c..b3f6cccf49 100644 --- a/api/kptfile/v1/validation_test.go +++ b/api/kptfile/v1/validation_test.go @@ -401,7 +401,7 @@ kind: Kustomization } for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { - got := isKustomization(yaml.MustParse(tc.input)) + got := IsKustomization(yaml.MustParse(tc.input)) if got != tc.exp { t.Fatalf("got %v expected %v", got, tc.exp) } diff --git a/cli/alpha/live/plan/command.go b/cli/alpha/live/plan/command.go index 91dda2ab9e..ffbde0baa6 100644 --- a/cli/alpha/live/plan/command.go +++ b/cli/alpha/live/plan/command.go @@ -120,7 +120,7 @@ func (r *Runner) RunE(c *cobra.Command, args []string) error { // Load the resources from disk or stdin and extract the // inventory information. - objs, inv, err := live.Load(r.factory, path, c.InOrStdin()) + objs, inv, err := live.Load(r.ctx, r.factory, path, c.InOrStdin()) if err != nil { return err } diff --git a/cli/fn/cmdeval/runfn.go b/cli/fn/cmdeval/runfn.go index 73635c65af..efc90f3ad3 100644 --- a/cli/fn/cmdeval/runfn.go +++ b/cli/fn/cmdeval/runfn.go @@ -37,6 +37,7 @@ import ( "sigs.k8s.io/kustomize/kyaml/kio/kioutil" "sigs.k8s.io/kustomize/kyaml/yaml" + "github.com/kptdev/kpt/pkg/lib/krmvalidate" "github.com/kptdev/kpt/pkg/lib/pkg" ) @@ -144,6 +145,14 @@ func (r RunFns) getNodesAndFilters() ( var outputPkg *kio.LocalPackageReadWriter if r.Path != "" { + // Non-KRM pre-scan, same as `kpt fn render`'s, except it skips + // subpackages a parent .krmignore excludes, like this reader does. + pr := printer.FromContextOrDie(r.Ctx) + scanResult, err := krmvalidate.ScanHonoringKrmignore(filesys.FileSystemOrOnDisk{}, string(r.uniquePath), pr.Printf) + if err != nil { + return nil, nil, nil, err + } + outputPkg = &kio.LocalPackageReadWriter{ PackagePath: string(r.uniquePath), MatchFilesGlob: pkg.MatchAllKRM, @@ -151,6 +160,7 @@ func (r RunFns) getNodesAndFilters() ( PackageFileName: kptfilev1.KptFileName, IncludeSubpackages: true, WrapBareSeqNode: true, + FileSkipFunc: scanResult.SkipFunc(string(r.uniquePath)), } } diff --git a/cli/fn/cmdeval/runfn_test.go b/cli/fn/cmdeval/runfn_test.go index 8240401d83..fe55cb086c 100644 --- a/cli/fn/cmdeval/runfn_test.go +++ b/cli/fn/cmdeval/runfn_test.go @@ -210,9 +210,17 @@ func TestCmd_Execute_notIncludeMetaResources(t *testing.T) { dir := setupTest(t) defer os.RemoveAll(dir) - // write a test filter to the directory of configuration + // Named (unlike the shared ValueReplacerYAMLData constant) so it's + // valid KRM and the non-KRM pre-scan doesn't block this test. + namedValueReplacerYAMLData := `apiVersion: v1 +kind: ValueReplacer +metadata: + name: filter +stringMatch: Deployment +replace: StatefulSet +` if !assert.NoError(t, os.WriteFile( - filepath.Join(dir, "filter.yaml"), []byte(ValueReplacerYAMLData), 0600)) { + filepath.Join(dir, "filter.yaml"), []byte(namedValueReplacerYAMLData), 0600)) { return } diff --git a/cli/live/apply/cmdapply.go b/cli/live/apply/cmdapply.go index f8ca62f2ab..df46c41c9c 100644 --- a/cli/live/apply/cmdapply.go +++ b/cli/live/apply/cmdapply.go @@ -190,7 +190,7 @@ func (r *Runner) runE(c *cobra.Command, args []string) error { } } - objs, inv, err := live.Load(r.factory, path, c.InOrStdin()) + objs, inv, err := live.Load(r.ctx, r.factory, path, c.InOrStdin()) if err != nil { return err } diff --git a/cli/live/destroy/cmddestroy.go b/cli/live/destroy/cmddestroy.go index ace69689ec..39d0d7579e 100644 --- a/cli/live/destroy/cmddestroy.go +++ b/cli/live/destroy/cmddestroy.go @@ -156,7 +156,7 @@ func (r *Runner) runE(c *cobra.Command, args []string) error { } } - _, inv, err := live.Load(r.factory, path, c.InOrStdin()) + _, inv, err := live.Load(r.ctx, r.factory, path, c.InOrStdin()) if err != nil { return err } diff --git a/cli/live/init/cmdliveinit.go b/cli/live/init/cmdliveinit.go index 4cc1ae05dc..b098625ba0 100644 --- a/cli/live/init/cmdliveinit.go +++ b/cli/live/init/cmdliveinit.go @@ -28,6 +28,7 @@ import ( "github.com/kptdev/kpt/internal/docs/generated/livedocs" "github.com/kptdev/kpt/pkg/kptfile/kptfileutil" "github.com/kptdev/kpt/pkg/lib/errors" + "github.com/kptdev/kpt/pkg/lib/krmvalidate" "github.com/kptdev/kpt/pkg/lib/pkg" "github.com/kptdev/kpt/pkg/lib/util/attribution" pathutil "github.com/kptdev/kpt/pkg/lib/util/path" @@ -38,6 +39,9 @@ import ( k8scmdutil "k8s.io/kubectl/pkg/cmd/util" "sigs.k8s.io/cli-utils/pkg/config" "sigs.k8s.io/kustomize/kyaml/filesys" + "sigs.k8s.io/kustomize/kyaml/kio/filters" + "sigs.k8s.io/kustomize/kyaml/kio/kioutil" + "sigs.k8s.io/kustomize/kyaml/openapi" "sigs.k8s.io/kustomize/kyaml/yaml" ) @@ -153,10 +157,42 @@ func (c *ConfigureInventoryInfo) Run(ctx context.Context) error { const op errors.Op = "cmdliveinit.Run" pr := printer.FromContextOrDie(ctx) - namespace, err := config.FindNamespace(c.Factory.ToRawKubeConfigLoader(), c.Pkg.UniquePath.String()) + // Pre-scan for non-KRM files before config.FindNamespace() reads them. + scanResult, err := krmvalidate.ScanFlatAndDecide(filesys.FileSystemOrOnDisk{}, c.Pkg.UniquePath.String(), pr.Printf) if err != nil { - return errors.E(op, c.Pkg.UniquePath, - fmt.Errorf("failed to determine inventory namespace from package resources: %w; please provide the namespace using the --namespace flag", err)) + return errors.E(op, c.Pkg.UniquePath, err) + } + + var namespace string + if len(scanResult.SkippedFiles()) > 0 { + // FindNamespace re-reads the package with no skip hook, so it + // would hit the excused files. Replicate its priority order + // instead: explicit --namespace wins, else check agreement. + loaderNamespace, enforceNamespace, nsErr := c.Factory.ToRawKubeConfigLoader().Namespace() + if nsErr != nil { + return errors.E(op, c.Pkg.UniquePath, + fmt.Errorf("failed to determine inventory namespace: %w; please provide the namespace using the --namespace flag", nsErr)) + } + namespace = loaderNamespace + if !enforceNamespace { + filtered, ferr := filterSkipped(scanResult.AllResources, scanResult.SkipFunc(c.Pkg.UniquePath.String())) + if ferr != nil { + return errors.E(op, c.Pkg.UniquePath, ferr) + } + ns, ok, aerr := allInSameNamespace(filtered) + if aerr != nil { + return errors.E(op, c.Pkg.UniquePath, aerr) + } + if ok { + namespace = ns + } + } + } else { + namespace, err = config.FindNamespace(c.Factory.ToRawKubeConfigLoader(), c.Pkg.UniquePath.String()) + if err != nil { + return errors.E(op, c.Pkg.UniquePath, + fmt.Errorf("failed to determine inventory namespace from package resources: %w; please provide the namespace using the --namespace flag", err)) + } } namespace = strings.TrimSpace(namespace) if !c.Quiet { @@ -214,11 +250,67 @@ func (c *ConfigureInventoryInfo) Run(ctx context.Context) error { } // add metrics annotation to package resources to track the usage as the resources // will be applied using kpt live group - at := attribution.Attributor{PackagePaths: []string{c.Pkg.UniquePath.String()}, CmdGroup: "live"} + at := attribution.Attributor{ + PackagePaths: []string{c.Pkg.UniquePath.String()}, + CmdGroup: "live", + FileSkipFunc: scanResult.SkipFunc(c.Pkg.UniquePath.String()), + } at.Process() return nil } +// allInSameNamespace is kpt's own copy of cli-utils's unexported +// config.allInSameNamespace, operating on an already-filtered +// resource list instead of re-reading the directory itself. +func allInSameNamespace(nodes []*yaml.RNode) (string, bool, error) { + nodes, err := (&filters.IsLocalConfig{}).Filter(nodes) + if err != nil { + return "", false, err + } + + var ns string + for _, node := range nodes { + rm, err := node.GetMeta() + if err != nil { + return "", false, err + } + if namespaced, found := openapi.IsNamespaceScoped(rm.TypeMeta); found && !namespaced { + continue + } + if rm.Namespace == "" { + return "", false, nil + } + if ns == "" { + ns = rm.Namespace + } else if rm.Namespace != ns { + return "", false, nil + } + } + if ns == "" { + return "", false, nil + } + return ns, true, nil +} + +// filterSkipped drops any node skip reports true for, by its +// kioutil.PathAnnotation. A nil skip leaves nodes unchanged. +func filterSkipped(nodes []*yaml.RNode, skip func(relPath string) bool) ([]*yaml.RNode, error) { + if skip == nil { + return nodes, nil + } + var out []*yaml.RNode + for _, n := range nodes { + path, _, err := kioutil.GetFileAnnotations(n) + if err != nil { + return nil, err + } + if !skip(path) { + out = append(out, n) + } + } + return out, nil +} + // createRGFile fills in the inventory object values into the resourcegroup object and writes to file storage. func createRGFile(p *pkg.Pkg, inv *kptfilev1.Inventory, filename string, force bool) error { const op errors.Op = "cmdliveinit.createRGFile" diff --git a/cli/live/init/cmdliveinit_test.go b/cli/live/init/cmdliveinit_test.go index 67331d8cbf..4fbf8ba67b 100644 --- a/cli/live/init/cmdliveinit_test.go +++ b/cli/live/init/cmdliveinit_test.go @@ -26,9 +26,12 @@ import ( "github.com/kptdev/kpt/pkg/lib/pkg" "github.com/kptdev/kpt/pkg/printer/fake" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "k8s.io/cli-runtime/pkg/genericclioptions" cmdtesting "k8s.io/kubectl/pkg/cmd/testing" "sigs.k8s.io/kustomize/kyaml/filesys" + "sigs.k8s.io/kustomize/kyaml/kio/kioutil" + "sigs.k8s.io/kustomize/kyaml/yaml" ) var ( @@ -345,6 +348,238 @@ func TestCmd_Run(t *testing.T) { } } +// TestCmd_Run_NonKRM covers the non-KRM pre-scan in ConfigureInventoryInfo.Run. +func TestCmd_Run_NonKRM(t *testing.T) { + t.Run("non-KRM file with no annotation blocks init with a consolidated, file-naming error", func(t *testing.T) { + tf := cmdtesting.NewTestFactory().WithNamespace("testns") + defer tf.Cleanup() + ioStreams, _, _, _ := genericclioptions.NewTestIOStreams() //nolint:dogsled + + w, clean := testutil.SetupWorkspace(t) + defer clean() + assert.NoError(t, os.WriteFile(filepath.Join(w.WorkspaceDirectory, kptfilev1.KptFileName), []byte(kptFile), 0600)) + assert.NoError(t, os.WriteFile(filepath.Join(w.WorkspaceDirectory, "foo.yaml"), []byte("- foo\n"), 0600)) + + revert := testutil.Chdir(t, w.WorkspaceDirectory) + defer revert() + + runner := NewRunner(fake.CtxWithDefaultPrinter(), tf, ioStreams) + runner.Command.SetArgs([]string{}) + err := runner.Command.Execute() + + if assert.Error(t, err) { + assert.Contains(t, err.Error(), "foo.yaml") + assert.Contains(t, err.Error(), "1 file(s)") + assert.NotContains(t, err.Error(), "wrong node kind") + assert.NotContains(t, err.Error(), "wrong Node Kind") + } + }) + + t.Run("annotated package skips the non-KRM file in memory, leaves it untouched on disk, no .krmignore written", func(t *testing.T) { + tf := cmdtesting.NewTestFactory().WithNamespace("testns") + defer tf.Cleanup() + ioStreams, _, _, _ := genericclioptions.NewTestIOStreams() //nolint:dogsled + + w, clean := testutil.SetupWorkspace(t) + defer clean() + annotatedKptfile := ` +apiVersion: kpt.dev/v1 +kind: Kptfile +metadata: + name: test1 + annotations: + kpt.dev/auto-ignore-non-krm: "true" +` + assert.NoError(t, os.WriteFile(filepath.Join(w.WorkspaceDirectory, kptfilev1.KptFileName), []byte(annotatedKptfile), 0600)) + assert.NoError(t, os.WriteFile(filepath.Join(w.WorkspaceDirectory, "foo.yaml"), []byte("- foo\n"), 0600)) + + revert := testutil.Chdir(t, w.WorkspaceDirectory) + defer revert() + + runner := NewRunner(fake.CtxWithDefaultPrinter(), tf, ioStreams) + runner.Command.SetArgs([]string{}) + err := runner.Command.Execute() + assert.NoError(t, err) + + content, readErr := os.ReadFile(filepath.Join(w.WorkspaceDirectory, "foo.yaml")) + assert.NoError(t, readErr) + assert.Equal(t, "- foo\n", string(content), "non-KRM file must be left completely untouched on disk") + + assert.NoFileExists(t, filepath.Join(w.WorkspaceDirectory, ".krmignore"), ".krmignore must never be written by the annotation path") + + rg, err := pkg.ReadRGFile(w.WorkspaceDirectory, "resourcegroup.yaml") + assert.NoError(t, err) + assert.NotNil(t, rg, "init must have succeeded and written the ResourceGroup file despite the non-KRM file") + }) + + t.Run("annotated package with no --namespace falls back cleanly instead of re-reading the excused file", func(t *testing.T) { + // Deliberately no .WithNamespace(...): this exercises + // FindNamespace's real auto-detection path, which the other + // subtests above bypass by forcing enforceNamespace=true. + tf := cmdtesting.NewTestFactory() + defer tf.Cleanup() + ioStreams, _, _, _ := genericclioptions.NewTestIOStreams() //nolint:dogsled + + w, clean := testutil.SetupWorkspace(t) + defer clean() + annotatedKptfile := ` +apiVersion: kpt.dev/v1 +kind: Kptfile +metadata: + name: test1 + annotations: + kpt.dev/auto-ignore-non-krm: "true" +` + assert.NoError(t, os.WriteFile(filepath.Join(w.WorkspaceDirectory, kptfilev1.KptFileName), []byte(annotatedKptfile), 0600)) + assert.NoError(t, os.WriteFile(filepath.Join(w.WorkspaceDirectory, "foo.yaml"), []byte("- foo\n"), 0600)) + + revert := testutil.Chdir(t, w.WorkspaceDirectory) + defer revert() + + runner := NewRunner(fake.CtxWithDefaultPrinter(), tf, ioStreams) + runner.Command.SetArgs([]string{}) + err := runner.Command.Execute() + + // Must not resurface the original crash this PR exists to fix. + if assert.NoError(t, err) { + content, readErr := os.ReadFile(filepath.Join(w.WorkspaceDirectory, "foo.yaml")) + assert.NoError(t, readErr) + assert.Equal(t, "- foo\n", string(content), "non-KRM file must be left untouched") + } + }) + + t.Run("annotated package still auto-detects the real namespace, matching what .krmignore would give", func(t *testing.T) { + // Deliberately no .WithNamespace(...), same as the subtest + // above: exercises real auto-detection (enforceNamespace + // false), whose kubeconfig-default fallback is "default" -- + // distinct from "foo" below, so a wrong answer is obvious. + tf := cmdtesting.NewTestFactory() + defer tf.Cleanup() + ioStreams, _, _, _ := genericclioptions.NewTestIOStreams() //nolint:dogsled + + w, clean := testutil.SetupWorkspace(t) + defer clean() + annotatedKptfile := ` +apiVersion: kpt.dev/v1 +kind: Kptfile +metadata: + name: test1 + annotations: + kpt.dev/auto-ignore-non-krm: "true" +` + cm := ` +apiVersion: v1 +kind: ConfigMap +metadata: + name: cm1 + namespace: foo +` + assert.NoError(t, os.WriteFile(filepath.Join(w.WorkspaceDirectory, kptfilev1.KptFileName), []byte(annotatedKptfile), 0600)) + assert.NoError(t, os.WriteFile(filepath.Join(w.WorkspaceDirectory, "cm.yaml"), []byte(cm), 0600)) + assert.NoError(t, os.WriteFile(filepath.Join(w.WorkspaceDirectory, "foo.yaml"), []byte("- foo\n"), 0600)) + + revert := testutil.Chdir(t, w.WorkspaceDirectory) + defer revert() + + runner := NewRunner(fake.CtxWithDefaultPrinter(), tf, ioStreams) + runner.Command.SetArgs([]string{}) + err := runner.Command.Execute() + assert.NoError(t, err) + + rg, readErr := pkg.ReadRGFile(w.WorkspaceDirectory, "resourcegroup.yaml") + assert.NoError(t, readErr) + if assert.NotNil(t, rg) { + assert.Equal(t, "foo", rg.Namespace, + "must match cm.yaml's actual namespace, not the kubeconfig default, the same as if foo.yaml had been excluded via .krmignore instead") + } + }) + + t.Run("excused mixed file is never read by attribution either, not just left unwritten", func(t *testing.T) { + tf := cmdtesting.NewTestFactory().WithNamespace("testns") + defer tf.Cleanup() + ioStreams, _, _, _ := genericclioptions.NewTestIOStreams() //nolint:dogsled + + w, clean := testutil.SetupWorkspace(t) + defer clean() + annotatedKptfile := ` +apiVersion: kpt.dev/v1 +kind: Kptfile +metadata: + name: test1 + annotations: + kpt.dev/auto-ignore-non-krm: "true" +` + // A mixed file: one non-KRM document (which excludes the + // whole file per Split's "excluded in its entirety" rule) + // alongside a real CNRM resource that attribution would + // normally annotate. + mixedContent := `foo: bar +baz: [1, 2, 3] +--- +apiVersion: resourcemanager.cnrm.cloud.google.com/v1beta1 +kind: Project +metadata: + name: my-project +` + assert.NoError(t, os.WriteFile(filepath.Join(w.WorkspaceDirectory, kptfilev1.KptFileName), []byte(annotatedKptfile), 0600)) + assert.NoError(t, os.WriteFile(filepath.Join(w.WorkspaceDirectory, "mixed.yaml"), []byte(mixedContent), 0600)) + + revert := testutil.Chdir(t, w.WorkspaceDirectory) + defer revert() + + runner := NewRunner(fake.CtxWithDefaultPrinter(), tf, ioStreams) + runner.Command.SetArgs([]string{}) + err := runner.Command.Execute() + assert.NoError(t, err) + + content, readErr := os.ReadFile(filepath.Join(w.WorkspaceDirectory, "mixed.yaml")) + assert.NoError(t, readErr) + assert.Equal(t, mixedContent, string(content), + "attribution must not add its annotation to the Project resource; the whole excused file must be left byte-for-byte untouched") + }) + + t.Run("bare directory with no Kptfile at all is still pre-scanned", func(t *testing.T) { + tf := cmdtesting.NewTestFactory().WithNamespace("testns") + defer tf.Cleanup() + ioStreams, _, _, _ := genericclioptions.NewTestIOStreams() //nolint:dogsled + + w, clean := testutil.SetupWorkspace(t) + defer clean() + assert.NoError(t, os.WriteFile(filepath.Join(w.WorkspaceDirectory, "foo.yaml"), []byte("- foo\n"), 0600)) + + revert := testutil.Chdir(t, w.WorkspaceDirectory) + defer revert() + + runner := NewRunner(fake.CtxWithDefaultPrinter(), tf, ioStreams) + runner.Command.SetArgs([]string{}) + err := runner.Command.Execute() + + if assert.Error(t, err) { + assert.Contains(t, err.Error(), "foo.yaml") + } + }) + + t.Run(".krmignore still works independently of the annotation", func(t *testing.T) { + tf := cmdtesting.NewTestFactory().WithNamespace("testns") + defer tf.Cleanup() + ioStreams, _, _, _ := genericclioptions.NewTestIOStreams() //nolint:dogsled + + w, clean := testutil.SetupWorkspace(t) + defer clean() + assert.NoError(t, os.WriteFile(filepath.Join(w.WorkspaceDirectory, kptfilev1.KptFileName), []byte(kptFile), 0600)) + assert.NoError(t, os.WriteFile(filepath.Join(w.WorkspaceDirectory, "foo.yaml"), []byte("- foo\n"), 0600)) + assert.NoError(t, os.WriteFile(filepath.Join(w.WorkspaceDirectory, ".krmignore"), []byte("foo.yaml\n"), 0600)) + + revert := testutil.Chdir(t, w.WorkspaceDirectory) + defer revert() + + runner := NewRunner(fake.CtxWithDefaultPrinter(), tf, ioStreams) + runner.Command.SetArgs([]string{}) + err := runner.Command.Execute() + assert.NoError(t, err) + }) +} + func TestCmd_Run_InvalidDirectoryName(t *testing.T) { tf := cmdtesting.NewTestFactory().WithNamespace("testns") defer tf.Cleanup() @@ -497,30 +732,101 @@ func assertGenInvID(t *testing.T, name, namespace, actual string) bool { return assert.Equal(t, expected, actual) } -func TestCmd_Run_FindNamespaceError(t *testing.T) { - tf := cmdtesting.NewTestFactory() - defer tf.Cleanup() - ioStreams, _, _, _ := genericclioptions.NewTestIOStreams() //nolint:dogsled - - w, clean := testutil.SetupWorkspace(t) - defer clean() - - err := os.WriteFile(filepath.Join(w.WorkspaceDirectory, kptfilev1.KptFileName), []byte(kptFile), 0600) - assert.NoError(t, err) - - // Create a non-KRM YAML file in the package that causes kio reader to fail - err = os.WriteFile(filepath.Join(w.WorkspaceDirectory, "non-krm.yaml"), []byte("- foo\n"), 0600) - assert.NoError(t, err) +// Note: TestCmd_Run_FindNamespaceError (added in #4812) is removed: +// its non-KRM fixture is now caught by the pre-scan before +// FindNamespace runs. See TestCmd_Run_NonKRM for that path. - revert := testutil.Chdir(t, w.WorkspaceDirectory) - defer revert() +func mustParseNode(t *testing.T, s string) *yaml.RNode { + t.Helper() + n, err := yaml.Parse(s) + require.NoError(t, err) + return n +} - runner := NewRunner(fake.CtxWithDefaultPrinter(), tf, ioStreams) - runner.Command.SetArgs([]string{}) - err = runner.Command.Execute() +func TestAllInSameNamespace(t *testing.T) { + t.Run("all resources agree on one namespace", func(t *testing.T) { + a := mustParseNode(t, "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: a\n namespace: foo\n") + b := mustParseNode(t, "apiVersion: v1\nkind: Secret\nmetadata:\n name: b\n namespace: foo\n") + ns, ok, err := allInSameNamespace([]*yaml.RNode{a, b}) + require.NoError(t, err) + assert.True(t, ok) + assert.Equal(t, "foo", ns) + }) + + t.Run("resources disagree on namespace", func(t *testing.T) { + a := mustParseNode(t, "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: a\n namespace: foo\n") + b := mustParseNode(t, "apiVersion: v1\nkind: Secret\nmetadata:\n name: b\n namespace: bar\n") + ns, ok, err := allInSameNamespace([]*yaml.RNode{a, b}) + require.NoError(t, err) + assert.False(t, ok) + assert.Empty(t, ns) + }) + + t.Run("a resource missing a namespace blocks agreement", func(t *testing.T) { + a := mustParseNode(t, "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: a\n namespace: foo\n") + b := mustParseNode(t, "apiVersion: v1\nkind: Secret\nmetadata:\n name: b\n") + ns, ok, err := allInSameNamespace([]*yaml.RNode{a, b}) + require.NoError(t, err) + assert.False(t, ok) + assert.Empty(t, ns) + }) + + t.Run("empty input means no agreement", func(t *testing.T) { + ns, ok, err := allInSameNamespace(nil) + require.NoError(t, err) + assert.False(t, ok) + assert.Empty(t, ns) + }) + + t.Run("local-config resources are excluded from the check", func(t *testing.T) { + a := mustParseNode(t, "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: a\n namespace: foo\n") + // No namespace at all, but marked local-config: must not + // block agreement, same as cli-utils's allInSameNamespace. + local := mustParseNode(t, `apiVersion: v1 +kind: ConfigMap +metadata: + name: marker + annotations: + config.kubernetes.io/local-config: "true" +`) + ns, ok, err := allInSameNamespace([]*yaml.RNode{a, local}) + require.NoError(t, err) + assert.True(t, ok) + assert.Equal(t, "foo", ns) + }) + + t.Run("cluster-scoped resources are excluded from the check", func(t *testing.T) { + a := mustParseNode(t, "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: a\n namespace: foo\n") + // Namespace itself is cluster-scoped and has no + // metadata.namespace: must not block agreement. + clusterScoped := mustParseNode(t, "apiVersion: v1\nkind: Namespace\nmetadata:\n name: foo\n") + ns, ok, err := allInSameNamespace([]*yaml.RNode{a, clusterScoped}) + require.NoError(t, err) + assert.True(t, ok) + assert.Equal(t, "foo", ns) + }) +} - if assert.Error(t, err) { - assert.Contains(t, err.Error(), "failed to determine inventory namespace from package resources") - assert.Contains(t, err.Error(), "please provide the namespace using the --namespace flag") - } +func TestFilterSkipped(t *testing.T) { + a := mustParseNode(t, "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: a\n") + require.NoError(t, a.PipeE(yaml.SetAnnotation(kioutil.PathAnnotation, "a.yaml"))) + b := mustParseNode(t, "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: b\n") + require.NoError(t, b.PipeE(yaml.SetAnnotation(kioutil.PathAnnotation, "values.yaml"))) + + t.Run("nil skip leaves nodes unchanged", func(t *testing.T) { + out, err := filterSkipped([]*yaml.RNode{a, b}, nil) + require.NoError(t, err) + assert.Len(t, out, 2) + }) + + t.Run("skip drops the matching node by path", func(t *testing.T) { + skip := func(relPath string) bool { return relPath == "values.yaml" } + out, err := filterSkipped([]*yaml.RNode{a, b}, skip) + require.NoError(t, err) + if assert.Len(t, out, 1) { + meta, err := out[0].GetMeta() + require.NoError(t, err) + assert.Equal(t, "a", meta.Name) + } + }) } diff --git a/cli/live/migrate/migratecmd.go b/cli/live/migrate/migratecmd.go index 42c45bfff2..f979748ac0 100644 --- a/cli/live/migrate/migratecmd.go +++ b/cli/live/migrate/migratecmd.go @@ -22,6 +22,7 @@ import ( "io" "os" "path/filepath" + "sort" kptfilev1 "github.com/kptdev/kpt/api/kptfile/v1" rgfilev1alpha1 "github.com/kptdev/kpt/api/resourcegroup/v1alpha1" @@ -29,6 +30,7 @@ import ( "github.com/kptdev/kpt/internal/docs/generated/livedocs" "github.com/kptdev/kpt/pkg/kptfile/kptfileutil" "github.com/kptdev/kpt/pkg/lib/errors" + "github.com/kptdev/kpt/pkg/lib/krmvalidate" "github.com/kptdev/kpt/pkg/lib/pkg" argsutil "github.com/kptdev/kpt/pkg/lib/util/args" pathutil "github.com/kptdev/kpt/pkg/lib/util/path" @@ -143,6 +145,35 @@ func (mr *Runner) Run(reader io.Reader, args []string) error { } } + // Non-KRM pre-scan (directory mode only). The auto-ignore + // annotation is deliberately NOT honored here, unlike every other + // `live` command: the legacy ConfigMap-inventory reader this + // command depends on has no way to skip a file in memory, so + // .krmignore is the only real remediation. + if mr.dir != "" { + found, _, _, err := krmvalidate.ScanFlat(filesys.FileSystemOrOnDisk{}, mr.dir) + if err != nil { + return fmt.Errorf("non-krm pre-scan: %w", err) + } + if len(found) > 0 { + // Not krmvalidate.DecideOutcome: its message recommends + // the annotation as a fix, which wouldn't work here. + sort.Slice(found, func(i, j int) bool { + return found[i].DisplayPath(mr.dir) < found[j].DisplayPath(mr.dir) + }) + var b bytes.Buffer + fmt.Fprintf(&b, "input resource list must contain only KRM resources (%d file(s)):", len(found)) + for _, f := range found { + fmt.Fprintf(&b, "\n %s: %s", f.DisplayPath(mr.dir), f.Reason) + } + fmt.Fprint(&b, "\nTo resolve: exclude these files via .krmignore (manually, or with a pattern rule). "+ + "Note: `kpt.dev/auto-ignore-non-krm` does not apply to `migrate` -- the legacy ConfigMap "+ + "inventory reader this command uses has no way to skip a file in memory, so it must be "+ + "excluded via .krmignore instead.") + return goerrors.New(b.String()) + } + } + // Apply the ResourceGroup CRD to the cluster, ignoring if it already exists. if err := mr.applyCRD(); err != nil { return err @@ -256,7 +287,7 @@ func (mr *Runner) migrateObjs(rgInvClient inventory.Client, } } - _, inv, err := live.Load(mr.factory, path, reader) + _, inv, err := live.Load(mr.ctx, mr.factory, path, reader) if err != nil { return err } diff --git a/cli/live/migrate/migratecmd_test.go b/cli/live/migrate/migratecmd_test.go index c2699963b9..b126aa4e9f 100644 --- a/cli/live/migrate/migratecmd_test.go +++ b/cli/live/migrate/migratecmd_test.go @@ -388,3 +388,142 @@ metadata: labels: cli-utils.sigs.k8s.io/inventory-id: SSSSSSSSSS-RRRRR ` + +func TestKptMigrate_Run_NonKRM(t *testing.T) { + t.Run("non-KRM file with no annotation blocks Run with a consolidated, file-naming error", func(t *testing.T) { + tf := cmdtesting.NewTestFactory().WithNamespace(inventoryNamespace) + defer tf.Cleanup() + ioStreams, _, _, _ := genericclioptions.NewTestIOStreams() //nolint:dogsled + + dir := t.TempDir() + err := os.WriteFile(filepath.Join(dir, "Kptfile"), []byte(kptFile), 0600) + assert.NoError(t, err) + err = os.WriteFile(filepath.Join(dir, "values.yaml"), []byte("replicaCount: 3\n"), 0600) + assert.NoError(t, err) + + ctx := fake.CtxWithDefaultPrinter() + cmLoader := manifestreader.NewManifestLoader(tf) + migrateRunner := NewRunner(ctx, tf, cmLoader, ioStreams) + migrateRunner.cmInvClientFunc = func(_ context.Context, _ util.Factory) (inventory.Client, error) { + return inventory.NewFakeClient([]object.ObjMetadata{}), nil + } + + err = migrateRunner.Run(strings.NewReader(""), []string{dir}) + if !assert.Error(t, err) { + t.FailNow() + } + assert.Contains(t, err.Error(), "values.yaml") + assert.Contains(t, err.Error(), "1 file(s)") + assert.Contains(t, err.Error(), ".krmignore") + assert.NotContains(t, err.Error(), "add `kpt.dev/auto-ignore-non-krm", "must not suggest the annotation as a fix") + }) + + t.Run("the auto-ignore annotation does NOT unblock migrate -- still fails with the same pre-scan error", func(t *testing.T) { + tf := cmdtesting.NewTestFactory().WithNamespace(inventoryNamespace) + defer tf.Cleanup() + ioStreams, _, _, _ := genericclioptions.NewTestIOStreams() //nolint:dogsled + + dir := t.TempDir() + kptFileWithAnnotation := ` +apiVersion: kpt.dev/v1 +kind: Kptfile +metadata: + name: test1 + annotations: + kpt.dev/auto-ignore-non-krm: "true" +upstreamLock: + type: git + git: + repo: git@github.com:seans3/blueprint-helloworld + directory: / + ref: master +` + err := os.WriteFile(filepath.Join(dir, "Kptfile"), []byte(kptFileWithAnnotation), 0600) + assert.NoError(t, err) + err = os.WriteFile(filepath.Join(dir, "values.yaml"), []byte("replicaCount: 3\n"), 0600) + assert.NoError(t, err) + + ctx := fake.CtxWithDefaultPrinter() + cmLoader := manifestreader.NewManifestLoader(tf) + migrateRunner := NewRunner(ctx, tf, cmLoader, ioStreams) + migrateRunner.dryRun = true + migrateRunner.cmInvClientFunc = func(_ context.Context, _ util.Factory) (inventory.Client, error) { + return inventory.NewFakeClient([]object.ObjMetadata{}), nil + } + + err = migrateRunner.Run(strings.NewReader(""), []string{dir}) + if !assert.Error(t, err) { + t.FailNow() + } + assert.Contains(t, err.Error(), "values.yaml") + assert.Contains(t, err.Error(), "1 file(s)") + + // Confirm the annotation did not cause the file to be skipped or modified. + content, readErr := os.ReadFile(filepath.Join(dir, "values.yaml")) + assert.NoError(t, readErr) + assert.Equal(t, "replicaCount: 3\n", string(content)) + _, statErr := os.Stat(filepath.Join(dir, ".krmignore")) + assert.True(t, os.IsNotExist(statErr), ".krmignore should not have been written") + }) + + t.Run("same-named non-KRM files in different subpackages are disambiguated by path", func(t *testing.T) { + tf := cmdtesting.NewTestFactory().WithNamespace(inventoryNamespace) + defer tf.Cleanup() + ioStreams, _, _, _ := genericclioptions.NewTestIOStreams() //nolint:dogsled + + dir := t.TempDir() + app1 := filepath.Join(dir, "apps", "app1") + app2 := filepath.Join(dir, "apps", "app2") + assert.NoError(t, os.MkdirAll(app1, 0700)) + assert.NoError(t, os.MkdirAll(app2, 0700)) + assert.NoError(t, os.WriteFile(filepath.Join(dir, "Kptfile"), []byte(kptFile), 0600)) + assert.NoError(t, os.WriteFile(filepath.Join(app1, "Kptfile"), []byte("apiVersion: kpt.dev/v1\nkind: Kptfile\nmetadata:\n name: app1\n"), 0600)) + assert.NoError(t, os.WriteFile(filepath.Join(app2, "Kptfile"), []byte("apiVersion: kpt.dev/v1\nkind: Kptfile\nmetadata:\n name: app2\n"), 0600)) + assert.NoError(t, os.WriteFile(filepath.Join(app1, "values.yaml"), []byte("replicaCount: 1\n"), 0600)) + assert.NoError(t, os.WriteFile(filepath.Join(app2, "values.yaml"), []byte("replicaCount: 2\n"), 0600)) + + ctx := fake.CtxWithDefaultPrinter() + cmLoader := manifestreader.NewManifestLoader(tf) + migrateRunner := NewRunner(ctx, tf, cmLoader, ioStreams) + migrateRunner.cmInvClientFunc = func(_ context.Context, _ util.Factory) (inventory.Client, error) { + return inventory.NewFakeClient([]object.ObjMetadata{}), nil + } + + err := migrateRunner.Run(strings.NewReader(""), []string{dir}) + if !assert.Error(t, err) { + t.FailNow() + } + assert.Contains(t, err.Error(), filepath.ToSlash(filepath.Join("apps", "app1", "values.yaml"))) + assert.Contains(t, err.Error(), filepath.ToSlash(filepath.Join("apps", "app2", "values.yaml"))) + assert.Contains(t, err.Error(), "2 file(s)") + }) + + t.Run(".krmignore still works independently of the annotation, no non-KRM-file error at all", func(t *testing.T) { + tf := cmdtesting.NewTestFactory().WithNamespace(inventoryNamespace) + defer tf.Cleanup() + ioStreams, _, _, _ := genericclioptions.NewTestIOStreams() //nolint:dogsled + + dir := t.TempDir() + err := os.WriteFile(filepath.Join(dir, "Kptfile"), []byte(kptFile), 0600) + assert.NoError(t, err) + err = os.WriteFile(filepath.Join(dir, "values.yaml"), []byte("replicaCount: 3\n"), 0600) + assert.NoError(t, err) + err = os.WriteFile(filepath.Join(dir, ".krmignore"), []byte("values.yaml\n"), 0600) + assert.NoError(t, err) + + ctx := fake.CtxWithDefaultPrinter() + cmLoader := manifestreader.NewManifestLoader(tf) + migrateRunner := NewRunner(ctx, tf, cmLoader, ioStreams) + migrateRunner.cmInvClientFunc = func(_ context.Context, _ util.Factory) (inventory.Client, error) { + return inventory.NewFakeClient([]object.ObjMetadata{}), nil + } + + err = migrateRunner.Run(strings.NewReader(""), []string{dir}) + // The pre-scan must not block this case. Any other downstream error + // (e.g. from the fake ConfigMap-based inventory lookup) is tolerated. + if err != nil { + assert.NotContains(t, err.Error(), "values.yaml") + assert.NotContains(t, err.Error(), "must contain only KRM resources") + } + }) +} diff --git a/cli/live/status/cmdstatus.go b/cli/live/status/cmdstatus.go index 582668fb10..0361774618 100644 --- a/cli/live/status/cmdstatus.go +++ b/cli/live/status/cmdstatus.go @@ -80,7 +80,7 @@ func (rir *RGInventoryLoader) GetInvInfo(cmd *cobra.Command, args []string) (inv } } - _, inv, err := live.Load(rir.factory, path, cmd.InOrStdin()) + _, inv, err := live.Load(rir.ctx, rir.factory, path, cmd.InOrStdin()) if err != nil { return nil, err } diff --git a/cli/live/status/fake-loader.go b/cli/live/status/fake-loader.go index 90db0edca8..f4a248dd50 100644 --- a/cli/live/status/fake-loader.go +++ b/cli/live/status/fake-loader.go @@ -62,7 +62,7 @@ func (r *FakeLoader) GetInvInfo(cmd *cobra.Command, args []string) (inventory.In } } - _, inv, err := live.Load(r.factory, path, cmd.InOrStdin()) + _, inv, err := live.Load(r.ctx, r.factory, path, cmd.InOrStdin()) if err != nil { return nil, err } diff --git a/cli/pkg/cmdsource/cmdsource.go b/cli/pkg/cmdsource/cmdsource.go index 8f727467b3..f0c06d15bb 100644 --- a/cli/pkg/cmdsource/cmdsource.go +++ b/cli/pkg/cmdsource/cmdsource.go @@ -23,11 +23,13 @@ import ( kptfilev1 "github.com/kptdev/kpt/api/kptfile/v1" "github.com/kptdev/kpt/cli/pkg/cmdcat" "github.com/kptdev/kpt/internal/docs/generated/fndocs" + "github.com/kptdev/kpt/pkg/lib/krmvalidate" "github.com/kptdev/kpt/pkg/lib/pkg" argsutil "github.com/kptdev/kpt/pkg/lib/util/args" "github.com/kptdev/kpt/pkg/lib/util/cmdutil" "github.com/kptdev/kpt/pkg/printer" "github.com/spf13/cobra" + "sigs.k8s.io/kustomize/kyaml/filesys" "sigs.k8s.io/kustomize/kyaml/kio" "sigs.k8s.io/kustomize/kyaml/kio/kioutil" "sigs.k8s.io/kustomize/kyaml/yaml" @@ -118,6 +120,14 @@ func (r *SourceRunner) runE(c *cobra.Command, args []string) error { if err != nil { return err } + + // Non-KRM pre-scan, same as `fn eval`'s (skips subpackages a + // parent .krmignore excludes, like this reader does). + scanResult, err := krmvalidate.ScanHonoringKrmignore(filesys.FileSystemOrOnDisk{}, resolvedPath, printer.FromContextOrDie(r.Ctx).Printf) + if err != nil { + return err + } + inputs = append(inputs, kio.LocalPackageReader{ PackagePath: resolvedPath, MatchFilesGlob: pkg.MatchAllKRM, @@ -125,6 +135,7 @@ func (r *SourceRunner) runE(c *cobra.Command, args []string) error { PackageFileName: kptfilev1.KptFileName, IncludeSubpackages: true, WrapBareSeqNode: true, + FileSkipFunc: scanResult.SkipFunc(resolvedPath), }) } diff --git a/cli/pkg/cmdsource/cmdsource_test.go b/cli/pkg/cmdsource/cmdsource_test.go index 29be446585..e74eddde89 100644 --- a/cli/pkg/cmdsource/cmdsource_test.go +++ b/cli/pkg/cmdsource/cmdsource_test.go @@ -17,6 +17,7 @@ package cmdsource import ( "bytes" + "fmt" "os" "path/filepath" "testing" @@ -34,6 +35,7 @@ func TestSourceCommand(t *testing.T) { defer os.RemoveAll(d) err = os.WriteFile(filepath.Join(d, "f1.yaml"), []byte(` +apiVersion: apps/v1 kind: Deployment metadata: labels: @@ -44,6 +46,7 @@ metadata: spec: replicas: 1 --- +apiVersion: v1 kind: Service metadata: name: foo @@ -95,7 +98,8 @@ spec: if !assert.Equal(t, `apiVersion: config.kubernetes.io/v1 kind: ResourceList items: -- kind: Deployment +- apiVersion: apps/v1 + kind: Deployment metadata: labels: app: nginx2 @@ -109,7 +113,8 @@ items: internal.config.kubernetes.io/seqindent: 'compact' spec: replicas: 1 -- kind: Service +- apiVersion: v1 + kind: Service metadata: name: foo annotations: @@ -166,6 +171,7 @@ func TestSourceCommand_Unwrap(t *testing.T) { defer os.RemoveAll(d) err = os.WriteFile(filepath.Join(d, "f1.yaml"), []byte(` +apiVersion: apps/v1 kind: Deployment metadata: labels: @@ -176,6 +182,7 @@ metadata: spec: replicas: 1 --- +apiVersion: v1 kind: Service metadata: name: foo @@ -219,7 +226,8 @@ spec: return } - if !assert.Equal(t, `kind: Deployment + if !assert.Equal(t, `apiVersion: apps/v1 +kind: Deployment metadata: labels: app: nginx2 @@ -229,6 +237,7 @@ metadata: spec: replicas: 1 --- +apiVersion: v1 kind: Service metadata: name: foo @@ -302,6 +311,7 @@ func TestSourceCommand_DefaultDir(t *testing.T) { defer os.RemoveAll(d) err = os.WriteFile(filepath.Join(d, "f1.yaml"), []byte(` +apiVersion: apps/v1 kind: Deployment metadata: labels: @@ -312,6 +322,7 @@ metadata: spec: replicas: 1 --- +apiVersion: v1 kind: Service metadata: name: foo @@ -366,7 +377,8 @@ spec: if !assert.Equal(t, `apiVersion: config.kubernetes.io/v1 kind: ResourceList items: -- kind: Deployment +- apiVersion: apps/v1 + kind: Deployment metadata: labels: app: nginx2 @@ -380,7 +392,8 @@ items: internal.config.kubernetes.io/seqindent: 'compact' spec: replicas: 1 -- kind: Service +- apiVersion: v1 + kind: Service metadata: name: foo annotations: @@ -438,6 +451,7 @@ func TestSourceCommandJSON(t *testing.T) { err = os.WriteFile(filepath.Join(d, "f1.json"), []byte(` { + "apiVersion": "apps/v1", "kind": "Deployment", "metadata": { "labels": { @@ -489,7 +503,7 @@ func TestSourceCommandJSON(t *testing.T) { expected := `apiVersion: config.kubernetes.io/v1 kind: ResourceList items: -- {"kind": "Deployment", "metadata": {"labels": {"app": "nginx2"}, "name": "foo", "annotations": {"app": "nginx2", config.kubernetes.io/index: '0', config.kubernetes.io/path: 'f1.json', internal.config.kubernetes.io/index: '0', internal.config.kubernetes.io/path: 'f1.json', internal.config.kubernetes.io/seqindent: 'compact'}}, "spec": {"replicas": 1}} +- {"apiVersion": "apps/v1", "kind": "Deployment", "metadata": {"labels": {"app": "nginx2"}, "name": "foo", "annotations": {"app": "nginx2", config.kubernetes.io/index: '0', config.kubernetes.io/path: 'f1.json', internal.config.kubernetes.io/index: '0', internal.config.kubernetes.io/path: 'f1.json', internal.config.kubernetes.io/seqindent: 'compact'}}, "spec": {"replicas": 1}} - {"apiVersion": "v1", "kind": "Abstraction", "metadata": {"name": "foo", "annotations": {"config.kubernetes.io/function": "container:\n image: ghcr.io/example/reconciler:v1\n", "config.kubernetes.io/local-config": "true", config.kubernetes.io/index: '0', config.kubernetes.io/path: 'f2.json', internal.config.kubernetes.io/index: '0', internal.config.kubernetes.io/path: 'f2.json', internal.config.kubernetes.io/seqindent: 'compact'}}, "spec": {"replicas": 3}} ` @@ -513,6 +527,7 @@ func TestSourceCommand_Symlink(t *testing.T) { return } err = os.WriteFile(filepath.Join(d, "foo", "f1.yaml"), []byte(` +apiVersion: apps/v1 kind: Deployment metadata: labels: @@ -523,6 +538,7 @@ metadata: spec: replicas: 1 --- +apiVersion: v1 kind: Service metadata: name: foo @@ -575,7 +591,8 @@ spec: if !assert.Equal(t, `apiVersion: config.kubernetes.io/v1 kind: ResourceList items: -- kind: Deployment +- apiVersion: apps/v1 + kind: Deployment metadata: labels: app: nginx2 @@ -589,7 +606,8 @@ items: internal.config.kubernetes.io/seqindent: 'compact' spec: replicas: 1 -- kind: Service +- apiVersion: v1 + kind: Service metadata: name: foo annotations: @@ -638,3 +656,138 @@ items: } assert.Contains(t, stderr.String(), "please note that the symlinks within the package are ignored") } + +// TestSourceCommand_NonKRM covers the non-KRM pre-scan in SourceRunner.runE. +func TestSourceCommand_NonKRM(t *testing.T) { + t.Run("non-KRM file with no annotation blocks source with a consolidated, file-naming error", func(t *testing.T) { + d := t.TempDir() + err := os.WriteFile(filepath.Join(d, "deployment.yaml"), []byte(` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo +spec: + replicas: 1 +`), 0600) + assert.NoError(t, err) + err = os.WriteFile(filepath.Join(d, "values.yaml"), []byte("replicaCount: 3\n"), 0600) + assert.NoError(t, err) + + b := &bytes.Buffer{} + r := GetSourceRunner(fake.CtxWithPrinter(b, nil), "") + r.Command.SetArgs([]string{d}) + err = r.Command.Execute() + + assert.Error(t, err) + assert.Contains(t, err.Error(), "values.yaml") + assert.Contains(t, err.Error(), "1 file(s)") + }) + + t.Run("annotated package skips the non-KRM file in memory, leaves it untouched on disk", func(t *testing.T) { + d := t.TempDir() + err := os.WriteFile(filepath.Join(d, "Kptfile"), []byte(`apiVersion: kpt.dev/v1 +kind: Kptfile +metadata: + name: test-pkg + annotations: + kpt.dev/auto-ignore-non-krm: "true" +`), 0600) + assert.NoError(t, err) + err = os.WriteFile(filepath.Join(d, "deployment.yaml"), []byte(` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo +spec: + replicas: 1 +`), 0600) + assert.NoError(t, err) + err = os.WriteFile(filepath.Join(d, "values.yaml"), []byte("replicaCount: 3\n"), 0600) + assert.NoError(t, err) + + b := &bytes.Buffer{} + r := GetSourceRunner(fake.CtxWithPrinter(b, nil), "") + r.Command.SetArgs([]string{d}) + err = r.Command.Execute() + assert.NoError(t, err) + + assert.NotContains(t, b.String(), "replicaCount", "non-KRM file must not appear in the ResourceList output") + assert.Contains(t, b.String(), "kind: Deployment") + + content, readErr := os.ReadFile(filepath.Join(d, "values.yaml")) + assert.NoError(t, readErr) + assert.Equal(t, "replicaCount: 3\n", string(content), "non-KRM file must be left completely untouched on disk") + assert.NoFileExists(t, filepath.Join(d, ".krmignore"), ".krmignore must never be written by the annotation path") + }) + + t.Run(".krmignore still works independently of the annotation, no non-KRM-file error at all", func(t *testing.T) { + d := t.TempDir() + err := os.WriteFile(filepath.Join(d, "deployment.yaml"), []byte(` +apiVersion: apps/v1 +kind: Deployment +metadata: + name: foo +spec: + replicas: 1 +`), 0600) + assert.NoError(t, err) + err = os.WriteFile(filepath.Join(d, "values.yaml"), []byte("replicaCount: 3\n"), 0600) + assert.NoError(t, err) + err = os.WriteFile(filepath.Join(d, ".krmignore"), []byte("values.yaml\n"), 0600) + assert.NoError(t, err) + + b := &bytes.Buffer{} + r := GetSourceRunner(fake.CtxWithPrinter(b, nil), "") + r.Command.SetArgs([]string{d}) + err = r.Command.Execute() + assert.NoError(t, err) + + assert.NotContains(t, b.String(), "replicaCount") + assert.Contains(t, b.String(), "kind: Deployment") + }) + + t.Run("root .krmignore skips a fixture subpackage under an ignored directory", func(t *testing.T) { + d := t.TempDir() + fixture := filepath.Join(d, "testdata", "fixture") + assert.NoError(t, os.MkdirAll(fixture, 0700)) + kptfile := "apiVersion: kpt.dev/v1\nkind: Kptfile\nmetadata:\n name: %s\n" + assert.NoError(t, os.WriteFile(filepath.Join(d, "Kptfile"), []byte(fmt.Sprintf(kptfile, "root")), 0600)) + assert.NoError(t, os.WriteFile(filepath.Join(d, "cm.yaml"), + []byte("apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: cm\n"), 0600)) + assert.NoError(t, os.WriteFile(filepath.Join(d, ".krmignore"), []byte("testdata/\n"), 0600)) + assert.NoError(t, os.WriteFile(filepath.Join(fixture, "Kptfile"), []byte(fmt.Sprintf(kptfile, "fixture")), 0600)) + assert.NoError(t, os.WriteFile(filepath.Join(fixture, "values.yaml"), []byte("replicaCount: 3\n"), 0600)) + + b := &bytes.Buffer{} + r := GetSourceRunner(fake.CtxWithPrinter(b, nil), "") + r.Command.SetArgs([]string{d}) + err := r.Command.Execute() + assert.NoError(t, err, "the ignored directory's subpackage must never be scanned") + + assert.NotContains(t, b.String(), "replicaCount") + assert.Contains(t, b.String(), "name: cm") + }) + + t.Run("ignoring one sibling subpackage still reports a non-KRM file in another", func(t *testing.T) { + d := t.TempDir() + kptfile := "apiVersion: kpt.dev/v1\nkind: Kptfile\nmetadata:\n name: %s\n" + assert.NoError(t, os.WriteFile(filepath.Join(d, "Kptfile"), []byte(fmt.Sprintf(kptfile, "root")), 0600)) + assert.NoError(t, os.WriteFile(filepath.Join(d, ".krmignore"), []byte("skipme/\n"), 0600)) + for _, name := range []string{"ok", "skipme"} { + sub := filepath.Join(d, name) + assert.NoError(t, os.MkdirAll(sub, 0700)) + assert.NoError(t, os.WriteFile(filepath.Join(sub, "Kptfile"), []byte(fmt.Sprintf(kptfile, name)), 0600)) + assert.NoError(t, os.WriteFile(filepath.Join(sub, "values.yaml"), []byte("replicaCount: 3\n"), 0600)) + } + + b := &bytes.Buffer{} + r := GetSourceRunner(fake.CtxWithPrinter(b, nil), "") + r.Command.SetArgs([]string{d}) + err := r.Command.Execute() + if assert.Error(t, err) { + assert.Contains(t, err.Error(), "ok/values.yaml") + assert.NotContains(t, err.Error(), "skipme", "the ignored sibling must not be reported") + assert.Contains(t, err.Error(), "1 file(s)") + } + }) +} diff --git a/documentation/content/en/book/03-packages/_index.md b/documentation/content/en/book/03-packages/_index.md index f43fffadbd..33b6d9680b 100644 --- a/documentation/content/en/book/03-packages/_index.md +++ b/documentation/content/en/book/03-packages/_index.md @@ -162,9 +162,26 @@ kpt fn eval wordpress -i search-replace:latest -- 'by-path=spec.selector.tier' ## Excluding files with `.krmignore` -A package may contain files that should not be processed by kpt, such as test fixtures or local development overrides. -You can exclude them by creating a `.krmignore` file in the package directory. It uses the same pattern format as -`.gitignore` (except recursive `**` patterns are not supported). +Every file in a package falls into one of three categories: + +1. **KRM files** that kpt should process: real KRM resources (`apiVersion`, `kind`, and `metadata.name` are + all present). These are read, hydrated, applied, and so on, as normal. +2. **KRM-shaped files kpt should leave alone**: valid KRM resources (or ordinary files that happen to sit + in the package directory) that you don't want kpt to touch at all, such as test fixtures or local + development overrides. You exclude these yourself with `.krmignore`. +3. **Non-KRM files kpt cannot work with**: files that look like they should be processed (a `.yaml`/`.yml`/ + `.json` file, or a file named `Kptfile`) but aren't actually valid KRM resources, such as a Helm + `values.yaml`. kpt doesn't know what to do with these, so they must be excluded one way or another, + either manually via `.krmignore` or automatically via the + [`kpt.dev/auto-ignore-non-krm`]({{% relref "/reference/annotations/auto-ignore-non-krm" %}}) annotation + described below. Like `.krmignore`, the annotation is scoped per package: a subpackage with its own + `Kptfile` does not inherit it from a parent package. + +### Excluding files manually with `.krmignore` + +You can exclude files from categories 2 and 3 above by creating a `.krmignore` file in the package +directory. It uses the same pattern format as `.gitignore` (except recursive `**` patterns are not +supported). For example, to exclude a `testdata/` directory and all `.bak` files: @@ -173,9 +190,21 @@ testdata/ *.bak ``` -Files matching these patterns are excluded from `kpt fn render`, `kpt fn source`, `kpt fn eval`, and `kpt live apply`. +Files matching these patterns are excluded from `kpt fn render`, `kpt fn source`, `kpt fn eval`, `kpt live apply`, +`kpt live status`, `kpt live destroy`, `kpt live init`, `kpt live migrate`, and `kpt alpha live plan`. Each subpackage can have its own `.krmignore` that applies only within that subpackage. +### Non-KRM files without `.krmignore` + +If a package contains a non-KRM file (category 3 above) that isn't excluded by `.krmignore`, the commands +listed above fail before doing any work, listing every such file found across the whole package tree in +one error. This applies to files that parse as YAML but aren't valid KRM; a file that isn't valid YAML at +all fails differently and isn't covered by this consolidated reporting or by the annotation below -- see +["Malformed YAML is different, and isn't covered"]({{% relref "/reference/annotations/auto-ignore-non-krm#malformed-yaml-is-different-and-isnt-covered" %}}) +for details. If you'd rather non-KRM files be skipped automatically instead of maintaining a `.krmignore` +list by hand, see the [`kpt.dev/auto-ignore-non-krm`]({{% relref "/reference/annotations/auto-ignore-non-krm" %}}) +annotation. + ## Editing a package kpt does not maintain any state on your local machine outside the directory from where you fetched the package. Making changes to the package is achieved by manipulating the local filesystem. At the lowest level, _editing_ a package is simply a process that does one of the following: diff --git a/documentation/content/en/reference/annotations/_index.md b/documentation/content/en/reference/annotations/_index.md index bc549484b2..7be50341c3 100644 --- a/documentation/content/en/reference/annotations/_index.md +++ b/documentation/content/en/reference/annotations/_index.md @@ -16,6 +16,7 @@ The following annotations are used to configure kpt features: | [config.kubernetes.io/depends-on] | specifies one or more resource dependencies | | [config.kubernetes.io/apply-time-mutation] | specifies one or more substitutions to make at apply time using dependencies as input | | [config.kubernetes.io/local-config] | specifies a resource to be skipped when applying | +| [kpt.dev/auto-ignore-non-krm] | automatically skip non-KRM files in a package instead of failing | The following annotations are used by kpt internally: @@ -42,3 +43,4 @@ For `kpt fn eval DIR`, the path is relative to the package directory DIR [config.kubernetes.io/depends-on]: /reference/annotations/depends-on/ [config.kubernetes.io/apply-time-mutation]: /reference/annotations/apply-time-mutation/ [config.kubernetes.io/local-config]: /reference/annotations/local-config/ +[kpt.dev/auto-ignore-non-krm]: /reference/annotations/auto-ignore-non-krm/ diff --git a/documentation/content/en/reference/annotations/auto-ignore-non-krm/_index.md b/documentation/content/en/reference/annotations/auto-ignore-non-krm/_index.md new file mode 100644 index 0000000000..c99875cbd7 --- /dev/null +++ b/documentation/content/en/reference/annotations/auto-ignore-non-krm/_index.md @@ -0,0 +1,185 @@ +--- +title: "`auto-ignore-non-krm`" +linkTitle: "auto-ignore-non-krm" + +description: | + Automatically skip non-KRM files in a package instead of failing. +--- + +The `kpt.dev/auto-ignore-non-krm` annotation tells kpt to skip any +non-KRM files in a package automatically, instead of failing with an +error. + +A non-KRM file is a `.yaml`/`.yml`/`.json` file (or `Kptfile`-named +file) whose content is not a valid KRM resource: it is missing +`apiVersion`, `kind`, or `metadata.name` (list-kind resources such as +`ConfigMapList` are exempt from the `metadata.name` requirement, and +`kustomization.yaml` files are exempt entirely). + +[`.krmignore`]({{% relref "/book/03-packages#excluding-files-with-krmignore" %}}) +already lets you exclude specific files or patterns by hand. This +annotation is for when you'd rather not maintain that list manually: +any non-KRM file found is skipped the same way, without needing to +know its name or path ahead of time. + +### Malformed YAML is different, and isn't covered + +This annotation only applies to files that parse as valid YAML but +aren't a valid KRM resource -- the definition above. A file that +isn't valid YAML at all (invalid syntax) is a different failure that +this annotation does not excuse: parsing fails before kpt ever checks +whether the content looks like a KRM resource, so there's no non-KRM +classification to skip. This is deliberate. The annotation exists to +let genuinely non-KRM files through, not to hide files that look like +KRM but are broken. + +In practice: the error looks different (a raw parse error, not +`resource must have` apiVersion/kind/name), it doesn't mention this +annotation as a fix, and if a package has multiple malformed files, +only the first one encountered is reported, not the consolidated +"every non-KRM file across the tree" report described below. + +[`.krmignore`]({{% relref "/book/03-packages#excluding-files-with-krmignore" %}}) +does work for a malformed file, because it excludes the file before +anything is parsed. It ignores any file you list, whether or not the +file is KRM. To work around a malformed file, fix its syntax, remove +it, or list it in `.krmignore`. + +### Schema + +The annotation value accepts string values of `true` and `false`. + +Make sure to surround the value in quotes, otherwise it will be +considered a YAML boolean (invalid annotation), not a string. + +### Behavior + +Set on a package's `Kptfile`, in `metadata.annotations`: + +```yaml +apiVersion: kpt.dev/v1 +kind: Kptfile +metadata: + name: my-package + annotations: + kpt.dev/auto-ignore-non-krm: "true" +``` + +With the annotation absent (or set to `false`) and a non-KRM file +present, the command fails before doing any work, listing every +non-KRM file found across the whole package tree in one error: + +``` +Error: input resource list must contain only KRM resources (1 file(s)): + values.yaml: resource must have `apiVersion` +To resolve: either exclude these files via .krmignore (manually, or with a +pattern rule), or add `kpt.dev/auto-ignore-non-krm: "true"` to this +package's Kptfile to skip any non-KRM files automatically. +``` + +With the annotation set to `"true"`, the same command instead skips +the file and proceeds, printing which files were skipped: + +``` +[info] ignoring 1 non-KRM file(s) in "my-package" due to kpt.dev/auto-ignore-non-krm annotation: values.yaml +``` + +The non-KRM file is skipped in memory for that invocation only. +Nothing is written to disk because of this annotation: the file +itself is left completely untouched, and no `.krmignore` file is ever +created or modified. + +#### Scoping: per package, like `.krmignore` + +The annotation only covers the package whose own `Kptfile` carries +it. It is **not** inherited by subpackages, and a subpackage's +annotation does not cover its parent, the same way a `.krmignore` +file only applies within the package directory it's placed in. + +For example, given a package `parent` with a subpackage +`parent/sub`, where `sub` contains a non-KRM file: + +``` +parent/ +├── Kptfile +├── deployment.yaml +└── sub/ + ├── Kptfile + ├── deployment.yaml + └── values.yaml # non-KRM +``` + +Adding the annotation to `parent/Kptfile` does **not** excuse +`sub/values.yaml`; rendering still fails, naming `values.yaml`. The +annotation has to be added to `sub/Kptfile` itself, the package that +actually contains the non-KRM file, for that file to be skipped. + +#### Supported commands + +| Command | Supported | +| ------- | --------- | +| `kpt fn render` | Yes | +| `kpt fn eval` | Yes | +| `kpt fn source` | Yes | +| `kpt live init` | Yes | +| `kpt live apply` | Yes | +| `kpt live status` | Yes | +| `kpt live destroy` | Yes | +| `kpt alpha live plan` | Yes | +| `kpt live migrate` | No. The legacy ConfigMap-based inventory reader this command uses to support migrating from older `kpt` versions has no way to skip a file in memory, so a non-KRM file always blocks `migrate`, with or without this annotation. Use `.krmignore` instead. | + +`kpt fn render` additionally records which files were skipped in the +root `Kptfile`'s `status.conditions` after a successful render; see +the [`fn render` Render Status Conditions]({{% relref "/reference/cli/fn/render#render-status-conditions" %}}) +section. + +### Example + +```shell +kpt pkg init my-pkg +cd my-pkg +``` + +```shell +cat > values.yaml << EOF +replicaCount: 3 +EOF +``` + +Rendering now fails, since `values.yaml` is not a valid KRM resource: + +```shell +kpt fn render +``` + +``` +Error: input resource list must contain only KRM resources (1 file(s)): + values.yaml: resource must have `apiVersion` +... +``` + +Open the Kptfile and add the annotation under `metadata.annotations` (`kpt pkg init` already added an +`annotations` key for `config.kubernetes.io/local-config`, so add the new line alongside it): + +```yaml +apiVersion: kpt.dev/v1 +kind: Kptfile +metadata: + name: my-pkg + annotations: + kpt.dev/auto-ignore-non-krm: "true" + config.kubernetes.io/local-config: "true" +info: + description: sample description +``` + +Rendering now succeeds, skipping `values.yaml`: + +```shell +kpt fn render +``` + +``` +[info] ignoring 1 non-KRM file(s) in "my-pkg" due to kpt.dev/auto-ignore-non-krm annotation: values.yaml +Successfully executed 0 function(s) in 1 package(s). +``` diff --git a/documentation/content/en/reference/cli/alpha/live/plan/_index.md b/documentation/content/en/reference/cli/alpha/live/plan/_index.md index 1e45bd7dad..30f1efabe8 100644 --- a/documentation/content/en/reference/cli/alpha/live/plan/_index.md +++ b/documentation/content/en/reference/cli/alpha/live/plan/_index.md @@ -20,6 +20,12 @@ overview of the impact of applying a package. Note that `plan` does only works reliably with server-side apply. +If the package contains a non-KRM file (not excluded by +[`.krmignore`]({{% relref "/book/03-packages#excluding-files-with-krmignore" %}})), `plan` fails, listing +every such file found. See the +[`kpt.dev/auto-ignore-non-krm`]({{% relref "/reference/annotations/auto-ignore-non-krm" %}}) annotation to +skip these files automatically instead. + ### Synopsis diff --git a/documentation/content/en/reference/cli/fn/eval/_index.md b/documentation/content/en/reference/cli/fn/eval/_index.md index a3a6588172..97dc51d381 100644 --- a/documentation/content/en/reference/cli/fn/eval/_index.md +++ b/documentation/content/en/reference/cli/fn/eval/_index.md @@ -16,6 +16,12 @@ as container images. If the function fails (i.e. exits with non-zero status code), `eval` will abort and the local filesystem is left intact. +If the target directory contains a non-KRM file (not excluded by +[`.krmignore`]({{% relref "/book/03-packages#excluding-files-with-krmignore" %}})), `eval` fails before +running the function, listing every such file found. See the +[`kpt.dev/auto-ignore-non-krm`]({{% relref "/reference/annotations/auto-ignore-non-krm" %}}) annotation to +skip these files automatically instead. + Refer to the [Imperative Function Execution] for detailed overview. ### Synopsis diff --git a/documentation/content/en/reference/cli/fn/render/_index.md b/documentation/content/en/reference/cli/fn/render/_index.md index 3661834620..1100eff522 100644 --- a/documentation/content/en/reference/cli/fn/render/_index.md +++ b/documentation/content/en/reference/cli/fn/render/_index.md @@ -24,6 +24,12 @@ from A is then written to the local filesystem in-place. If any of the functions in the pipeline fails, then the entire pipeline is aborted and the local filesystem is left intact. +If a package contains a non-KRM file (not excluded by +[`.krmignore`]({{% relref "/book/03-packages#excluding-files-with-krmignore" %}})), `render` fails before +running any function, listing every such file found across the whole package tree. See the +[`kpt.dev/auto-ignore-non-krm`]({{% relref "/reference/annotations/auto-ignore-non-krm" %}}) annotation to +skip these files automatically instead. + Refer to the [Declarative Functions Execution] for more details. ### Synopsis @@ -115,7 +121,10 @@ the last render succeeded or failed. #### Behavior - **Success**: When rendering completes successfully, the Kptfile receives a condition with `type: Rendered`, -`status: "True"`, and `reason: RenderSuccess`. +`status: "True"`, and `reason: RenderSuccess`. If any non-KRM files were skipped because of the +[`kpt.dev/auto-ignore-non-krm`]({{% relref "/reference/annotations/auto-ignore-non-krm" %}}) annotation, the +condition also gets a `message` field naming them, so that fact is recorded even after this invocation's +terminal output is gone. - **Failure**: When rendering fails, the Kptfile receives a condition with `type: Rendered`, `status: "False"`, `reason: RenderFailed`, and a `message` field containing the error details. - **In-place render only**: The status condition is written only when rendering in-place (the default behavior). @@ -140,6 +149,26 @@ status: reason: RenderSuccess ``` +If the package's Kptfile carries the `kpt.dev/auto-ignore-non-krm` annotation and non-KRM files were skipped, +the condition instead looks like: + +```yaml +apiVersion: kpt.dev/v1 +kind: Kptfile +metadata: + name: my-package + annotations: + kpt.dev/auto-ignore-non-krm: "true" +status: + conditions: + - type: Rendered + status: "True" + reason: RenderSuccess + message: |- + These non-KRM resources were ignored during render due to the kpt.dev/auto-ignore-non-krm annotation + being present on this package's Kptfile: values.yaml +``` + #### Failure Example After a failed render, the root Kptfile will contain: diff --git a/documentation/content/en/reference/cli/fn/source/_index.md b/documentation/content/en/reference/cli/fn/source/_index.md index 59909b4e13..fcb563180e 100644 --- a/documentation/content/en/reference/cli/fn/source/_index.md +++ b/documentation/content/en/reference/cli/fn/source/_index.md @@ -16,6 +16,12 @@ the `source` can be pipe'd to commands such as `kpt fn eval` that accepts Functi is useful for chaining functions using Unix pipe. For more details, refer to [Chaining functions]({{% relref "/book/04-using-functions#chaining-functions-using-the-unix-pipe" %}}). +If the target directory contains a non-KRM file (not excluded by +[`.krmignore`]({{% relref "/book/03-packages#excluding-files-with-krmignore" %}})), `source` fails, listing +every such file found. See the +[`kpt.dev/auto-ignore-non-krm`]({{% relref "/reference/annotations/auto-ignore-non-krm" %}}) annotation to +skip these files automatically instead. + ### Synopsis diff --git a/documentation/content/en/reference/cli/live/apply/_index.md b/documentation/content/en/reference/cli/live/apply/_index.md index f0ed18aeb2..9b20c5e6f2 100644 --- a/documentation/content/en/reference/cli/live/apply/_index.md +++ b/documentation/content/en/reference/cli/live/apply/_index.md @@ -13,6 +13,12 @@ description: | `apply` creates, updates and deletes resources in the cluster to make the remote cluster resources match the local package configuration. +If the package contains a non-KRM file (not excluded by +[`.krmignore`]({{% relref "/book/03-packages#excluding-files-with-krmignore" %}})), `apply` fails before +making any cluster changes, listing every such file found. See the +[`kpt.dev/auto-ignore-non-krm`]({{% relref "/reference/annotations/auto-ignore-non-krm" %}}) annotation to +skip these files automatically instead. + ### Synopsis diff --git a/documentation/content/en/reference/cli/live/destroy/_index.md b/documentation/content/en/reference/cli/live/destroy/_index.md index 065006eb23..07f6e2056c 100644 --- a/documentation/content/en/reference/cli/live/destroy/_index.md +++ b/documentation/content/en/reference/cli/live/destroy/_index.md @@ -12,6 +12,12 @@ description: | `destroy` removes all files belonging to a package from the cluster. +If the package contains a non-KRM file (not excluded by +[`.krmignore`]({{% relref "/book/03-packages#excluding-files-with-krmignore" %}})), `destroy` fails before +making any cluster changes, listing every such file found. See the +[`kpt.dev/auto-ignore-non-krm`]({{% relref "/reference/annotations/auto-ignore-non-krm" %}}) annotation to +skip these files automatically instead. + ### Synopsis diff --git a/documentation/content/en/reference/cli/live/init/_index.md b/documentation/content/en/reference/cli/live/init/_index.md index c33c47aa8c..7420e25a14 100644 --- a/documentation/content/en/reference/cli/live/init/_index.md +++ b/documentation/content/en/reference/cli/live/init/_index.md @@ -13,6 +13,12 @@ description: | `init` initializes the package with the name, namespace and id of the resource that will keep track of the package inventory. +If the package contains a non-KRM file (not excluded by +[`.krmignore`]({{% relref "/book/03-packages#excluding-files-with-krmignore" %}})), `init` fails, listing +every such file found. See the +[`kpt.dev/auto-ignore-non-krm`]({{% relref "/reference/annotations/auto-ignore-non-krm" %}}) annotation to +skip these files automatically instead. + ### Synopsis diff --git a/documentation/content/en/reference/cli/live/migrate/_index.md b/documentation/content/en/reference/cli/live/migrate/_index.md index cef10c5c54..9e5d7379a2 100644 --- a/documentation/content/en/reference/cli/live/migrate/_index.md +++ b/documentation/content/en/reference/cli/live/migrate/_index.md @@ -25,6 +25,12 @@ Previous 1.0.0-alpha or 1.0.0-beta versions of kpt would store the inventory metadata in the `Kptfile`. Running this command will move the metadata from the `Kptfile` in a `ResourceGroup` manifest in the `resourcegroup.yaml` file. +If the package contains a non-KRM file (not excluded by +[`.krmignore`]({{% relref "/book/03-packages#excluding-files-with-krmignore" %}})), `migrate` fails before +making any changes, listing every such file found. Unlike the other `live` commands, the +[`kpt.dev/auto-ignore-non-krm`]({{% relref "/reference/annotations/auto-ignore-non-krm" %}}) annotation does +**not** unblock `migrate`: the legacy ConfigMap-based inventory reader it uses has no way to skip a file in +memory, so `.krmignore` is the only way to exclude a non-KRM file for this command. ### Synopsis diff --git a/documentation/content/en/reference/cli/live/status/_index.md b/documentation/content/en/reference/cli/live/status/_index.md index 8a767fb84f..f451ec9e53 100644 --- a/documentation/content/en/reference/cli/live/status/_index.md +++ b/documentation/content/en/reference/cli/live/status/_index.md @@ -12,6 +12,12 @@ description: | `status` shows the resource status for resources belonging to the package. +If the package contains a non-KRM file (not excluded by +[`.krmignore`]({{% relref "/book/03-packages#excluding-files-with-krmignore" %}})), `status` fails, listing +every such file found. See the +[`kpt.dev/auto-ignore-non-krm`]({{% relref "/reference/annotations/auto-ignore-non-krm" %}}) annotation to +skip these files automatically instead. + ### Synopsis diff --git a/documentation/content/en/reference/cli/pkg/update/_index.md b/documentation/content/en/reference/cli/pkg/update/_index.md index 6b0c59156e..9257b0e7af 100644 --- a/documentation/content/en/reference/cli/pkg/update/_index.md +++ b/documentation/content/en/reference/cli/pkg/update/_index.md @@ -16,6 +16,12 @@ may be applied using one of several strategies. Since this will update the local package, all changes must be committed to git before running `update`. +A `.yaml`/`.yml`/`Kptfile`-named file whose content is not a valid KRM resource (e.g. a Helm +`values.yaml`) is never merged field-by-field, regardless of its filename. It is classified by its +content, the same way `.krmignore`-aware commands do, and handled like any other non-KRM file: upstream's +version is synced in only if the local file is unmodified; a locally modified or local-only file is +always kept as-is, even if upstream also changed it. + > **Want to understand 3-way merge in detail?** See the [3-Way Merge Guide]({{% relref "/guides/3-way-merge" %}}) for comprehensive documentation on how kpt merges packages, merge strategies, conflict resolution, and best practices. ### Synopsis diff --git a/e2e/testdata/fn-eval/non-krm-resource/.expected/config.yaml b/e2e/testdata/fn-eval/non-krm-resource/.expected/config.yaml index 5553beb38b..d57baceb67 100644 --- a/e2e/testdata/fn-eval/non-krm-resource/.expected/config.yaml +++ b/e2e/testdata/fn-eval/non-krm-resource/.expected/config.yaml @@ -18,5 +18,5 @@ args: namespace: staging exitCode: 1 stdErr: | - [FAIL] "ghcr.io/kptdev/krm-functions-catalog/set-namespace:latest" in 0s - error: input resource list must contain only KRM resources: nonkrm.yaml: resource must have `apiVersion` + error: input resource list must contain only KRM resources (1 file(s)): + nonkrm.yaml: resource must have `apiVersion` diff --git a/e2e/testdata/fn-render/non-krm-resource-no-pipeline/.expected/config.yaml b/e2e/testdata/fn-render/non-krm-resource-no-pipeline/.expected/config.yaml index 25f612255e..5e58a4dbf8 100644 --- a/e2e/testdata/fn-render/non-krm-resource-no-pipeline/.expected/config.yaml +++ b/e2e/testdata/fn-render/non-krm-resource-no-pipeline/.expected/config.yaml @@ -13,4 +13,4 @@ # limitations under the License. exitCode: 1 -stdErr: "Error: input resource list must contain only KRM resources: non-krm.yaml: resource must have `apiVersion`" \ No newline at end of file +stdErr: "Error: input resource list must contain only KRM resources (1 file(s)):\n non-krm.yaml: resource must have `apiVersion`" \ No newline at end of file diff --git a/e2e/testdata/fn-render/non-krm-resource-no-pipeline/.expected/diff.patch b/e2e/testdata/fn-render/non-krm-resource-no-pipeline/.expected/diff.patch index 16b239070d..98aeb9cbda 100644 --- a/e2e/testdata/fn-render/non-krm-resource-no-pipeline/.expected/diff.patch +++ b/e2e/testdata/fn-render/non-krm-resource-no-pipeline/.expected/diff.patch @@ -1,8 +1,8 @@ diff --git a/Kptfile b/Kptfile -index d9e2f05..775d110 100644 +index d9e2f05..04e4264 100644 --- a/Kptfile +++ b/Kptfile -@@ -2,3 +2,9 @@ apiVersion: kpt.dev/v1 +@@ -2,3 +2,12 @@ apiVersion: kpt.dev/v1 kind: Kptfile metadata: name: app @@ -11,4 +11,7 @@ index d9e2f05..775d110 100644 + - type: Rendered + status: "False" + reason: RenderFailed -+ message: 'pkg.render: pkg .: input resource list must contain only KRM resources: non-krm.yaml: resource must have `apiVersion`' ++ message: |- ++ input resource list must contain only KRM resources (1 file(s)): ++ non-krm.yaml: resource must have `apiVersion` ++ To resolve: either exclude these files via .krmignore (manually, or with a pattern rule), or add `kpt.dev/auto-ignore-non-krm: "true"` to this package's Kptfile to skip any non-KRM files automatically from this package. diff --git a/e2e/testdata/fn-render/non-krm-resource/.expected/config.yaml b/e2e/testdata/fn-render/non-krm-resource/.expected/config.yaml index 25f612255e..5e58a4dbf8 100644 --- a/e2e/testdata/fn-render/non-krm-resource/.expected/config.yaml +++ b/e2e/testdata/fn-render/non-krm-resource/.expected/config.yaml @@ -13,4 +13,4 @@ # limitations under the License. exitCode: 1 -stdErr: "Error: input resource list must contain only KRM resources: non-krm.yaml: resource must have `apiVersion`" \ No newline at end of file +stdErr: "Error: input resource list must contain only KRM resources (1 file(s)):\n non-krm.yaml: resource must have `apiVersion`" \ No newline at end of file diff --git a/e2e/testdata/fn-render/non-krm-resource/.expected/diff.patch b/e2e/testdata/fn-render/non-krm-resource/.expected/diff.patch index 57838e4c3d..68582ceff0 100644 --- a/e2e/testdata/fn-render/non-krm-resource/.expected/diff.patch +++ b/e2e/testdata/fn-render/non-krm-resource/.expected/diff.patch @@ -1,8 +1,8 @@ diff --git a/Kptfile b/Kptfile -index ae603c9..38d3a2a 100644 +index ae603c9..8d7e65d 100644 --- a/Kptfile +++ b/Kptfile -@@ -10,3 +10,17 @@ pipeline: +@@ -10,3 +10,12 @@ pipeline: - image: ghcr.io/kptdev/krm-functions-catalog/set-labels:latest configMap: tier: backend @@ -12,11 +12,6 @@ index ae603c9..38d3a2a 100644 + status: "False" + reason: RenderFailed + message: |- -+ pkg.render: pkg .: -+ pipeline.run: input resource list must contain only KRM resources: non-krm.yaml: resource must have `apiVersion` -+ renderStatus: -+ mutationSteps: -+ - image: ghcr.io/kptdev/krm-functions-catalog/set-namespace:latest -+ executionError: 'input resource list must contain only KRM resources: non-krm.yaml: resource must have `apiVersion`' -+ exitCode: 1 -+ errorSummary: 'ghcr.io/kptdev/krm-functions-catalog/set-namespace:latest: input resource list must contain only KRM resources: non-krm.yaml: resource must have `apiVersion`' ++ input resource list must contain only KRM resources (1 file(s)): ++ non-krm.yaml: resource must have `apiVersion` ++ To resolve: either exclude these files via .krmignore (manually, or with a pattern rule), or add `kpt.dev/auto-ignore-non-krm: "true"` to this package's Kptfile to skip any non-KRM files automatically from this package. diff --git a/e2e/testdata/fn-render/subpkg-has-invalid-kptfile/.expected/diff.patch b/e2e/testdata/fn-render/subpkg-has-invalid-kptfile/.expected/diff.patch index edbf350762..0ed8a025d3 100644 --- a/e2e/testdata/fn-render/subpkg-has-invalid-kptfile/.expected/diff.patch +++ b/e2e/testdata/fn-render/subpkg-has-invalid-kptfile/.expected/diff.patch @@ -1,8 +1,8 @@ diff --git a/Kptfile b/Kptfile -index ae603c9..98eeb85 100644 +index ae603c9..cdbf71b 100644 --- a/Kptfile +++ b/Kptfile -@@ -10,3 +10,11 @@ pipeline: +@@ -10,3 +10,9 @@ pipeline: - image: ghcr.io/kptdev/krm-functions-catalog/set-labels:latest configMap: tier: backend @@ -11,6 +11,4 @@ index ae603c9..98eeb85 100644 + - type: Rendered + status: "False" + reason: RenderFailed -+ message: |- -+ pkg.render: pkg .: -+ pkg.Subpackages: pkg ./db: error reading Kptfile at "./db": yaml: line 10: mapping values are not allowed in this context ++ message: 'non-krm pre-scan: failed to read package at .: pkg.Subpackages: pkg ./db: error reading Kptfile at "./db": yaml: line 10: mapping values are not allowed in this context' diff --git a/pkg/fn/runtime/runner.go b/pkg/fn/runtime/runner.go index faf8b71d38..722d87b50e 100644 --- a/pkg/fn/runtime/runner.go +++ b/pkg/fn/runtime/runner.go @@ -33,6 +33,7 @@ import ( "github.com/kptdev/kpt/pkg/fn" "github.com/kptdev/kpt/pkg/lib/builtins" "github.com/kptdev/kpt/pkg/lib/errors" + "github.com/kptdev/kpt/pkg/lib/krmvalidate" "github.com/kptdev/kpt/pkg/lib/pkg" "github.com/kptdev/kpt/pkg/lib/util/shlex" "github.com/kptdev/kpt/pkg/printer" @@ -375,7 +376,7 @@ func (fr *FunctionRunner) SetFnConfig(conf *yaml.RNode) { // do executes the kpt function and returns the modified resources. // fnResult is updated with the function results returned by the kpt function. func (fr *FunctionRunner) do(input []*yaml.RNode) (output []*yaml.RNode, err error) { - if krmErr := kptfilev1.AreKRM(input); krmErr != nil { + if krmErr := krmvalidate.AreKRM(input); krmErr != nil { return output, fmt.Errorf("input resource list must contain only KRM resources: %w", krmErr) } @@ -390,7 +391,7 @@ func (fr *FunctionRunner) do(input []*yaml.RNode) (output []*yaml.RNode, err err if pathErr := enforcePathInvariants(output); pathErr != nil { return output, pathErr } - if krmErr := kptfilev1.AreKRM(output); krmErr != nil { + if krmErr := krmvalidate.AreKRM(output); krmErr != nil { return output, fmt.Errorf("output resource list must contain only KRM resources: %w", krmErr) } diff --git a/pkg/fn/runtime/runner_test.go b/pkg/fn/runtime/runner_test.go index efcc553c88..9208aa4dc2 100644 --- a/pkg/fn/runtime/runner_test.go +++ b/pkg/fn/runtime/runner_test.go @@ -19,6 +19,7 @@ package runtime import ( "bytes" "context" + "io" "os" "path/filepath" "strings" @@ -33,6 +34,7 @@ import ( "github.com/stretchr/testify/require" "sigs.k8s.io/kustomize/kyaml/filesys" "sigs.k8s.io/kustomize/kyaml/fn/framework" + "sigs.k8s.io/kustomize/kyaml/fn/runtime/runtimeutil" "sigs.k8s.io/kustomize/kyaml/kio" "sigs.k8s.io/kustomize/kyaml/yaml" ) @@ -1121,3 +1123,35 @@ func TestPrintFnResult(t *testing.T) { }) } } + +// TestFunctionRunner_do_AcceptsNamelessList: a nameless *List-kind +// resource (e.g. ConfigMapList) must pass do()'s validation on both +// the input and output side, matching krmvalidate.IsKRM's exemption +// rather than the stricter, name-requiring validator this used before. +func TestFunctionRunner_do_AcceptsNamelessList(t *testing.T) { + input := `apiVersion: v1 +kind: ConfigMapList +items: [] +` + r := kio.ByteReadWriter{ + Reader: bytes.NewBufferString(input), + WrapBareSeqNode: true, + } + nodes, err := r.Read() + require.NoError(t, err) + + fr := &FunctionRunner{ + filter: &runtimeutil.FunctionFilter{ + // Pass-through: echoes the wrapped ResourceList back unchanged. + Run: func(reader io.Reader, writer io.Writer) error { + _, err := io.Copy(writer, reader) + return err + }, + }, + fnResult: &fnresultv1.Result{}, + fnResults: fnresultv1.NewResultList(), + } + + _, err = fr.do(nodes) + assert.NoError(t, err, "a nameless *List resource must not be rejected as non-KRM") +} diff --git a/pkg/lib/kptops/autoignore_test.go b/pkg/lib/kptops/autoignore_test.go new file mode 100644 index 0000000000..4401e28abf --- /dev/null +++ b/pkg/lib/kptops/autoignore_test.go @@ -0,0 +1,305 @@ +// Copyright 2026 The kpt Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kptops + +import ( + "bytes" + "context" + "path/filepath" + "testing" + + kptfilev1 "github.com/kptdev/kpt/api/kptfile/v1" + "github.com/kptdev/kpt/api/runneroptions" + "github.com/kptdev/kpt/pkg/kptfile/kptfileutil" + "github.com/kptdev/kpt/pkg/lib/krmvalidate" + "github.com/kptdev/kpt/pkg/printer" + "github.com/stretchr/testify/assert" + "sigs.k8s.io/kustomize/kyaml/filesys" +) + +// writeKptfile writes a minimal Kptfile at dir, optionally with the +// krmvalidate.AutoIgnoreNonKRMAnnotation set to "true" when optedIn is true, and +// optionally with a single mutator step when withMutator is true (using +// mockFnRuntime, so no real container/exec ever runs). +func writeKptfile(t *testing.T, fsys filesys.FileSystem, dir, name string, optedIn, withMutator bool) { + t.Helper() + annotations := "" + if optedIn { + annotations = " annotations:\n " + krmvalidate.AutoIgnoreNonKRMAnnotation + ": \"true\"\n" + } + pipeline := "" + if withMutator { + pipeline = "pipeline:\n mutators:\n - image: example.com/mutate:v1\n" + } + content := "apiVersion: kpt.dev/v1\nkind: Kptfile\nmetadata:\n name: " + name + "\n" + annotations + pipeline + assert.NoError(t, fsys.WriteFile(filepath.Join(dir, "Kptfile"), []byte(content))) +} + +func writeValidKRM(t *testing.T, fsys filesys.FileSystem, dir, resourceName string) { + t.Helper() + content := "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: " + resourceName + "\n" + assert.NoError(t, fsys.WriteFile(filepath.Join(dir, "cm.yaml"), []byte(content))) +} + +// writeNonKRM writes a YAML file with no apiVersion/kind/metadata.name. +func writeNonKRM(t *testing.T, fsys filesys.FileSystem, dir, fileName string) { + t.Helper() + assert.NoError(t, fsys.WriteFile(filepath.Join(dir, fileName), []byte("not: krm\njust: [a, plain, map]\nwithout: apiVersion-or-kind\n"))) +} + +// newAutoignoreTestRenderer builds a Renderer rooted at "/root", the +// path every test in this file uses. +func newAutoignoreTestRenderer(fsys filesys.FileSystem, outBuf *bytes.Buffer, mock *mockFnRuntime) *Renderer { + opts := runneroptions.RunnerOptions{} + opts.InitDefaults(runneroptions.GHCRImagePrefix) + return &Renderer{ + PkgPath: "/root", + FileSystem: fsys, + Output: outBuf, + Runtime: mock, + RunnerOptions: opts, + } +} + +// TestNonKRM_NoAnnotation_BlocksEntireRender verifies a non-KRM file +// with no opt-in annotation fails the whole render, and that package's +// mutator never runs at all. +func TestNonKRM_NoAnnotation_BlocksEntireRender(t *testing.T) { + fsys := filesys.MakeFsInMemory() + root := "/root" + assert.NoError(t, fsys.Mkdir(root)) + writeKptfile(t, fsys, root, "root-pkg", false, true) + writeValidKRM(t, fsys, root, "root-cm") + writeNonKRM(t, fsys, root, "notes.yaml") + + var out bytes.Buffer + mock := &mockFnRuntime{} + ctx := printer.WithContext(context.Background(), printer.New(&out, &out)) + r := newAutoignoreTestRenderer(fsys, &out, mock) + + _, err := r.Execute(ctx) + + assert.Error(t, err) + assert.Contains(t, err.Error(), "input resource list must contain only KRM resources") + assert.Contains(t, err.Error(), "notes.yaml") + // The mutator must never have been invoked. + assert.Empty(t, mock.capturedItems, "mutator ran even though the package had a non-KRM file with no opt-in annotation") +} + +// TestNonKRM_ConsolidatedError_ListsEveryFile verifies non-KRM files +// across multiple packages are all reported in a single error. +func TestNonKRM_ConsolidatedError_ListsEveryFile(t *testing.T) { + fsys := filesys.MakeFsInMemory() + root := "/root" + sub := "/root/subpkg" + assert.NoError(t, fsys.Mkdir(root)) + assert.NoError(t, fsys.Mkdir(sub)) + writeKptfile(t, fsys, root, "root-pkg", false, false) + writeValidKRM(t, fsys, root, "root-cm") + writeNonKRM(t, fsys, root, "root-notes.yaml") + writeKptfile(t, fsys, sub, "sub-pkg", false, false) + writeValidKRM(t, fsys, sub, "sub-cm") + writeNonKRM(t, fsys, sub, "sub-notes.yaml") + + var out bytes.Buffer + mock := &mockFnRuntime{} + ctx := printer.WithContext(context.Background(), printer.New(&out, &out)) + r := newAutoignoreTestRenderer(fsys, &out, mock) + + _, err := r.Execute(ctx) + + assert.Error(t, err) + assert.Contains(t, err.Error(), "root-notes.yaml") + assert.Contains(t, err.Error(), "sub-notes.yaml") + assert.Contains(t, err.Error(), "2 file(s)") +} + +// TestNonKRM_AnnotatedPackage_SkipsInMemory_NoDiskWrite verifies an +// opted-in package renders successfully with the non-KRM file excluded +// from the pipeline's input and left untouched on disk. +func TestNonKRM_AnnotatedPackage_SkipsInMemory_NoDiskWrite(t *testing.T) { + fsys := filesys.MakeFsInMemory() + root := "/root" + assert.NoError(t, fsys.Mkdir(root)) + writeKptfile(t, fsys, root, "root-pkg", true, true) + writeValidKRM(t, fsys, root, "root-cm") + writeNonKRM(t, fsys, root, "notes.yaml") + + var out bytes.Buffer + mock := &mockFnRuntime{} + ctx := printer.WithContext(context.Background(), printer.New(&out, &out)) + r := newAutoignoreTestRenderer(fsys, &out, mock) + + _, err := r.Execute(ctx) + + assert.NoError(t, err) + // The mutator ran, and the only items it saw were the Kptfile itself + // and the valid ConfigMap -- the non-KRM file never reached the + // pipeline (LocalResources() always includes the package's own + // Kptfile as a KRM resource; that's expected, unrelated behavior). + assert.Len(t, mock.capturedItems, 1) + assert.ElementsMatch(t, []string{"root-pkg", "root-cm"}, mock.capturedItems[0]) + + // The non-KRM file is untouched on disk. + notesContent, readErr := fsys.ReadFile(filepath.Join(root, "notes.yaml")) + assert.NoError(t, readErr) + assert.Contains(t, string(notesContent), "not: krm") + + // .krmignore was never created. + assert.False(t, fsys.Exists(filepath.Join(root, ".krmignore")), ".krmignore should never be written by the annotation path") + + // The render output mentions the skip. + assert.Contains(t, out.String(), "notes.yaml") + assert.Contains(t, out.String(), krmvalidate.AutoIgnoreNonKRMAnnotation) +} + +// TestNonKRM_AnnotationNotInherited verifies the annotation is scoped +// to the package whose own Kptfile carries it, not inherited. +func TestNonKRM_AnnotationNotInherited(t *testing.T) { + t.Run("parent annotation does not cover child's non-KRM file", func(t *testing.T) { + fsys := filesys.MakeFsInMemory() + root := "/root" + sub := "/root/subpkg" + assert.NoError(t, fsys.Mkdir(root)) + assert.NoError(t, fsys.Mkdir(sub)) + writeKptfile(t, fsys, root, "root-pkg", true, false) // opted in + writeValidKRM(t, fsys, root, "root-cm") + writeKptfile(t, fsys, sub, "sub-pkg", false, false) // NOT opted in + writeValidKRM(t, fsys, sub, "sub-cm") + writeNonKRM(t, fsys, sub, "sub-notes.yaml") + + var out bytes.Buffer + mock := &mockFnRuntime{} + ctx := printer.WithContext(context.Background(), printer.New(&out, &out)) + r := newAutoignoreTestRenderer(fsys, &out, mock) + + _, err := r.Execute(ctx) + + assert.Error(t, err) + assert.Contains(t, err.Error(), "sub-notes.yaml") + }) + + t.Run("child annotation does not cover parent's non-KRM file", func(t *testing.T) { + fsys := filesys.MakeFsInMemory() + root := "/root" + sub := "/root/subpkg" + assert.NoError(t, fsys.Mkdir(root)) + assert.NoError(t, fsys.Mkdir(sub)) + writeKptfile(t, fsys, root, "root-pkg", false, false) // NOT opted in + writeValidKRM(t, fsys, root, "root-cm") + writeNonKRM(t, fsys, root, "root-notes.yaml") + writeKptfile(t, fsys, sub, "sub-pkg", true, false) // opted in + writeValidKRM(t, fsys, sub, "sub-cm") + + var out bytes.Buffer + mock := &mockFnRuntime{} + ctx := printer.WithContext(context.Background(), printer.New(&out, &out)) + r := newAutoignoreTestRenderer(fsys, &out, mock) + + _, err := r.Execute(ctx) + + assert.Error(t, err) + assert.Contains(t, err.Error(), "root-notes.yaml") + }) + + t.Run("only the annotated package's non-KRM file is excused, others still block", func(t *testing.T) { + fsys := filesys.MakeFsInMemory() + root := "/root" + sub := "/root/subpkg" + assert.NoError(t, fsys.Mkdir(root)) + assert.NoError(t, fsys.Mkdir(sub)) + writeKptfile(t, fsys, root, "root-pkg", false, false) // NOT opted in + writeValidKRM(t, fsys, root, "root-cm") + writeNonKRM(t, fsys, root, "root-notes.yaml") + writeKptfile(t, fsys, sub, "sub-pkg", true, false) // opted in + writeValidKRM(t, fsys, sub, "sub-cm") + writeNonKRM(t, fsys, sub, "sub-notes.yaml") + + var out bytes.Buffer + mock := &mockFnRuntime{} + ctx := printer.WithContext(context.Background(), printer.New(&out, &out)) + r := newAutoignoreTestRenderer(fsys, &out, mock) + + _, err := r.Execute(ctx) + + assert.Error(t, err) + assert.Contains(t, err.Error(), "root-notes.yaml") + // Only one file should be blocking -- sub-notes.yaml's package opted in. + assert.NotContains(t, err.Error(), "sub-notes.yaml") + }) +} + +// TestNonKRM_CleanTree_NoOverheadOnOutcome verifies a tree with no +// non-KRM files renders exactly as before. +func TestNonKRM_CleanTree_NoOverheadOnOutcome(t *testing.T) { + fsys := filesys.MakeFsInMemory() + root := "/root" + assert.NoError(t, fsys.Mkdir(root)) + writeKptfile(t, fsys, root, "root-pkg", false, true) + writeValidKRM(t, fsys, root, "root-cm") + + var out bytes.Buffer + mock := &mockFnRuntime{} + ctx := printer.WithContext(context.Background(), printer.New(&out, &out)) + r := newAutoignoreTestRenderer(fsys, &out, mock) + + _, err := r.Execute(ctx) + + assert.NoError(t, err) + assert.Len(t, mock.capturedItems, 1) + assert.ElementsMatch(t, []string{"root-pkg", "root-cm"}, mock.capturedItems[0]) + assert.NotContains(t, out.String(), krmvalidate.AutoIgnoreNonKRMAnnotation) +} + +// TestNonKRM_AnnotatedPackage_RecordsSkipInKptfileCondition verifies a +// successful, annotation-excused render leaves a message naming the +// skipped file in the Kptfile's status condition. +func TestNonKRM_AnnotatedPackage_RecordsSkipInKptfileCondition(t *testing.T) { + fsys := filesys.MakeFsInMemory() + root := "/root" + assert.NoError(t, fsys.Mkdir(root)) + writeKptfile(t, fsys, root, "root-pkg", true, true) + writeValidKRM(t, fsys, root, "root-cm") + writeNonKRM(t, fsys, root, "notes.yaml") + + var out bytes.Buffer + mock := &mockFnRuntime{} + ctx := printer.WithContext(context.Background(), printer.New(&out, &out)) + opts := runneroptions.RunnerOptions{} + opts.InitDefaults(runneroptions.GHCRImagePrefix) + // Output deliberately nil: the in-place path, the only one that + // writes status.conditions. + r := &Renderer{ + PkgPath: root, + FileSystem: fsys, + Runtime: mock, + RunnerOptions: opts, + } + + _, err := r.Execute(ctx) + assert.NoError(t, err) + + rootKf, readErr := kptfileutil.ReadKptfile(fsys, root) + assert.NoError(t, readErr) + assert.NotNil(t, rootKf.Status) + assert.Len(t, rootKf.Status.Conditions, 1) + cond := rootKf.Status.Conditions[0] + assert.Equal(t, kptfilev1.ConditionTrue, cond.Status) + assert.Equal(t, kptfilev1.ReasonRenderSuccess, cond.Reason) + assert.Contains(t, cond.Message, "notes.yaml") + assert.Contains(t, cond.Message, krmvalidate.AutoIgnoreNonKRMAnnotation) +} + +// Low-level KRM-validity/split/decide tests live in pkg/lib/krmvalidate. diff --git a/pkg/lib/kptops/render_executor.go b/pkg/lib/kptops/render_executor.go index 50962bd4f3..aa2c038f2c 100644 --- a/pkg/lib/kptops/render_executor.go +++ b/pkg/lib/kptops/render_executor.go @@ -30,6 +30,7 @@ import ( fnruntime "github.com/kptdev/kpt/pkg/fn/runtime" "github.com/kptdev/kpt/pkg/kptfile/kptfileutil" "github.com/kptdev/kpt/pkg/lib/errors" + "github.com/kptdev/kpt/pkg/lib/krmvalidate" "github.com/kptdev/kpt/pkg/lib/pkg" "github.com/kptdev/kpt/pkg/lib/util/attribution" "github.com/kptdev/kpt/pkg/printer" @@ -94,6 +95,17 @@ func (e *Renderer) Execute(ctx context.Context) (*fnresultv1.ResultList, error) return nil, fmt.Errorf("failed to read Kptfile: %w", err) } + // Non-KRM pre-scan, before any pipeline starts. See krmvalidate.Scan. + preScanResult, preScanErr := krmvalidate.Scan(e.FileSystem, e.PkgPath, pr.Printf) + if preScanErr != nil { + if e.Output == nil { + updateRenderStatus(hctx, preScanErr) + } + return hctx.fnResults, errors.E(op, root.pkg.UniquePath, preScanErr) + } + hctx.preScannedResources = preScanResult.AllResources + hctx.skippedNonKRMFiles = preScanResult.SkippedFiles() + // Read save-on-render-failure behavior from Kptfile annotation if value, exists := kptfile.Annotations[kptfilev1.SaveOnRenderFailureAnnotation]; exists && kptfilev1.ToCondition(value) == kptfilev1.ConditionTrue { hctx.saveOnRenderFailure = true @@ -213,7 +225,8 @@ func (e *Renderer) printPipelineExecutionSummary(pr printer.Printer, hctx hydrat } // updateRenderStatus writes a Rendered status condition and RenderStatus to the root Kptfile. -// On success, the root package gets a True condition. +// On success, the root package gets a True condition, naming any files +// skipped via AutoIgnoreNonKRMAnnotation. // On failure, the root package gets a False condition with the error message. func updateRenderStatus(hctx *hydrationContext, hydErr error) { if hctx.fileSystem == nil { @@ -228,11 +241,29 @@ func updateRenderStatus(hctx *hydrationContext, hydErr error) { conditionStatus = kptfilev1.ConditionFalse reason = kptfilev1.ReasonRenderFailed message = strings.ReplaceAll(hydErr.Error(), rootPath, ".") + } else if skippedMsg := buildSkippedNonKRMMessage(hctx.skippedNonKRMFiles); skippedMsg != "" { + message = skippedMsg } renderStatus := buildRenderStatus(hctx, hydErr) setRenderStatus(hctx.fileSystem, rootPath, kptfilev1.NewRenderedCondition(conditionStatus, reason, message), renderStatus) } +// buildSkippedNonKRMMessage names every file skipped via the +// annotation, or "" if none were. +func buildSkippedNonKRMMessage(skipped []krmvalidate.NonKRMFile) string { + if len(skipped) == 0 { + return "" + } + names := make([]string, 0, len(skipped)) + for _, f := range skipped { + names = append(names, f.RelPath) + } + slices.Sort(names) + return fmt.Sprintf( + "These non-KRM resources were ignored during render due to the %s annotation being present on this package's Kptfile: %s", + krmvalidate.AutoIgnoreNonKRMAnnotation, strings.Join(names, ", ")) +} + // buildRenderStatus constructs a RenderStatus from the tracked pipeline step results. func buildRenderStatus(hctx *hydrationContext, hydErr error) *kptfilev1.RenderStatus { if len(hctx.mutationSteps) == 0 && len(hctx.validationSteps) == 0 { @@ -337,6 +368,14 @@ type hydrationContext struct { // should be saved when rendering fails. Read from the root Kptfile annotation. saveOnRenderFailure bool + // preScannedResources caches each package's resource list from the + // pre-scan (see krmvalidate.Scan), avoiding a second disk read. + preScannedResources map[string][]*yaml.RNode + + // skippedNonKRMFiles is every non-KRM file excused via + // AutoIgnoreNonKRMAnnotation, surfaced in updateRenderStatus. + skippedNonKRMFiles []krmvalidate.NonKRMFile + // mutationSteps and validationSteps track per-function results for RenderStatus. mutationSteps []kptfilev1.PipelineStepResult validationSteps []kptfilev1.PipelineStepResult @@ -440,7 +479,15 @@ func hydrate(ctx context.Context, pn *pkgNode, hctx *hydrationContext) (output [ var input []*yaml.RNode // gather resources present at the current package - currPkgResources, err := curr.pkg.LocalResources() + currPkgResources, err := krmvalidate.CachedLocalResources(curr.pkg, hctx.preScannedResources) + if err != nil { + return output, errors.E(op, curr.pkg.UniquePath, err) + } + + // No-op unless this package opted in via the annotation; the + // pre-scan in Execute() already decided whether it's safe to be + // here at all. + currPkgResources, _, err = krmvalidate.Split(curr.pkg.UniquePath.String(), currPkgResources) if err != nil { return output, errors.E(op, curr.pkg.UniquePath, err) } @@ -572,10 +619,16 @@ func validatePackageState(current *pkgNode, hctx *hydrationContext) error { // loadPackageResources loads local resources for the package and tracks input files func loadPackageResources(current *pkgNode, hctx *hydrationContext) error { - localResources, err := current.pkg.LocalResources() + localResources, err := krmvalidate.CachedLocalResources(current.pkg, hctx.preScannedResources) if err != nil { return err } + + localResources, _, err = krmvalidate.Split(current.pkg.UniquePath.String(), localResources) + if err != nil { + return err + } + current.resources = localResources relPath, err := current.pkg.RelativePathTo(hctx.root.pkg) @@ -714,7 +767,7 @@ func (pn *pkgNode) runPipeline(ctx context.Context, hctx *hydrationContext, inpu } if pl.IsEmpty() { - if err := kptfilev1.AreKRM(input); err != nil { + if err := krmvalidate.AreKRM(input); err != nil { return nil, fmt.Errorf("input resource list must contain only KRM resources: %w", err) } return input, nil diff --git a/pkg/lib/kptops/render_executor_test.go b/pkg/lib/kptops/render_executor_test.go index 24cf1baa33..61c58141ad 100644 --- a/pkg/lib/kptops/render_executor_test.go +++ b/pkg/lib/kptops/render_executor_test.go @@ -27,6 +27,7 @@ import ( "github.com/kptdev/kpt/api/runneroptions" fnruntime "github.com/kptdev/kpt/pkg/fn/runtime" "github.com/kptdev/kpt/pkg/kptfile/kptfileutil" + "github.com/kptdev/kpt/pkg/lib/krmvalidate" "github.com/kptdev/kpt/pkg/lib/pkg" "github.com/kptdev/kpt/pkg/printer" "github.com/kptdev/kpt/pkg/printer/fake" @@ -767,6 +768,51 @@ metadata: assert.Equal(t, kptfilev1.ReasonRenderSuccess, rootKf.Status.Conditions[0].Reason) } +// TestUpdateRenderStatus_SkippedNonKRMFiles verifies a successful +// render names its annotation-skipped files in the condition message. +func TestUpdateRenderStatus_SkippedNonKRMFiles(t *testing.T) { + mockFS := filesys.MakeFsInMemory() + rootPath := rootString + assert.NoError(t, mockFS.Mkdir(rootPath)) + + assert.NoError(t, mockFS.WriteFile(filepath.Join(rootPath, "Kptfile"), []byte(` +apiVersion: kpt.dev/v1 +kind: Kptfile +metadata: + name: root-package + annotations: + kpt.dev/auto-ignore-non-krm: "true" +`))) + + rootPkg, err := pkg.New(mockFS, rootPath) + assert.NoError(t, err) + + hctx := &hydrationContext{ + root: &pkgNode{pkg: rootPkg}, + pkgs: map[kptfilev1.UniquePath]*pkgNode{}, + fileSystem: mockFS, + skippedNonKRMFiles: []krmvalidate.NonKRMFile{ + {PkgPath: rootPath, RelPath: "values.yaml", Reason: "resource must have `apiVersion`"}, + {PkgPath: rootPath, RelPath: "defaults.yaml", Reason: "resource must have `apiVersion`"}, + }, + } + hctx.pkgs[rootPkg.UniquePath] = &pkgNode{pkg: rootPkg} + + updateRenderStatus(hctx, nil) + + rootKf, err := kptfileutil.ReadKptfile(mockFS, rootPath) + assert.NoError(t, err) + assert.NotNil(t, rootKf.Status) + assert.Len(t, rootKf.Status.Conditions, 1) + cond := rootKf.Status.Conditions[0] + assert.Equal(t, kptfilev1.ConditionTrue, cond.Status) + assert.Equal(t, kptfilev1.ReasonRenderSuccess, cond.Reason) + assert.Contains(t, cond.Message, "defaults.yaml") + assert.Contains(t, cond.Message, "values.yaml") + assert.Contains(t, cond.Message, krmvalidate.AutoIgnoreNonKRMAnnotation) + assert.Less(t, strings.Index(cond.Message, "defaults.yaml"), strings.Index(cond.Message, "values.yaml")) +} + func TestUpdateRenderStatus_Failure(t *testing.T) { mockFS := filesys.MakeFsInMemory() rootPath := rootString diff --git a/pkg/lib/krmvalidate/iskrm.go b/pkg/lib/krmvalidate/iskrm.go new file mode 100644 index 0000000000..af48edb756 --- /dev/null +++ b/pkg/lib/krmvalidate/iskrm.go @@ -0,0 +1,88 @@ +// Copyright 2026 The kpt Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package krmvalidate is the shared implementation of "is this a valid +// KRM resource" and the tree-wide non-KRM pre-scan built on it, used by +// every kpt command that needs to detect non-KRM files consistently. +package krmvalidate + +import ( + "fmt" + "strings" + + kptfilev1 "github.com/kptdev/kpt/api/kptfile/v1" + "sigs.k8s.io/kustomize/kyaml/kio/filters" + "sigs.k8s.io/kustomize/kyaml/kio/kioutil" + "sigs.k8s.io/kustomize/kyaml/yaml" +) + +// IsKRM reports whether n is a valid KRM resource for classifying +// pipeline input: apiVersion and kind are always required. metadata.name +// is required too, except for kustomization files (any shape), "*List" +// kinds, resources using metadata.generateName instead, and resources +// annotated local-config. +// +// This deliberately differs from kptfilev1.IsKRM, which validates a +// function config and always requires a name (a function config needs a +// stable name to be addressed by). Don't merge them: that would make +// `fn eval --fn-config` accept generateName-only or nameless configs. +func IsKRM(n *yaml.RNode) error { + if kptfilev1.IsKustomization(n) { + return nil + } + meta, err := n.GetMeta() + if err != nil { + return fmt.Errorf("resource must have `apiVersion`, `kind`, and `name`") + } + if meta.APIVersion == "" { + return fmt.Errorf("resource must have `apiVersion`") + } + if meta.Kind == "" { + return fmt.Errorf("resource must have `kind`") + } + if meta.Name == "" && !strings.HasSuffix(meta.Kind, "List") && + !hasGenerateName(n) && !isLocalConfig(meta) { + return fmt.Errorf("resource must have `metadata.name`") + } + return nil +} + +// hasGenerateName reports whether n sets metadata.generateName, a +// legitimate alternative to metadata.name that kyaml's typed +// ResourceMeta doesn't expose, so it needs raw node access. +func hasGenerateName(n *yaml.RNode) bool { + v, _ := n.Pipe(yaml.Lookup("metadata", "generateName")) + return v != nil && v.YNode().Value != "" +} + +// isLocalConfig reports whether meta carries the local-config +// annotation with a non-"false" value, mirroring pkg/live's own +// filterLocalConfig so both agree on what's exempt from needing a name. +func isLocalConfig(meta yaml.ResourceMeta) bool { + v, ok := meta.Annotations[filters.LocalConfigAnnotation] + return ok && v != "false" +} + +// AreKRM checks every node in nodes via IsKRM, stopping at the first +// failure. Prefer ScanTree for anything with tree-wide context; use +// AreKRM only where there's no tree to batch across. +func AreKRM(nodes []*yaml.RNode) error { + for i := range nodes { + if err := IsKRM(nodes[i]); err != nil { + path, _, _ := kioutil.GetFileAnnotations(nodes[i]) + return fmt.Errorf("%s: %w", path, err) + } + } + return nil +} diff --git a/pkg/lib/krmvalidate/iskrm_test.go b/pkg/lib/krmvalidate/iskrm_test.go new file mode 100644 index 0000000000..6a19afcf22 --- /dev/null +++ b/pkg/lib/krmvalidate/iskrm_test.go @@ -0,0 +1,238 @@ +// Copyright 2026 The kpt Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package krmvalidate + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "sigs.k8s.io/kustomize/kyaml/yaml" +) + +func mustParse(t *testing.T, s string) *yaml.RNode { + t.Helper() + n, err := yaml.Parse(s) + assert.NoError(t, err) + return n +} + +func TestIsKRM_ValidResource(t *testing.T) { + n := mustParse(t, ` +apiVersion: v1 +kind: ConfigMap +metadata: + name: my-cm +`) + assert.NoError(t, IsKRM(n)) +} + +func TestIsKRM_MissingAPIVersion(t *testing.T) { + n := mustParse(t, ` +kind: ConfigMap +metadata: + name: my-cm +`) + err := IsKRM(n) + assert.Error(t, err) + assert.Contains(t, err.Error(), "apiVersion") +} + +func TestIsKRM_MissingKind(t *testing.T) { + n := mustParse(t, ` +apiVersion: v1 +metadata: + name: my-cm +`) + err := IsKRM(n) + assert.Error(t, err) + assert.Contains(t, err.Error(), "kind") +} + +func TestIsKRM_MissingName(t *testing.T) { + n := mustParse(t, ` +apiVersion: v1 +kind: ConfigMap +metadata: {} +`) + err := IsKRM(n) + assert.Error(t, err) + assert.Contains(t, err.Error(), "metadata.name") +} + +func TestIsKRM_NoMetadataAtAll(t *testing.T) { + n := mustParse(t, ` +replicaCount: 3 +image: + repository: nginx +`) + err := IsKRM(n) + assert.Error(t, err) +} + +// TestIsKRM_ListKindDoesNotRequireName verifies a nameless List-kind +// aggregate (e.g. ConfigMapList) is valid KRM. +func TestIsKRM_ListKindDoesNotRequireName(t *testing.T) { + n := mustParse(t, ` +apiVersion: v1 +kind: ConfigMapList +items: +- apiVersion: v1 + kind: ConfigMap + metadata: + name: item-one +`) + assert.NoError(t, IsKRM(n)) +} + +// TestIsKRM_ListKindStillRequiresAPIVersionAndKind verifies the List +// carve-out only waives the name requirement. +func TestIsKRM_ListKindStillRequiresAPIVersionAndKind(t *testing.T) { + t.Run("missing apiVersion", func(t *testing.T) { + n := mustParse(t, ` +kind: ConfigMapList +items: [] +`) + err := IsKRM(n) + assert.Error(t, err) + assert.Contains(t, err.Error(), "apiVersion") + }) + t.Run("missing kind entirely", func(t *testing.T) { + n := mustParse(t, ` +apiVersion: v1 +items: [] +`) + err := IsKRM(n) + assert.Error(t, err) + assert.Contains(t, err.Error(), "kind") + }) +} + +// TestIsKRM_GenerateNameExemptsFromName verifies a resource using +// metadata.generateName instead of metadata.name is valid KRM. +func TestIsKRM_GenerateNameExemptsFromName(t *testing.T) { + n := mustParse(t, ` +apiVersion: batch/v1 +kind: Job +metadata: + generateName: my-job- +`) + assert.NoError(t, IsKRM(n)) +} + +// TestIsKRM_EmptyGenerateNameStillRequiresName verifies an explicitly +// empty generateName doesn't count as having one. +func TestIsKRM_EmptyGenerateNameStillRequiresName(t *testing.T) { + n := mustParse(t, ` +apiVersion: batch/v1 +kind: Job +metadata: + generateName: "" +`) + err := IsKRM(n) + assert.Error(t, err) + assert.Contains(t, err.Error(), "metadata.name") +} + +// TestIsKRM_LocalConfigExemptsFromName verifies a nameless resource +// annotated as local-config (kpt's "config.kubernetes.io/local-config" +// convention) is valid KRM, matching pkg/live's own filterLocalConfig. +func TestIsKRM_LocalConfigExemptsFromName(t *testing.T) { + n := mustParse(t, ` +apiVersion: v1 +kind: ConfigMap +metadata: + annotations: + config.kubernetes.io/local-config: "true" +`) + assert.NoError(t, IsKRM(n)) +} + +// TestIsKRM_LocalConfigFalseStillRequiresName verifies an explicit +// "false" value (meaning "not local-config", per filterLocalConfig) +// does not exempt the resource from needing a name. +func TestIsKRM_LocalConfigFalseStillRequiresName(t *testing.T) { + n := mustParse(t, ` +apiVersion: v1 +kind: ConfigMap +metadata: + annotations: + config.kubernetes.io/local-config: "false" +`) + err := IsKRM(n) + assert.Error(t, err) + assert.Contains(t, err.Error(), "metadata.name") +} + +func TestIsKRM_KustomizationFileNameCarveOut(t *testing.T) { + n := mustParse(t, ` +resources: +- deployment.yaml +`) + assert.NoError(t, n.PipeE(yaml.SetAnnotation("config.kubernetes.io/path", "overlays/dev/kustomization.yaml"))) + assert.NoError(t, IsKRM(n), "a file named kustomization.yaml must be exempt from KRM validation regardless of its shape") +} + +func TestIsKRM_KustomizationAPIGroupCarveOut(t *testing.T) { + n := mustParse(t, ` +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: +- deployment.yaml +`) + assert.NoError(t, IsKRM(n)) +} + +func TestIsKRM_BareKustomizationKindNoAPIVersion(t *testing.T) { + n := mustParse(t, ` +kind: Kustomization +resources: +- deployment.yaml +`) + assert.NoError(t, IsKRM(n)) +} + +func TestAreKRM_ReportsFirstFailureWithPath(t *testing.T) { + valid := mustParse(t, ` +apiVersion: v1 +kind: ConfigMap +metadata: + name: good +`) + assert.NoError(t, valid.PipeE(yaml.SetAnnotation("config.kubernetes.io/path", "good.yaml"))) + + invalid := mustParse(t, ` +replicaCount: 3 +`) + assert.NoError(t, invalid.PipeE(yaml.SetAnnotation("config.kubernetes.io/path", "bad.yaml"))) + + err := AreKRM([]*yaml.RNode{valid, invalid}) + assert.Error(t, err) + assert.Contains(t, err.Error(), "bad.yaml") +} + +func TestAreKRM_AllValidSucceeds(t *testing.T) { + a := mustParse(t, ` +apiVersion: v1 +kind: ConfigMap +metadata: + name: a +`) + b := mustParse(t, ` +apiVersion: v1 +kind: ConfigMapList +items: [] +`) + assert.NoError(t, AreKRM([]*yaml.RNode{a, b})) +} diff --git a/pkg/lib/krmvalidate/prescan.go b/pkg/lib/krmvalidate/prescan.go new file mode 100644 index 0000000000..66568bc866 --- /dev/null +++ b/pkg/lib/krmvalidate/prescan.go @@ -0,0 +1,597 @@ +// Copyright 2026 The kpt Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The non-KRM pre-scan: walk a package tree read-only, decide whether +// any non-KRM file found is fatal or excused by annotation, and report +// everything found in one pass, before any KRM-assuming command starts +// doing real work. +package krmvalidate + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + kptfilev1 "github.com/kptdev/kpt/api/kptfile/v1" + "github.com/kptdev/kpt/pkg/lib/pkg" + "sigs.k8s.io/kustomize/kyaml/filesys" + "sigs.k8s.io/kustomize/kyaml/kio" + "sigs.k8s.io/kustomize/kyaml/kio/kioutil" + "sigs.k8s.io/kustomize/kyaml/yaml" +) + +// AutoIgnoreNonKRMAnnotation is the per-package Kptfile annotation +// that excuses non-KRM files instead of failing on them. +const AutoIgnoreNonKRMAnnotation = "kpt.dev/auto-ignore-non-krm" + +// NonKRMFile records one non-KRM file found during a scan. +type NonKRMFile struct { + PkgPath string // absolute path of the owning package + RelPath string // path relative to PkgPath + Reason string // the IsKRM() validation error +} + +// DisplayPath returns f's path relative to rootPath, for display. +// f.RelPath alone is relative to f's own (possibly sub) package, so +// same-named files in different subpackages would look identical. +func (f NonKRMFile) DisplayPath(rootPath string) string { + abs := filepath.Join(f.PkgPath, f.RelPath) + rel, err := filepath.Rel(rootPath, abs) + if err != nil { + return f.RelPath + } + return filepath.ToSlash(rel) +} + +// Split partitions files into valid KRM resources and non-KRM files. +// A file containing any non-KRM document is excluded in its entirety. +func Split(pkgDir string, nodes []*yaml.RNode) (valid []*yaml.RNode, nonKRM []NonKRMFile, err error) { + nonKRMPaths := make(map[string]bool) + for _, n := range nodes { + if krmErr := IsKRM(n); krmErr != nil { + relPath, _, pathErr := kioutil.GetFileAnnotations(n) + if pathErr != nil { + return nil, nil, fmt.Errorf("non-krm-detection: resource in %s missing path annotation: %w", pkgDir, pathErr) + } + if !nonKRMPaths[relPath] { + nonKRM = append(nonKRM, NonKRMFile{PkgPath: pkgDir, RelPath: relPath, Reason: krmErr.Error()}) + nonKRMPaths[relPath] = true + } + } + } + for _, n := range nodes { + relPath, _, _ := kioutil.GetFileAnnotations(n) + if !nonKRMPaths[relPath] { + valid = append(valid, n) + } + } + return valid, nonKRM, nil +} + +// PackageOptedIn reports whether kf's package carries +// AutoIgnoreNonKRMAnnotation="true". False if kf is nil. +func PackageOptedIn(kf *kptfilev1.KptFile) bool { + if kf == nil { + return false + } + return kptfilev1.ToCondition(kf.Annotations[AutoIgnoreNonKRMAnnotation]) == kptfilev1.ConditionTrue +} + +// ScanResult is the outcome of a successful Scan: the caller is clear +// to proceed. +type ScanResult struct { + // AllResources is every (sub)package's resource list, keyed by + // absolute package path, reusable via CachedLocalResources to + // avoid a second disk read. + AllResources map[string][]*yaml.RNode + + skipped []NonKRMFile +} + +// Scan runs ScanTree, DecideOutcome, and PrintSkippedSummary for +// pkgPath: the single entry point most callers should use before +// starting any KRM-assuming work. +func Scan(fsys filesys.FileSystem, pkgPath string, printf func(format string, args ...any)) (*ScanResult, error) { + return scan(fsys, pkgPath, printf, false) +} + +// ScanHonoringKrmignore is Scan for callers whose real reader skips +// subpackages a parent .krmignore excludes (fn eval, fn source). fn +// render must keep using Scan: its executor still hydrates those +// subpackages, so it needs them validated up front. +func ScanHonoringKrmignore(fsys filesys.FileSystem, pkgPath string, printf func(format string, args ...any)) (*ScanResult, error) { + return scan(fsys, pkgPath, printf, true) +} + +func scan(fsys filesys.FileSystem, pkgPath string, printf func(format string, args ...any), honorKrmignore bool) (*ScanResult, error) { + found, optedIn, allResources, err := scanTree(fsys, pkgPath, pkg.MatchAllKRM, honorKrmignore) + if err != nil { + return nil, fmt.Errorf("non-krm pre-scan: %w", err) + } + skipped, err := decideAndCollectSkipped(found, optedIn, pkgPath, printf) + if err != nil { + return nil, err + } + return &ScanResult{AllResources: allResources, skipped: skipped}, nil +} + +// decideAndCollectSkipped is the step Scan and ScanFlatAndDecide share +// after walking: fail on any non-KRM file whose package didn't opt in, +// report the ones that did, and return those excused files. +func decideAndCollectSkipped(found []NonKRMFile, optedIn map[string]bool, rootPath string, + printf func(format string, args ...any)) ([]NonKRMFile, error) { + if err := DecideOutcome(found, optedIn, rootPath); err != nil { + return nil, err + } + PrintSkippedSummary(printf, found, optedIn) + + var skipped []NonKRMFile + for _, f := range found { + if optedIn[f.PkgPath] { + skipped = append(skipped, f) + } + } + return skipped, nil +} + +// SkippedFiles returns every non-KRM file Scan excused via the +// annotation. Safe to call on a nil receiver. +func (res *ScanResult) SkippedFiles() []NonKRMFile { + if res == nil { + return nil + } + return res.skipped +} + +// SkipFunc returns a FileSkipFunc for the files Scan excused, for +// callers wiring a reader's own skip hook (so those files are never +// read at all, rather than filtered out after the fact). +func (res *ScanResult) SkipFunc(pkgRootPath string) func(relPath string) bool { + if res == nil { + return nil + } + return skipFuncFor(res.skipped, pkgRootPath) +} + +// ScanTree walks pkgPath and its subpackages read-only, returning +// every non-KRM file found, opted-in packages, and each package's +// resources. matchGlob should match the caller's real reader. +func ScanTree(fsys filesys.FileSystem, pkgPath string, matchGlob []string) ( + found []NonKRMFile, optedIn map[string]bool, allResources map[string][]*yaml.RNode, err error, +) { + return scanTree(fsys, pkgPath, matchGlob, false) +} + +// scanTree is ScanTree; honorKrmignore additionally prunes subpackages +// the real fn reader can't reach because a parent .krmignore excludes them. +func scanTree(fsys filesys.FileSystem, pkgPath string, matchGlob []string, honorKrmignore bool) ( + found []NonKRMFile, optedIn map[string]bool, allResources map[string][]*yaml.RNode, err error, +) { + optedIn = map[string]bool{} + allResources = map[string][]*yaml.RNode{} + + root, err := pkg.New(fsys, pkgPath) + if err != nil { + return nil, nil, nil, err + } + + // Loaded once upfront when honorKrmignore is true, before any + // DirectSubpackages() call can read Kptfiles inside ignored dirs. + var reachable map[string]bool + if honorKrmignore { + reachable = reachableKptfileDirs(fsys, pkgPath, kptfilev1.KptFileName) + } + + var walk func(p *pkg.Pkg, isRoot bool) error + walk = func(p *pkg.Pkg, isRoot bool) error { + dir := p.UniquePath.String() + + kf, kfErr := p.Kptfile() + if kfErr != nil && !errors.Is(kfErr, os.ErrNotExist) { + return kfErr + } + // kf is nil here either because Kptfile() returned + // os.ErrNotExist (tolerated, see doc comment) or because kf + // itself is a genuinely nil *KptFile; PackageOptedIn handles + // nil already. + optedIn[dir] = PackageOptedIn(kf) + + var resources []*yaml.RNode + var resErr error + if isRoot && kf == nil { + // LocalResources() returns empty for a Kptfile-less dir, + // but a bare-directory target still has its real reader + // read every file regardless; read the same way here. + resources, resErr = (&kio.LocalPackageReader{ + PackagePath: dir, + PackageFileName: kptfilev1.KptFileName, + IncludeSubpackages: false, + MatchFilesGlob: matchGlob, + WrapBareSeqNode: true, + FileSystem: filesys.FileSystemOrOnDisk{FileSystem: fsys}, + }).Read() + } else { + // LocalResources() always reads with pkg.MatchAllKRM; + // narrow to matchGlob here for callers whose real + // reader is stricter (e.g. live's yaml/yml-only). + resources, resErr = p.LocalResources() + resources = filterByMatchGlob(resources, matchGlob) + } + if resErr != nil { + return resErr + } + allResources[dir] = resources + + _, nonKRM, splitErr := Split(dir, resources) + if splitErr != nil { + return splitErr + } + found = append(found, nonKRM...) + + if reachable != nil { + return nil + } + + subpkgs, subErr := p.DirectSubpackages() + if subErr != nil { + return subErr + } + for _, sub := range subpkgs { + if err := walk(sub, false); err != nil { + return err + } + } + return nil + } + + if err := walk(root, true); err != nil { + return nil, nil, nil, err + } + if reachable != nil { + for _, dir := range sortedSubpackageDirs(reachable, root.UniquePath.String()) { + sub, subErr := pkg.New(fsys, dir) + if subErr != nil { + return nil, nil, nil, subErr + } + if err := walk(sub, false); err != nil { + return nil, nil, nil, err + } + } + } + return found, optedIn, allResources, nil +} + +// sortedSubpackageDirs returns every directory in reachable other than +// rootDir, sorted for deterministic traversal. +func sortedSubpackageDirs(reachable map[string]bool, rootDir string) []string { + dirs := make([]string, 0, len(reachable)) + for dir := range reachable { + if dir != rootDir { + dirs = append(dirs, dir) + } + } + sort.Strings(dirs) + return dirs +} + +// reachableKptfileDirs returns the directory of every Kptfile a +// LocalPackageReader can reach from rootPath, honoring .krmignore with +// kyaml's exact semantics. Pass packageFileName=kptfilev1.KptFileName +// for package-scoped .krmignore (fn eval/source), or "" for flat +// root-only .krmignore (live). +// Returns nil on any read error, meaning "don't prune": the per-package +// reads that follow surface the real error. +func reachableKptfileDirs(fsys filesys.FileSystem, rootPath, packageFileName string) map[string]bool { + nodes, err := (&kio.LocalPackageReader{ + PackagePath: rootPath, + PackageFileName: packageFileName, + IncludeSubpackages: packageFileName != "", + MatchFilesGlob: []string{kptfilev1.KptFileName}, + FileSystem: filesys.FileSystemOrOnDisk{FileSystem: fsys}, + }).Read() + if err != nil { + return nil + } + dirs := make(map[string]bool, len(nodes)) + for _, n := range nodes { + relPath, _, pathErr := kioutil.GetFileAnnotations(n) + if pathErr != nil { + return nil + } + dirs[filepath.Join(rootPath, filepath.Dir(relPath))] = true + } + return dirs +} + +// liveReaderMatchGlob mirrors pkg/live's real reader (defaults to +// kio.DefaultMatch: *.yaml/*.yml only). Used so a file the live +// reader never touches, e.g. a stray package.json, is never flagged. +var liveReaderMatchGlob = kio.DefaultMatch + +// filterByMatchGlob drops nodes whose source file doesn't match +// matchGlob, narrowing an already-MatchAllKRM-globbed read without a +// second disk read. +func filterByMatchGlob(nodes []*yaml.RNode, matchGlob []string) []*yaml.RNode { + var out []*yaml.RNode + for _, n := range nodes { + relPath, _, err := kioutil.GetFileAnnotations(n) + if err != nil { + continue + } + base := filepath.Base(relPath) + for _, g := range matchGlob { + if match, _ := filepath.Match(g, base); match { + out = append(out, n) + break + } + } + } + return out +} + +// collectOptedIn walks pkgPath and every subpackage read-only, +// Kptfile only (no resource content), mirroring ScanTree's tree +// structure. Used by scanFlatTree to attribute found files to their +// owning package for opt-in purposes, without a second content read. +func collectOptedIn(fsys filesys.FileSystem, pkgPath string) (optedIn map[string]bool, pkgPaths []string, err error) { + optedIn = map[string]bool{} + + root, err := pkg.New(fsys, pkgPath) + if err != nil { + return nil, nil, err + } + + // Honor root .krmignore upfront (flat, matching scanFlatTree's real + // reader) before DirectSubpackages() can read Kptfiles inside + // ignored directories. + reachable := reachableKptfileDirs(fsys, pkgPath, "") + + var walk func(p *pkg.Pkg) error + walk = func(p *pkg.Pkg) error { + dir := p.UniquePath.String() + kf, kfErr := p.Kptfile() + if kfErr != nil && !errors.Is(kfErr, os.ErrNotExist) { + return kfErr + } + optedIn[dir] = PackageOptedIn(kf) + pkgPaths = append(pkgPaths, dir) + + if reachable != nil { + return nil + } + + subpkgs, subErr := p.DirectSubpackages() + if subErr != nil { + return subErr + } + for _, sub := range subpkgs { + if err := walk(sub); err != nil { + return err + } + } + return nil + } + if err := walk(root); err != nil { + return nil, nil, err + } + if reachable != nil { + for _, dir := range sortedSubpackageDirs(reachable, root.UniquePath.String()) { + sub, subErr := pkg.New(fsys, dir) + if subErr != nil { + return nil, nil, subErr + } + if err := walk(sub); err != nil { + return nil, nil, err + } + } + } + return optedIn, pkgPaths, nil +} + +// reattributeToOwningPackage re-roots f, found via one flat read from +// pkgPath, to whichever known package path most tightly contains it, +// so per-package opt-in lookups work without a second content read. +func reattributeToOwningPackage(f NonKRMFile, pkgPath string, pkgPaths []string) NonKRMFile { + abs := filepath.Join(pkgPath, f.RelPath) + owner := pkgPath + for _, p := range pkgPaths { + rel, err := filepath.Rel(p, abs) + if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + continue + } + if len(p) > len(owner) { + owner = p + } + } + rel, err := filepath.Rel(owner, abs) + if err != nil { + return f + } + return NonKRMFile{PkgPath: owner, RelPath: rel, Reason: f.Reason} +} + +// scanFlatTree is ScanTree's counterpart for flat-reader callers (the +// `live` family): one read matching the real reader's own setup (no +// subpackage-boundary concept, so .krmignore cascades exactly like it +// does for the real reader), plus a separate Kptfile-only walk to +// attribute each found file to its owning package for opt-in. +func scanFlatTree(fsys filesys.FileSystem, pkgPath string) (found []NonKRMFile, optedIn map[string]bool, resources []*yaml.RNode, err error) { + optedIn, pkgPaths, err := collectOptedIn(fsys, pkgPath) + if err != nil { + return nil, nil, nil, err + } + + resources, err = (&kio.LocalPackageReader{ + PackagePath: pkgPath, + MatchFilesGlob: liveReaderMatchGlob, + WrapBareSeqNode: true, + FileSystem: filesys.FileSystemOrOnDisk{FileSystem: fsys}, + }).Read() + if err != nil { + return nil, nil, nil, err + } + + _, nonKRM, splitErr := Split(pkgPath, resources) + if splitErr != nil { + return nil, nil, nil, splitErr + } + for i, f := range nonKRM { + nonKRM[i] = reattributeToOwningPackage(f, pkgPath, pkgPaths) + } + return nonKRM, optedIn, resources, nil +} + +// ScanFlat wraps scanFlatTree for callers whose real reader has no +// subpackage concept (e.g. the `live` family's flat reader). +func ScanFlat(fsys filesys.FileSystem, pkgPath string) (found []NonKRMFile, optedIn bool, allResources []*yaml.RNode, err error) { + treeFound, treeOptedIn, resources, scanErr := scanFlatTree(fsys, pkgPath) + if scanErr != nil { + return nil, false, nil, scanErr + } + + // optedIn is pkgPath's own root only; migrate (the one caller + // that still reads it) ignores it anyway. + optedIn = treeOptedIn[pkgPath] + found = treeFound + allResources = resources + return found, optedIn, allResources, nil +} + +// FlatScanResult mirrors ScanResult for ScanFlat's flat-reader model. +type FlatScanResult struct { + AllResources []*yaml.RNode + + skipped []NonKRMFile +} + +// ScanFlatAndDecide is Scan for flat-reader callers: same per-package +// opt-in via scanFlatTree, flattened into FlatScanResult's shape. +func ScanFlatAndDecide(fsys filesys.FileSystem, pkgPath string, printf func(format string, args ...any)) (*FlatScanResult, error) { + found, optedIn, resources, err := scanFlatTree(fsys, pkgPath) + if err != nil { + return nil, fmt.Errorf("non-krm pre-scan: %w", err) + } + skipped, err := decideAndCollectSkipped(found, optedIn, pkgPath, printf) + if err != nil { + return nil, err + } + return &FlatScanResult{AllResources: resources, skipped: skipped}, nil +} + +// SkippedFiles mirrors ScanResult.SkippedFiles for FlatScanResult. +func (res *FlatScanResult) SkippedFiles() []NonKRMFile { + if res == nil { + return nil + } + return res.skipped +} + +// SkipFunc mirrors ScanResult.SkipFunc, re-rooting each file's path +// (relative to its own, possibly-sub, package) against pkgRootPath. +func (res *FlatScanResult) SkipFunc(pkgRootPath string) func(relPath string) bool { + if res == nil { + return nil + } + return skipFuncFor(res.skipped, pkgRootPath) +} + +// skipFuncFor builds a FileSkipFunc from a list of excused files, +// re-rooting each one's path against pkgRootPath. Shared by +// ScanResult.SkipFunc and FlatScanResult.SkipFunc. +func skipFuncFor(skipped []NonKRMFile, pkgRootPath string) func(relPath string) bool { + if len(skipped) == 0 { + return nil + } + skip := make(map[string]bool, len(skipped)) + for _, f := range skipped { + abs := filepath.Join(f.PkgPath, f.RelPath) + rel, err := filepath.Rel(pkgRootPath, abs) + if err != nil { + continue + } + skip[filepath.ToSlash(rel)] = true + } + return func(relPath string) bool { + return skip[filepath.ToSlash(relPath)] + } +} + +// CachedLocalResources returns p's resources from cache if present, +// falling back to a real LocalResources() read otherwise. +func CachedLocalResources(p *pkg.Pkg, cache map[string][]*yaml.RNode) ([]*yaml.RNode, error) { + if cache != nil { + if resources, ok := cache[p.UniquePath.String()]; ok { + return resources, nil + } + } + return p.LocalResources() +} + +// DecideOutcome returns an error listing every blocking file found, +// or nil if all were excused. rootPath displays each file's path +// unambiguously (see NonKRMFile.DisplayPath). +func DecideOutcome(found []NonKRMFile, optedIn map[string]bool, rootPath string) error { + var blocking []NonKRMFile + for _, f := range found { + if !optedIn[f.PkgPath] { + blocking = append(blocking, f) + } + } + if len(blocking) == 0 { + return nil + } + + sort.Slice(blocking, func(i, j int) bool { + return blocking[i].DisplayPath(rootPath) < blocking[j].DisplayPath(rootPath) + }) + + var b strings.Builder + fmt.Fprintf(&b, "input resource list must contain only KRM resources (%d file(s)):", len(blocking)) + for _, f := range blocking { + fmt.Fprintf(&b, "\n %s: %s", f.DisplayPath(rootPath), f.Reason) + } + fmt.Fprintf(&b, "\nTo resolve: either exclude these files via .krmignore (manually, or with a pattern rule), "+ + "or add `%s: \"true\"` to this package's Kptfile to skip any non-KRM files automatically from this package.", AutoIgnoreNonKRMAnnotation) + return errors.New(b.String()) +} + +// PrintSkippedSummary prints which non-KRM files were excused per +// opted-in package. No-op if nothing was skipped. +func PrintSkippedSummary(printf func(format string, args ...any), found []NonKRMFile, optedIn map[string]bool) { + byPkg := map[string][]NonKRMFile{} + var pkgOrder []string + for _, f := range found { + if !optedIn[f.PkgPath] { + continue + } + if _, seen := byPkg[f.PkgPath]; !seen { + pkgOrder = append(pkgOrder, f.PkgPath) + } + byPkg[f.PkgPath] = append(byPkg[f.PkgPath], f) + } + sort.Strings(pkgOrder) + + for _, pkgPath := range pkgOrder { + files := byPkg[pkgPath] + names := make([]string, 0, len(files)) + for _, f := range files { + names = append(names, f.RelPath) + } + printf("[info] ignoring %d non-KRM file(s) in %q due to %s annotation: %s\n", + len(files), pkgPath, AutoIgnoreNonKRMAnnotation, strings.Join(names, ", ")) + } +} diff --git a/pkg/lib/krmvalidate/prescan_test.go b/pkg/lib/krmvalidate/prescan_test.go new file mode 100644 index 0000000000..a614e46544 --- /dev/null +++ b/pkg/lib/krmvalidate/prescan_test.go @@ -0,0 +1,768 @@ +// Copyright 2026 The kpt Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package krmvalidate + +import ( + "fmt" + "path/filepath" + "testing" + + "github.com/kptdev/kpt/pkg/lib/pkg" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "sigs.k8s.io/kustomize/kyaml/filesys" + "sigs.k8s.io/kustomize/kyaml/yaml" +) + +func writeKptfileForTest(t *testing.T, fsys filesys.FileSystem, dir, name string) { + t.Helper() + content := "apiVersion: kpt.dev/v1\nkind: Kptfile\nmetadata:\n name: " + name + "\n" + assert.NoError(t, fsys.WriteFile(filepath.Join(dir, "Kptfile"), []byte(content))) +} + +func writeOptedInKptfileForTest(t *testing.T, fsys filesys.FileSystem, dir, name string) { + t.Helper() + content := "apiVersion: kpt.dev/v1\nkind: Kptfile\nmetadata:\n name: " + name + + "\n annotations:\n " + AutoIgnoreNonKRMAnnotation + ": \"true\"\n" + assert.NoError(t, fsys.WriteFile(filepath.Join(dir, "Kptfile"), []byte(content))) +} + +// printfCollector returns a printf-shaped func that records every +// formatted line it receives, for asserting on Scan/PrintSkippedSummary output. +func printfCollector() (func(format string, args ...any), *[]string) { + var lines []string + return func(format string, args ...any) { + lines = append(lines, fmt.Sprintf(format, args...)) + }, &lines +} + +func writeValidKRMForTest(t *testing.T, fsys filesys.FileSystem, dir string) { + t.Helper() + content := "apiVersion: v1\nkind: ConfigMap\nmetadata:\n name: a\n" + assert.NoError(t, fsys.WriteFile(filepath.Join(dir, "a.yaml"), []byte(content))) +} + +func writeNonKRMForTest(t *testing.T, fsys filesys.FileSystem, dir, fileName string) { + t.Helper() + assert.NoError(t, fsys.WriteFile(filepath.Join(dir, fileName), []byte("not: krm\njust: [a, plain, map]\nwithout: apiVersion-or-kind\n"))) +} + +// TestSplit unit-tests the low-level partitioning function directly. +func TestSplit(t *testing.T) { + fsys := filesys.MakeFsInMemory() + dir := "/pkg" + assert.NoError(t, fsys.Mkdir(dir)) + writeKptfileForTest(t, fsys, dir, "p") + writeValidKRMForTest(t, fsys, dir) + writeNonKRMForTest(t, fsys, dir, "b.yaml") + + p, err := pkg.New(fsys, dir) + assert.NoError(t, err) + nodes, err := p.LocalResources() + assert.NoError(t, err) + + valid, nonKRM, err := Split(dir, nodes) + assert.NoError(t, err) + assert.Len(t, valid, 2) // the Kptfile ("p") plus the ConfigMap ("a") + var validNames []string + for _, n := range valid { + validNames = append(validNames, n.GetName()) + } + assert.ElementsMatch(t, []string{"p", "a"}, validNames) + assert.Len(t, nonKRM, 1) + assert.Equal(t, "b.yaml", nonKRM[0].RelPath) + assert.Equal(t, dir, nonKRM[0].PkgPath) +} + +// TestDecideOutcome_SortedDeterministicMessage verifies the error is +// sorted by pkg path then rel path, with a remediation hint. +func TestDecideOutcome_SortedDeterministicMessage(t *testing.T) { + found := []NonKRMFile{ + {PkgPath: "/root/z", RelPath: "b.txt", Reason: "boom"}, + {PkgPath: "/root/a", RelPath: "a.txt", Reason: "boom"}, + {PkgPath: "/root/a", RelPath: "z.txt", Reason: "boom"}, + } + err := DecideOutcome(found, map[string]bool{}, "/root") + assert.Error(t, err) + msg := err.Error() + idxA1 := indexOf(msg, "a.txt") + idxA2 := indexOf(msg, "z.txt") + idxZ := indexOf(msg, "b.txt") + assert.True(t, idxA1 < idxA2) + assert.True(t, idxA2 < idxZ) + assert.Contains(t, msg, AutoIgnoreNonKRMAnnotation) + assert.Contains(t, msg, ".krmignore") +} + +// TestDecideOutcome_OptedInPackageExcused verifies an opted-in +// package's non-KRM file doesn't block, while a non-opted-in one does. +func TestDecideOutcome_OptedInPackageExcused(t *testing.T) { + found := []NonKRMFile{ + {PkgPath: "/root", RelPath: "blocked.yaml", Reason: "boom"}, + {PkgPath: "/root/sub", RelPath: "excused.yaml", Reason: "boom"}, + } + optedIn := map[string]bool{"/root/sub": true} + err := DecideOutcome(found, optedIn, "/root") + assert.Error(t, err) + assert.Contains(t, err.Error(), "blocked.yaml") + assert.NotContains(t, err.Error(), "excused.yaml") +} + +// TestDecideOutcome_NoBlockingFiles_ReturnsNil verifies an all-excused +// or empty found list returns no error. +func TestDecideOutcome_NoBlockingFiles_ReturnsNil(t *testing.T) { + assert.NoError(t, DecideOutcome(nil, nil, "/root")) + + found := []NonKRMFile{{PkgPath: "/root", RelPath: "a.yaml", Reason: "boom"}} + assert.NoError(t, DecideOutcome(found, map[string]bool{"/root": true}, "/root")) +} + +// TestDecideOutcome_DisambiguatesSameNamedFilesAcrossSubpackages +// verifies the fix for the reviewer's ambiguity report: two +// subpackages each with a same-named non-KRM file must print +// distinguishable, root-relative paths, not both as just the bare +// filename. +func TestDecideOutcome_DisambiguatesSameNamedFilesAcrossSubpackages(t *testing.T) { + found := []NonKRMFile{ + {PkgPath: "/root/apps/app1", RelPath: "values.yaml", Reason: "boom"}, + {PkgPath: "/root/apps/app2", RelPath: "values.yaml", Reason: "boom"}, + } + err := DecideOutcome(found, map[string]bool{}, "/root") + assert.Error(t, err) + msg := err.Error() + assert.Contains(t, msg, "apps/app1/values.yaml") + assert.Contains(t, msg, "apps/app2/values.yaml") +} + +func TestPackageOptedIn(t *testing.T) { + assert.False(t, PackageOptedIn(nil), "nil Kptfile is never opted in") + + fsys := filesys.MakeFsInMemory() + assert.NoError(t, fsys.Mkdir("/plain")) + writeKptfileForTest(t, fsys, "/plain", "plain") + plainPkg, err := pkg.New(fsys, "/plain") + assert.NoError(t, err) + plainKf, err := plainPkg.Kptfile() + assert.NoError(t, err) + assert.False(t, PackageOptedIn(plainKf)) + + assert.NoError(t, fsys.Mkdir("/optedin")) + writeOptedInKptfileForTest(t, fsys, "/optedin", "optedin") + optedInPkg, err := pkg.New(fsys, "/optedin") + assert.NoError(t, err) + optedInKf, err := optedInPkg.Kptfile() + assert.NoError(t, err) + assert.True(t, PackageOptedIn(optedInKf)) +} + +func TestScan(t *testing.T) { + t.Run("clean package: no error, nothing printed, nothing skipped", func(t *testing.T) { + fsys := filesys.MakeFsInMemory() + dir := "/pkg" + assert.NoError(t, fsys.Mkdir(dir)) + writeKptfileForTest(t, fsys, dir, "p") + writeValidKRMForTest(t, fsys, dir) + + printf, printed := printfCollector() + res, err := Scan(fsys, dir, printf) + assert.NoError(t, err) + assert.Empty(t, *printed) + assert.Empty(t, res.SkippedFiles()) + assert.Contains(t, res.AllResources, dir) + }) + + t.Run("non-KRM file with no annotation blocks with an error", func(t *testing.T) { + fsys := filesys.MakeFsInMemory() + dir := "/pkg" + assert.NoError(t, fsys.Mkdir(dir)) + writeKptfileForTest(t, fsys, dir, "p") + writeNonKRMForTest(t, fsys, dir, "bad.yaml") + + printf, _ := printfCollector() + res, err := Scan(fsys, dir, printf) + assert.Error(t, err) + assert.Nil(t, res) + assert.Contains(t, err.Error(), "bad.yaml") + }) + + t.Run("annotated package excuses the non-KRM file and reports the skip", func(t *testing.T) { + fsys := filesys.MakeFsInMemory() + dir := "/pkg" + assert.NoError(t, fsys.Mkdir(dir)) + writeOptedInKptfileForTest(t, fsys, dir, "p") + writeNonKRMForTest(t, fsys, dir, "bad.yaml") + + printf, printed := printfCollector() + res, err := Scan(fsys, dir, printf) + assert.NoError(t, err) + if assert.Len(t, res.SkippedFiles(), 1) { + assert.Equal(t, "bad.yaml", res.SkippedFiles()[0].RelPath) + } + if assert.NotEmpty(t, *printed) { + assert.Contains(t, (*printed)[0], "bad.yaml") + } + }) + + t.Run("non-KRM .json file still blocks: render/eval's real reader does read .json", func(t *testing.T) { + fsys := filesys.MakeFsInMemory() + dir := "/pkg" + assert.NoError(t, fsys.Mkdir(dir)) + writeKptfileForTest(t, fsys, dir, "p") + writeNonKRMForTest(t, fsys, dir, "bad.json") + + printf, _ := printfCollector() + res, err := Scan(fsys, dir, printf) + assert.Error(t, err) + assert.Nil(t, res) + assert.Contains(t, err.Error(), "bad.json") + }) +} + +// TestScan_VsScanHonoringKrmignore pins the deliberate difference: fn +// render's executor still hydrates a subpackage a parent .krmignore +// excludes, so Scan must keep validating it. fn eval/source's readers +// skip it, so they use ScanHonoringKrmignore. +func TestScan_VsScanHonoringKrmignore(t *testing.T) { + newFS := func() filesys.FileSystem { + fsys := filesys.MakeFsInMemory() + fixture := "/root/testdata/fixture" + assert.NoError(t, fsys.MkdirAll(fixture)) + writeKptfileForTest(t, fsys, "/root", "root") + assert.NoError(t, fsys.WriteFile("/root/.krmignore", []byte("testdata/\n"))) + writeKptfileForTest(t, fsys, fixture, "fixture") + writeNonKRMForTest(t, fsys, fixture, "values.yaml") + return fsys + } + printf, _ := printfCollector() + + _, err := Scan(newFS(), "/root", printf) + if assert.Error(t, err, "render path must keep validating the ignored subpackage") { + assert.Contains(t, err.Error(), "testdata/fixture/values.yaml") + } + + res, err := ScanHonoringKrmignore(newFS(), "/root", printf) + assert.NoError(t, err, "eval/source path must skip it, like their real reader") + assert.NotNil(t, res) +} + +func TestScanResult_NilReceiver_IsSafe(t *testing.T) { + var res *ScanResult + assert.Nil(t, res.SkippedFiles()) + assert.Nil(t, res.SkipFunc("/root")) +} + +func TestScanResult_SkipFunc(t *testing.T) { + res := &ScanResult{skipped: []NonKRMFile{ + {PkgPath: "/root/sub", RelPath: "values.yaml"}, + }} + skip := res.SkipFunc("/root") + assert.True(t, skip("sub/values.yaml")) + assert.False(t, skip("sub/other.yaml")) + assert.False(t, skip("values.yaml")) + + assert.Nil(t, (&ScanResult{}).SkipFunc("/root"), "no skipped files means no skip func needed") +} + +func TestScanTree(t *testing.T) { + t.Run("single package: non-KRM file found, resources captured", func(t *testing.T) { + fsys := filesys.MakeFsInMemory() + dir := "/pkg" + assert.NoError(t, fsys.Mkdir(dir)) + writeKptfileForTest(t, fsys, dir, "p") + writeValidKRMForTest(t, fsys, dir) + writeNonKRMForTest(t, fsys, dir, "bad.yaml") + + found, optedIn, allResources, err := ScanTree(fsys, dir, pkg.MatchAllKRM) + assert.NoError(t, err) + if assert.Len(t, found, 1) { + assert.Equal(t, "bad.yaml", found[0].RelPath) + } + assert.False(t, optedIn[dir]) + assert.Contains(t, allResources, dir) + }) + + t.Run("subpackages are walked and each tracked by its own opt-in", func(t *testing.T) { + fsys := filesys.MakeFsInMemory() + root := "/root" + sub := "/root/sub" + assert.NoError(t, fsys.MkdirAll(sub)) + writeKptfileForTest(t, fsys, root, "root") + writeValidKRMForTest(t, fsys, root) + writeOptedInKptfileForTest(t, fsys, sub, "sub") + writeNonKRMForTest(t, fsys, sub, "bad.yaml") + + found, optedIn, allResources, err := ScanTree(fsys, root, pkg.MatchAllKRM) + assert.NoError(t, err) + if assert.Len(t, found, 1) { + assert.Equal(t, sub, found[0].PkgPath) + } + assert.False(t, optedIn[root]) + assert.True(t, optedIn[sub]) + assert.Contains(t, allResources, root) + assert.Contains(t, allResources, sub) + }) + + t.Run("root .krmignore skips a subpackage under an ignored directory", func(t *testing.T) { + fsys := filesys.MakeFsInMemory() + root := "/root" + fixture := "/root/testdata/fixture" + assert.NoError(t, fsys.MkdirAll(fixture)) + writeKptfileForTest(t, fsys, root, "root") + writeValidKRMForTest(t, fsys, root) + assert.NoError(t, fsys.WriteFile(filepath.Join(root, ".krmignore"), []byte("testdata/\n"))) + writeKptfileForTest(t, fsys, fixture, "fixture") + writeNonKRMForTest(t, fsys, fixture, "values.yaml") + + found, _, allResources, err := scanTree(fsys, root, pkg.MatchAllKRM, true) + assert.NoError(t, err) + assert.Empty(t, found, "an ignored directory's subpackage must never be scanned") + assert.NotContains(t, allResources, fixture) + }) + + t.Run("root .krmignore skips a directly ignored subpackage", func(t *testing.T) { + fsys := filesys.MakeFsInMemory() + root := "/root" + ignored := "/root/ignored" + assert.NoError(t, fsys.MkdirAll(ignored)) + writeKptfileForTest(t, fsys, root, "root") + assert.NoError(t, fsys.WriteFile(filepath.Join(root, ".krmignore"), []byte("ignored/\n"))) + writeKptfileForTest(t, fsys, ignored, "ignored") + writeNonKRMForTest(t, fsys, ignored, "values.yaml") + + found, _, allResources, err := scanTree(fsys, root, pkg.MatchAllKRM, true) + assert.NoError(t, err) + assert.Empty(t, found) + assert.NotContains(t, allResources, ignored) + }) + + t.Run("ignoring one sibling subpackage still reports a non-KRM file in another", func(t *testing.T) { + fsys := filesys.MakeFsInMemory() + root := "/root" + ok := "/root/ok" + skipme := "/root/skipme" + assert.NoError(t, fsys.MkdirAll(ok)) + assert.NoError(t, fsys.MkdirAll(skipme)) + writeKptfileForTest(t, fsys, root, "root") + assert.NoError(t, fsys.WriteFile(filepath.Join(root, ".krmignore"), []byte("skipme/\n"))) + writeKptfileForTest(t, fsys, ok, "ok") + writeNonKRMForTest(t, fsys, ok, "values.yaml") + writeKptfileForTest(t, fsys, skipme, "skipme") + writeNonKRMForTest(t, fsys, skipme, "values.yaml") + + found, _, _, err := scanTree(fsys, root, pkg.MatchAllKRM, true) + assert.NoError(t, err) + if assert.Len(t, found, 1, "pruning must be selective, not a blanket skip") { + assert.Equal(t, ok, found[0].PkgPath) + } + }) + + t.Run("a parent .krmignore file pattern does not reach into a non-ignored subpackage", func(t *testing.T) { + // Matches the real fn reader: .krmignore covers only its own + // package, so a subpackage's file is still scanned. + fsys := filesys.MakeFsInMemory() + root := "/root" + sub := "/root/sub" + assert.NoError(t, fsys.MkdirAll(sub)) + writeKptfileForTest(t, fsys, root, "root") + assert.NoError(t, fsys.WriteFile(filepath.Join(root, ".krmignore"), []byte("values.yaml\n"))) + writeKptfileForTest(t, fsys, sub, "sub") + writeNonKRMForTest(t, fsys, sub, "values.yaml") + + found, _, _, err := scanTree(fsys, root, pkg.MatchAllKRM, true) + assert.NoError(t, err) + assert.Len(t, found, 1) + }) + + t.Run("bare directory with no root Kptfile is still scanned", func(t *testing.T) { + fsys := filesys.MakeFsInMemory() + dir := "/bare" + assert.NoError(t, fsys.Mkdir(dir)) + writeNonKRMForTest(t, fsys, dir, "bad.yaml") + + found, optedIn, _, err := ScanTree(fsys, dir, pkg.MatchAllKRM) + assert.NoError(t, err) + assert.Len(t, found, 1) + assert.False(t, optedIn[dir]) + }) + + t.Run("nonexistent target directory propagates an error", func(t *testing.T) { + fsys := filesys.MakeFsInMemory() + _, _, _, err := ScanTree(fsys, "/does/not/exist", pkg.MatchAllKRM) + assert.Error(t, err) + }) + + t.Run("a candidate subpackage with an unparseable Kptfile is silently not discovered, not an error", func(t *testing.T) { + fsys := filesys.MakeFsInMemory() + root := "/root" + sub := "/root/sub" + assert.NoError(t, fsys.MkdirAll(sub)) + writeKptfileForTest(t, fsys, root, "root") + assert.NoError(t, fsys.WriteFile(filepath.Join(sub, "Kptfile"), []byte("not: [valid, kptfile\n"))) + + found, optedIn, allResources, err := ScanTree(fsys, root, pkg.MatchAllKRM) + assert.NoError(t, err) + assert.Empty(t, found) + assert.Contains(t, allResources, root) + assert.NotContains(t, allResources, sub, "an unparseable Kptfile means the directory is never recognized as a subpackage") + assert.NotContains(t, optedIn, sub) + }) +} + +func TestScanFlat(t *testing.T) { + t.Run("root opt-in is reported regardless of where found files actually live", func(t *testing.T) { + fsys := filesys.MakeFsInMemory() + root := "/root" + sub := "/root/sub" + assert.NoError(t, fsys.MkdirAll(sub)) + writeOptedInKptfileForTest(t, fsys, root, "root") + writeNonKRMForTest(t, fsys, root, "root-bad.yaml") + writeKptfileForTest(t, fsys, sub, "sub") + writeNonKRMForTest(t, fsys, sub, "sub-bad.yaml") + + found, optedIn, allResources, err := ScanFlat(fsys, root) + assert.NoError(t, err) + assert.True(t, optedIn) + assert.Len(t, found, 2) + assert.NotEmpty(t, allResources) + }) + + t.Run("clean tree returns no found files", func(t *testing.T) { + fsys := filesys.MakeFsInMemory() + dir := "/pkg" + assert.NoError(t, fsys.Mkdir(dir)) + writeKptfileForTest(t, fsys, dir, "p") + writeValidKRMForTest(t, fsys, dir) + + found, optedIn, _, err := ScanFlat(fsys, dir) + assert.NoError(t, err) + assert.False(t, optedIn) + assert.Empty(t, found) + }) + + t.Run("non-KRM .json file is never flagged: the live reader never reads .json", func(t *testing.T) { + fsys := filesys.MakeFsInMemory() + dir := "/pkg" + assert.NoError(t, fsys.Mkdir(dir)) + writeKptfileForTest(t, fsys, dir, "p") + writeValidKRMForTest(t, fsys, dir) + writeNonKRMForTest(t, fsys, dir, "unrelated.json") + + found, optedIn, _, err := ScanFlat(fsys, dir) + assert.NoError(t, err) + assert.False(t, optedIn) + assert.Empty(t, found) + }) +} + +func TestScanFlatAndDecide(t *testing.T) { + t.Run("blocking non-KRM file returns an error, nothing printed", func(t *testing.T) { + fsys := filesys.MakeFsInMemory() + dir := "/pkg" + assert.NoError(t, fsys.Mkdir(dir)) + writeKptfileForTest(t, fsys, dir, "p") + writeNonKRMForTest(t, fsys, dir, "bad.yaml") + + printf, printed := printfCollector() + res, err := ScanFlatAndDecide(fsys, dir, printf) + assert.Error(t, err) + assert.Nil(t, res) + assert.Contains(t, err.Error(), "bad.yaml") + assert.Empty(t, *printed) + }) + + t.Run("excused subpackage file is reported and skipped", func(t *testing.T) { + fsys := filesys.MakeFsInMemory() + root := "/root" + sub := "/root/sub" + assert.NoError(t, fsys.MkdirAll(sub)) + writeKptfileForTest(t, fsys, root, "root") + writeOptedInKptfileForTest(t, fsys, sub, "sub") + writeNonKRMForTest(t, fsys, sub, "bad.yaml") + + printf, printed := printfCollector() + res, err := ScanFlatAndDecide(fsys, root, printf) + assert.NoError(t, err) + if assert.Len(t, res.SkippedFiles(), 1) { + assert.Equal(t, "bad.yaml", res.SkippedFiles()[0].RelPath) + } + assert.NotEmpty(t, *printed) + }) + + t.Run("clean tree returns an empty result, nothing printed", func(t *testing.T) { + fsys := filesys.MakeFsInMemory() + dir := "/pkg" + assert.NoError(t, fsys.Mkdir(dir)) + writeKptfileForTest(t, fsys, dir, "p") + writeValidKRMForTest(t, fsys, dir) + + printf, printed := printfCollector() + res, err := ScanFlatAndDecide(fsys, dir, printf) + assert.NoError(t, err) + assert.Empty(t, res.SkippedFiles()) + assert.Empty(t, *printed) + }) + + t.Run("nameless local-config resource does not block", func(t *testing.T) { + fsys := filesys.MakeFsInMemory() + dir := "/pkg" + assert.NoError(t, fsys.Mkdir(dir)) + writeKptfileForTest(t, fsys, dir, "p") + content := "apiVersion: v1\nkind: ConfigMap\nmetadata:\n" + + " annotations:\n config.kubernetes.io/local-config: \"true\"\n" + assert.NoError(t, fsys.WriteFile(filepath.Join(dir, "lc.yaml"), []byte(content))) + + printf, printed := printfCollector() + res, err := ScanFlatAndDecide(fsys, dir, printf) + assert.NoError(t, err) + assert.Empty(t, res.SkippedFiles()) + assert.Empty(t, *printed) + }) + + t.Run("root .krmignore reaches into a subpackage file, same as the real flat reader", func(t *testing.T) { + fsys := filesys.MakeFsInMemory() + root := "/root" + sub := "/root/sub" + assert.NoError(t, fsys.MkdirAll(sub)) + writeKptfileForTest(t, fsys, root, "root") + assert.NoError(t, fsys.WriteFile(filepath.Join(root, ".krmignore"), []byte("values.yaml\n"))) + writeKptfileForTest(t, fsys, sub, "sub") + writeValidKRMForTest(t, fsys, sub) + writeNonKRMForTest(t, fsys, sub, "values.yaml") + + printf, printed := printfCollector() + res, err := ScanFlatAndDecide(fsys, root, printf) + assert.NoError(t, err, "root's .krmignore must cover sub/values.yaml, same as the real reader") + assert.Empty(t, res.SkippedFiles()) + assert.Empty(t, *printed) + }) + + t.Run("root .krmignore excludes a whole directory holding a fixture Kptfile", func(t *testing.T) { + fsys := filesys.MakeFsInMemory() + root := "/root" + testdata := "/root/testdata" + assert.NoError(t, fsys.MkdirAll(testdata)) + writeKptfileForTest(t, fsys, root, "root") + assert.NoError(t, fsys.WriteFile(filepath.Join(root, ".krmignore"), []byte("testdata/\n"))) + writeKptfileForTest(t, fsys, testdata, "fixture") + writeNonKRMForTest(t, fsys, testdata, "values.yaml") + + printf, printed := printfCollector() + res, err := ScanFlatAndDecide(fsys, root, printf) + assert.NoError(t, err, "the whole ignored directory must never be scanned, same as the real reader") + assert.Empty(t, res.SkippedFiles()) + assert.Empty(t, *printed) + }) + + t.Run("AllResources includes subpackage resources too, matching the real reader's own flat read", func(t *testing.T) { + fsys := filesys.MakeFsInMemory() + root := "/root" + sub := "/root/sub" + assert.NoError(t, fsys.MkdirAll(sub)) + writeKptfileForTest(t, fsys, root, "root") + writeValidKRMForTest(t, fsys, root) + writeKptfileForTest(t, fsys, sub, "sub") + writeValidKRMForTest(t, fsys, sub) + + printf, _ := printfCollector() + res, err := ScanFlatAndDecide(fsys, root, printf) + require.NoError(t, err) + assert.Len(t, res.AllResources, 2, "both root's and sub's resource") + }) + + t.Run("non-KRM .json file does not block: the live reader never reads .json", func(t *testing.T) { + fsys := filesys.MakeFsInMemory() + dir := "/pkg" + assert.NoError(t, fsys.Mkdir(dir)) + writeKptfileForTest(t, fsys, dir, "p") + writeNonKRMForTest(t, fsys, dir, "unrelated.json") + + printf, printed := printfCollector() + res, err := ScanFlatAndDecide(fsys, dir, printf) + assert.NoError(t, err) + assert.Empty(t, res.SkippedFiles()) + assert.Empty(t, *printed) + }) +} + +func TestFlatScanResult_NilReceiver_IsSafe(t *testing.T) { + var res *FlatScanResult + assert.Nil(t, res.SkippedFiles()) + assert.Nil(t, res.SkipFunc("/root")) +} + +func TestFlatScanResult_SkipFunc_ReRootsSubpackagePaths(t *testing.T) { + res := &FlatScanResult{skipped: []NonKRMFile{ + {PkgPath: "/root/sub", RelPath: "values.yaml"}, + }} + skip := res.SkipFunc("/root") + assert.True(t, skip("sub/values.yaml"), "a subpackage file's skip path must be re-rooted against pkgRootPath") + assert.False(t, skip("values.yaml")) + + assert.Nil(t, (&FlatScanResult{}).SkipFunc("/root")) +} + +func TestCachedLocalResources(t *testing.T) { + fsys := filesys.MakeFsInMemory() + dir := "/pkg" + assert.NoError(t, fsys.Mkdir(dir)) + writeKptfileForTest(t, fsys, dir, "p") + writeValidKRMForTest(t, fsys, dir) + + p, err := pkg.New(fsys, dir) + assert.NoError(t, err) + + t.Run("cache miss falls back to a real read", func(t *testing.T) { + resources, err := CachedLocalResources(p, nil) + assert.NoError(t, err) + assert.NotEmpty(t, resources) + }) + + t.Run("cache hit returns the cached slice, not a fresh read", func(t *testing.T) { + cache := map[string][]*yaml.RNode{p.UniquePath.String(): {}} // deliberately empty, unlike a real read + resources, err := CachedLocalResources(p, cache) + assert.NoError(t, err) + assert.Empty(t, resources, "must return the cached (empty) slice, not re-read from disk") + }) +} + +func TestPrintSkippedSummary(t *testing.T) { + t.Run("nothing skipped: no-op", func(t *testing.T) { + printf, printed := printfCollector() + PrintSkippedSummary(printf, nil, nil) + assert.Empty(t, *printed) + }) + + t.Run("multiple opted-in packages are printed once each, sorted by path", func(t *testing.T) { + found := []NonKRMFile{ + {PkgPath: "/root/z", RelPath: "b.yaml"}, + {PkgPath: "/root/a", RelPath: "a.yaml"}, + {PkgPath: "/root/a", RelPath: "c.yaml"}, + {PkgPath: "/root/not-opted-in", RelPath: "skip-me.yaml"}, + } + optedIn := map[string]bool{"/root/z": true, "/root/a": true} + + printf, printed := printfCollector() + PrintSkippedSummary(printf, found, optedIn) + + if assert.Len(t, *printed, 2) { + assert.Contains(t, (*printed)[0], "/root/a") + assert.Contains(t, (*printed)[0], "a.yaml") + assert.Contains(t, (*printed)[0], "c.yaml") + assert.Contains(t, (*printed)[1], "/root/z") + assert.Contains(t, (*printed)[1], "b.yaml") + } + for _, line := range *printed { + assert.NotContains(t, line, "skip-me.yaml") + } + }) +} + +func indexOf(haystack, needle string) int { + for i := 0; i+len(needle) <= len(haystack); i++ { + if haystack[i:i+len(needle)] == needle { + return i + } + } + return -1 +} + +// TestScanIgnoredSubpackageUnreadableKptfile pins that neither +// ScanHonoringKrmignore (fn eval/source) nor ScanFlatAndDecide (live) +// reads or parses a Kptfile inside a directory excluded by .krmignore, +// while still failing when an unignored subpackage has an unreadable Kptfile. +func TestScanIgnoredSubpackageUnreadableKptfile(t *testing.T) { + cases := []struct { + name string + kptfile string + }{ + { + name: "v1alpha1 Kptfile", + kptfile: "apiVersion: kpt.dev/v1alpha1\nkind: Kptfile\nmetadata:\n name: fixture\n", + }, + { + name: "invalid YAML Kptfile", + kptfile: ": invalid: [yaml\n", + }, + { + name: "unknown field in Kptfile", + kptfile: "apiVersion: kpt.dev/v1\nkind: Kptfile\nmetadata:\n name: fixture\nunknownField: value\n", + }, + { + name: "empty Kptfile", + kptfile: "", + }, + } + + for _, tc := range cases { + t.Run("ScanHonoringKrmignore ignores "+tc.name, func(t *testing.T) { + fsys := filesys.MakeFsOnDisk() + root := t.TempDir() + fixture := filepath.Join(root, "testdata", "fixture") + require.NoError(t, fsys.MkdirAll(fixture)) + writeKptfileForTest(t, fsys, root, "root") + writeValidKRMForTest(t, fsys, root) + require.NoError(t, fsys.WriteFile(filepath.Join(root, ".krmignore"), []byte("testdata/\n"))) + require.NoError(t, fsys.WriteFile(filepath.Join(fixture, "Kptfile"), []byte(tc.kptfile))) + + printf, _ := printfCollector() + res, err := ScanHonoringKrmignore(fsys, root, printf) + assert.NoError(t, err) + assert.NotContains(t, res.AllResources, fixture) + }) + + t.Run("ScanFlatAndDecide ignores "+tc.name, func(t *testing.T) { + fsys := filesys.MakeFsOnDisk() + root := t.TempDir() + fixture := filepath.Join(root, "testdata", "fixture") + require.NoError(t, fsys.MkdirAll(fixture)) + writeKptfileForTest(t, fsys, root, "root") + writeValidKRMForTest(t, fsys, root) + require.NoError(t, fsys.WriteFile(filepath.Join(root, ".krmignore"), []byte("testdata/\n"))) + require.NoError(t, fsys.WriteFile(filepath.Join(fixture, "Kptfile"), []byte(tc.kptfile))) + + printf, _ := printfCollector() + _, err := ScanFlatAndDecide(fsys, root, printf) + assert.NoError(t, err) + }) + } + + for _, tc := range cases[:3] { + t.Run("ScanHonoringKrmignore still fails on unignored "+tc.name, func(t *testing.T) { + fsys := filesys.MakeFsOnDisk() + root := t.TempDir() + sub := filepath.Join(root, "sub") + require.NoError(t, fsys.MkdirAll(sub)) + writeKptfileForTest(t, fsys, root, "root") + require.NoError(t, fsys.WriteFile(filepath.Join(sub, "Kptfile"), []byte(tc.kptfile))) + + printf, _ := printfCollector() + _, err := ScanHonoringKrmignore(fsys, root, printf) + assert.Error(t, err) + }) + + t.Run("ScanFlatAndDecide still fails on unignored "+tc.name, func(t *testing.T) { + fsys := filesys.MakeFsOnDisk() + root := t.TempDir() + sub := filepath.Join(root, "sub") + require.NoError(t, fsys.MkdirAll(sub)) + writeKptfileForTest(t, fsys, root, "root") + require.NoError(t, fsys.WriteFile(filepath.Join(sub, "Kptfile"), []byte(tc.kptfile))) + + printf, _ := printfCollector() + _, err := ScanFlatAndDecide(fsys, root, printf) + assert.Error(t, err) + }) + } +} diff --git a/pkg/lib/update/pruninglocalpackagereader.go b/pkg/lib/update/pruninglocalpackagereader.go index 78b7173ec7..3d75caeb52 100644 --- a/pkg/lib/update/pruninglocalpackagereader.go +++ b/pkg/lib/update/pruninglocalpackagereader.go @@ -17,6 +17,7 @@ package update import ( "strings" + "github.com/kptdev/kpt/pkg/lib/krmvalidate" "sigs.k8s.io/kustomize/kyaml/kio" "sigs.k8s.io/kustomize/kyaml/kio/kioutil" "sigs.k8s.io/kustomize/kyaml/yaml" @@ -27,6 +28,12 @@ import ( type PruningLocalPackageReader struct { LocalPackageReader kio.LocalPackageReader Exclusions []string + + // ExcludeNonKRM, if true, also drops any node failing + // krmvalidate.IsKRM() (e.g. a Helm values.yaml), so it's handled + // by ReplaceNonKRMFiles's raw-copy path instead of the 3-way + // merge. Must agree with getSubDirsAndNonKrmFiles's classification. + ExcludeNonKRM bool } func (p PruningLocalPackageReader) Read() ([]*yaml.RNode, error) { @@ -52,7 +59,31 @@ func (p PruningLocalPackageReader) Read() ([]*yaml.RNode, error) { } filteredNodes = append(filteredNodes, node) } - return filteredNodes, nil + if !p.ExcludeNonKRM { + return filteredNodes, nil + } + + nonKRMPaths := make(map[string]bool) + for _, node := range filteredNodes { + if krmvalidate.IsKRM(node) != nil { + path, _, err := kioutil.GetFileAnnotations(node) + if err != nil { + return nil, err + } + nonKRMPaths[path] = true + } + } + var validNodes []*yaml.RNode + for _, node := range filteredNodes { + path, _, err := kioutil.GetFileAnnotations(node) + if err != nil { + return nil, err + } + if !nonKRMPaths[path] { + validNodes = append(validNodes, node) + } + } + return validNodes, nil } func (p PruningLocalPackageReader) isExcluded(path string) bool { diff --git a/pkg/lib/update/resource-merge.go b/pkg/lib/update/resource-merge.go index 93ed3aa630..7980a0e3e3 100644 --- a/pkg/lib/update/resource-merge.go +++ b/pkg/lib/update/resource-merge.go @@ -22,6 +22,7 @@ import ( "strings" ko "github.com/kptdev/kpt/api/kubeobject" + "github.com/kptdev/kpt/pkg/lib/krmvalidate" pkgdiff "github.com/kptdev/kpt/pkg/lib/pkg/diff" "github.com/kptdev/kpt/pkg/lib/update/merge3" "sigs.k8s.io/kustomize/kyaml/pathutil" @@ -275,8 +276,9 @@ func ReplaceNonKRMFiles(updatedDir, originalDir, localDir string) error { return nil } -// getSubDirsAndNonKrmFiles returns the list of all non git sub dirs and, non git+non KRM files -// in the root directory +// getSubDirsAndNonKrmFiles returns the list of all non git sub dirs and, +// non git+non KRM files in the root directory. Must classify a given +// file the same way PruningLocalPackageReader{ExcludeNonKRM: true} does. func getSubDirsAndNonKrmFiles(root string) (sets.String, sets.String, error) { const op errors.Op = "update.getSubDirsAndNonKrmFiles" files := sets.String{} @@ -313,18 +315,50 @@ func getSubDirsAndNonKrmFiles(root string) (sets.String, sets.String, error) { var krmFilesGlob = append([]string{kptfilev1.KptFileName}, kio.DefaultMatch...) -// isKrmFile checks if the file pointed to by the path is a yaml file (including -// the Kptfile). +// isKrmFile reports whether path's filename matches krmFilesGlob AND +// its content is valid KRM per krmvalidate.IsKRM(). +// +// Parsed with DisableUnwrapping so a bare `kind: List` stays itself +// instead of being unwrapped into its items, matching how the real merge +// reader (kio.LocalPackageReader) classifies the same file. func isKrmFile(path string) (bool, error) { const op errors.Op = "update.isKrmFile" + matchesGlob := false for _, g := range krmFilesGlob { if match, err := filepath.Match(g, filepath.Base(path)); err != nil { return false, errors.E(op, err) } else if match { + matchesGlob = true + break + } + } + if !matchesGlob { + return false, nil + } + + content, err := os.ReadFile(path) + if err != nil { + return false, errors.E(op, errors.IO, err) + } + nodes, err := (&kio.ByteReader{ + Reader: bytes.NewReader(content), + OmitReaderAnnotations: true, + DisableUnwrapping: true, + }).Read() + if err != nil { + return false, nil + } + for _, name := range kptfilev1.RecognizedKustomizationFileNames() { + if filepath.Base(path) == name { return true, nil } } - return false, nil + for _, n := range nodes { + if krmvalidate.IsKRM(n) != nil { + return false, nil + } + } + return true, nil } // compareFiles returns true if src file content is equal to dst file content @@ -395,7 +429,8 @@ func loadResourcesFromDirectory(directoryPath string, mergeSourceAnnotation stri PreserveSeqIndent: true, WrapBareSeqNode: true, }, - Exclusions: exclusions, + Exclusions: exclusions, + ExcludeNonKRM: true, } nodes, err := reader.Read() if err != nil { @@ -412,6 +447,9 @@ func writeResourcesToDirectory(destPath string, kubeObjects ko.KubeObjects) erro SetAnnotations: map[string]string{"config.kubernetes.io/merge-source": "dest"}, PreserveSeqIndent: true, WrapBareSeqNode: true, + // Skip non-KRM files so Write never deletes them as + // "read but not in the output"; ReplaceNonKRMFiles owns them. + FileSkipFunc: skipNonKrmFileFunc(destPath), } _, err := destWriter.Read() if err != nil { @@ -423,3 +461,12 @@ func writeResourcesToDirectory(destPath string, kubeObjects ko.KubeObjects) erro } return nil } + +// skipNonKrmFileFunc returns a FileSkipFunc matching isKrmFile's +// classification, for readers that need to leave non-KRM files alone. +func skipNonKrmFileFunc(root string) func(relPath string) bool { + return func(relPath string) bool { + isKrm, err := isKrmFile(filepath.Join(root, relPath)) + return err != nil || !isKrm + } +} diff --git a/pkg/lib/update/resource-merge_test.go b/pkg/lib/update/resource-merge_test.go index d869c7320d..19b14082ce 100644 --- a/pkg/lib/update/resource-merge_test.go +++ b/pkg/lib/update/resource-merge_test.go @@ -305,6 +305,229 @@ func TestUpdate_ResourceMerge(t *testing.T) { } } +// TestUpdate_NonKRMFileNotMergedByContent verifies a .yaml file whose +// content isn't valid KRM is raw-copied/diffed, never merged. +func TestUpdate_NonKRMFileNotMergedByContent(t *testing.T) { + origin := pkgbuilder.NewRootPkg(). + WithResource(pkgbuilder.DeploymentResource). + WithFile("values.yaml", "replicaCount: 3\nimage:\n repository: nginx\n tag: latest\n") + local := pkgbuilder.NewRootPkg(). + WithKptfile( + pkgbuilder.NewKptfile(). + WithUpstream(kptRepo, "/", "master", "resource-merge"). + WithUpstreamLock(kptRepo, "/", "master", "abc123"), + ). + WithResource(pkgbuilder.DeploymentResource). + WithFile("values.yaml", "replicaCount: 3\nimage:\n repository: nginx\n tag: latest\n# local edit\n") + updated := pkgbuilder.NewRootPkg(). + WithResource(pkgbuilder.DeploymentResource). + WithFile("values.yaml", "replicaCount: 5\ntotallyDifferentField: yes\n") + // Locally modified: local's edit is kept untouched, same as + // ReplaceNonKRMFiles already does for any other non-KRM file. + expected := pkgbuilder.NewRootPkg(). + WithKptfile( + pkgbuilder.NewKptfile(). + WithUpstream(kptRepo, "/", "master", "resource-merge"). + WithUpstreamLock(kptRepo, "/", "master", "abc123"), + ). + WithResource(pkgbuilder.DeploymentResource). + WithFile("values.yaml", "replicaCount: 3\nimage:\n repository: nginx\n tag: latest\n# local edit\n") + + repos := testutil.EmptyReposInfo + originPath := origin.ExpandPkg(t, repos) + localPath := local.ExpandPkg(t, repos) + updatedPath := updated.ExpandPkg(t, repos) + + updater := &update.ResourceMergeUpdater{} + err := updater.Update(updatetypes.Options{ + RelPackagePath: "/", + OriginPath: originPath, + LocalPath: localPath, + UpdatedPath: updatedPath, + IsRoot: true, + }) + require.NoError(t, err) + + gotContent, err := os.ReadFile(filepath.Join(localPath, "values.yaml")) + require.NoError(t, err) + assert.Equal(t, "replicaCount: 3\nimage:\n repository: nginx\n tag: latest\n# local edit\n", string(gotContent), + "non-KRM file must never be field-merged, and a locally modified file is kept, not overwritten") + + expectedPath := expected.ExpandPkg(t, repos) + testutil.KptfileAwarePkgEqual(t, localPath, expectedPath, false) +} + +// TestUpdate_NonKRMFile_LocalOnlySurvives: a non-KRM file that exists +// only locally (absent from both origin and upstream) must survive an +// update untouched, not be deleted by the merge-destination writer +// before ReplaceNonKRMFiles runs. +func TestUpdate_NonKRMFile_LocalOnlySurvives(t *testing.T) { + origin := pkgbuilder.NewRootPkg(). + WithResource(pkgbuilder.DeploymentResource) + local := pkgbuilder.NewRootPkg(). + WithKptfile( + pkgbuilder.NewKptfile(). + WithUpstream(kptRepo, "/", "master", "resource-merge"). + WithUpstreamLock(kptRepo, "/", "master", "abc123"), + ). + WithResource(pkgbuilder.DeploymentResource). + WithFile("values.yaml", "replicaCount: 3\n") + updated := pkgbuilder.NewRootPkg(). + WithResource(pkgbuilder.DeploymentResource) + + repos := testutil.EmptyReposInfo + originPath := origin.ExpandPkg(t, repos) + localPath := local.ExpandPkg(t, repos) + updatedPath := updated.ExpandPkg(t, repos) + + updater := &update.ResourceMergeUpdater{} + err := updater.Update(updatetypes.Options{ + RelPackagePath: "/", + OriginPath: originPath, + LocalPath: localPath, + UpdatedPath: updatedPath, + IsRoot: true, + }) + require.NoError(t, err) + + gotContent, err := os.ReadFile(filepath.Join(localPath, "values.yaml")) + require.NoError(t, err, "local-only non-KRM file must not be deleted during update") + assert.Equal(t, "replicaCount: 3\n", string(gotContent)) +} + +// TestUpdate_NonKRMListFileSurvives verifies a file with a bare `kind: +// List` (no apiVersion of its own) wrapping a valid inner resource is +// classified consistently by isKrmFile and the merge reader, so it's +// preserved rather than silently deleted as "no longer in the merge output." +func TestUpdate_NonKRMListFileSurvives(t *testing.T) { + const listFileContent = `kind: List +items: +- apiVersion: v1 + kind: ConfigMap + metadata: + name: cm1 + data: + key: value +` + origin := pkgbuilder.NewRootPkg(). + WithResource(pkgbuilder.DeploymentResource). + WithFile("list.yaml", listFileContent) + local := pkgbuilder.NewRootPkg(). + WithKptfile( + pkgbuilder.NewKptfile(). + WithUpstream(kptRepo, "/", "master", "resource-merge"). + WithUpstreamLock(kptRepo, "/", "master", "abc123"), + ). + WithResource(pkgbuilder.DeploymentResource). + WithFile("list.yaml", listFileContent) + // Only the unrelated Deployment resource changes upstream; list.yaml + // itself is untouched between origin and updated. + updated := pkgbuilder.NewRootPkg(). + WithResource(pkgbuilder.ConfigMapResource). + WithFile("list.yaml", listFileContent) + + repos := testutil.EmptyReposInfo + originPath := origin.ExpandPkg(t, repos) + localPath := local.ExpandPkg(t, repos) + updatedPath := updated.ExpandPkg(t, repos) + + updater := &update.ResourceMergeUpdater{} + err := updater.Update(updatetypes.Options{ + RelPackagePath: "/", + OriginPath: originPath, + LocalPath: localPath, + UpdatedPath: updatedPath, + IsRoot: true, + }) + require.NoError(t, err) + + gotContent, err := os.ReadFile(filepath.Join(localPath, "list.yaml")) + require.NoError(t, err, "list.yaml must not be deleted during update") + assert.Equal(t, listFileContent, string(gotContent)) +} + +// TestUpdate_GenerateNameJobMerged verifies a Job using +// metadata.generateName (no metadata.name) is classified as KRM and +// goes through the real 3-way merge, so a local edit (added label) and +// an upstream edit (changed image) both land in the result. Before the +// fix, IsKRM required metadata.name unconditionally, so this file was +// misclassified non-KRM and routed to ReplaceNonKRMFiles's raw-copy +// path - which, for a locally modified file, keeps it untouched and +// silently drops the upstream image change entirely. +func TestUpdate_GenerateNameJobMerged(t *testing.T) { + const origJob = `apiVersion: batch/v1 +kind: Job +metadata: + generateName: my-job- +spec: + template: + spec: + containers: + - name: c + image: busybox:1.0 +` + const localJob = `apiVersion: batch/v1 +kind: Job +metadata: + generateName: my-job- + labels: + local: "true" +spec: + template: + spec: + containers: + - name: c + image: busybox:1.0 +` + const updatedJob = `apiVersion: batch/v1 +kind: Job +metadata: + generateName: my-job- +spec: + template: + spec: + containers: + - name: c + image: busybox:2.0 +` + origin := pkgbuilder.NewRootPkg(). + WithResource(pkgbuilder.DeploymentResource). + WithFile("job.yaml", origJob) + local := pkgbuilder.NewRootPkg(). + WithKptfile( + pkgbuilder.NewKptfile(). + WithUpstream(kptRepo, "/", "master", "resource-merge"). + WithUpstreamLock(kptRepo, "/", "master", "abc123"), + ). + WithResource(pkgbuilder.DeploymentResource). + WithFile("job.yaml", localJob) + updated := pkgbuilder.NewRootPkg(). + WithResource(pkgbuilder.DeploymentResource). + WithFile("job.yaml", updatedJob) + + repos := testutil.EmptyReposInfo + originPath := origin.ExpandPkg(t, repos) + localPath := local.ExpandPkg(t, repos) + updatedPath := updated.ExpandPkg(t, repos) + + updater := &update.ResourceMergeUpdater{} + err := updater.Update(updatetypes.Options{ + RelPackagePath: "/", + OriginPath: originPath, + LocalPath: localPath, + UpdatedPath: updatedPath, + IsRoot: true, + }) + require.NoError(t, err) + + gotContent, err := os.ReadFile(filepath.Join(localPath, "job.yaml")) + require.NoError(t, err) + got := string(gotContent) + assert.Contains(t, got, "local: \"true\"", "the local label must survive the merge") + assert.Contains(t, got, "busybox:2.0", "the upstream image change must not be silently dropped") + assert.NotContains(t, got, "busybox:1.0", "the stale local image must be replaced, not kept verbatim") +} + func TestKeepTaggedNull(t *testing.T) { updater := update.ResourceMergeUpdater{} diff --git a/pkg/lib/update/update_test.go b/pkg/lib/update/update_test.go index f8e134da92..869fb924c3 100644 --- a/pkg/lib/update/update_test.go +++ b/pkg/lib/update/update_test.go @@ -4082,10 +4082,12 @@ func TestReplaceNonKRMFiles(t *testing.T) { err = os.WriteFile(filepath.Join(expectedLocal, "somefunction.py"), []byte("Print some other thing"), 0600) assert.NoError(t, err) } - // Add a yaml file in updated that should never be moved to - // expectedLocal. + // A .yaml file whose content isn't valid KRM; must be + // synced in by this raw-copy path like any other new file. err = os.WriteFile(filepath.Join(updated, "new.yaml"), []byte("a: b"), 0600) assert.NoError(t, err) + err = os.WriteFile(filepath.Join(expectedLocal, "new.yaml"), []byte("a: b"), 0600) + assert.NoError(t, err) err = update.ReplaceNonKRMFiles(updated, original, local) assert.NoError(t, err) tg := testutil.TestGitRepo{} diff --git a/pkg/lib/util/attribution/attribution.go b/pkg/lib/util/attribution/attribution.go index 5d3d31ab41..0a27be3173 100644 --- a/pkg/lib/util/attribution/attribution.go +++ b/pkg/lib/util/attribution/attribution.go @@ -39,6 +39,11 @@ type Attributor struct { // CmdGroup is the command groups in krm, e.g., pkg, fn, live CmdGroup string + + // FileSkipFunc, if set, is passed to processPackage's own reader + // so a file the caller's non-KRM pre-scan already excused is + // never read here either, not just left unwritten. + FileSkipFunc kio.LocalPackageSkipFileFunc } // Process invokes Attribution kyaml filter on the resources in input packages paths @@ -71,7 +76,12 @@ func (a *Attributor) Process() { func (a *Attributor) processPackage(path string) error { changedFiles := map[string]bool{} return kio.Pipeline{ - Inputs: []kio.Reader{kio.LocalPackageReader{PackagePath: path, PreserveSeqIndent: true, WrapBareSeqNode: true}}, + Inputs: []kio.Reader{kio.LocalPackageReader{ + PackagePath: path, + PreserveSeqIndent: true, + WrapBareSeqNode: true, + FileSkipFunc: a.FileSkipFunc, + }}, Filters: []kio.Filter{kio.FilterAll(kyaml.FilterFunc(func(node *kyaml.RNode) (*kyaml.RNode, error) { prevAnnoVal := node.GetAnnotations()[CNRMMetricsAnnotation] if _, err := a.Filter(node); err != nil { diff --git a/pkg/live/load.go b/pkg/live/load.go index 7d39bcf7ea..c7f9eba3af 100644 --- a/pkg/live/load.go +++ b/pkg/live/load.go @@ -16,6 +16,7 @@ package live import ( "bytes" + "context" "fmt" "io" @@ -24,9 +25,11 @@ import ( schema "github.com/kptdev/kpt/api/schema/v1" "github.com/kptdev/kpt/pkg/kptfile/kptfileutil" "github.com/kptdev/kpt/pkg/lib/errors" + "github.com/kptdev/kpt/pkg/lib/krmvalidate" "github.com/kptdev/kpt/pkg/lib/pkg" pathutil "github.com/kptdev/kpt/pkg/lib/util/path" "github.com/kptdev/kpt/pkg/lib/util/strings" + "github.com/kptdev/kpt/pkg/printer" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/klog/v2" "k8s.io/kubectl/pkg/cmd/util" @@ -59,11 +62,14 @@ func (e *InventoryInfoValidationError) Error() string { // for inventory information inside Kptfile or resourcegroup resources. // It returns the resources in unstructured format and the inventory information. // If no inventory information is found, that is not considered an error here. -func Load(f util.Factory, path string, stdIn io.Reader) ([]*unstructured.Unstructured, kptfilev1.Inventory, error) { +// When reading from disk, non-KRM files are pre-scanned and rejected +// (or excused by annotation) before conversion to Unstructured; stdin +// input is not pre-scanned. +func Load(ctx context.Context, f util.Factory, path string, stdIn io.Reader) ([]*unstructured.Unstructured, kptfilev1.Inventory, error) { if path == "-" { return loadFromStream(f, stdIn) } - return loadFromDisk(f, path) + return loadFromDisk(ctx, f, path) } // loadFromStream reads resources from the provided reader and returns the @@ -160,7 +166,7 @@ func readInvInfoFromStream(in io.Reader) (kptfilev1.Inventory, error) { // It returns the filtered resources and any inventory information found in // Kptfile resources. // Only the Kptfile in the root directory will be checked for inventory information. -func loadFromDisk(f util.Factory, path string) ([]*unstructured.Unstructured, kptfilev1.Inventory, error) { +func loadFromDisk(ctx context.Context, f util.Factory, path string) ([]*unstructured.Unstructured, kptfilev1.Inventory, error) { invInfo, err := readInvInfoFromDisk(path) if err != nil { return nil, kptfilev1.Inventory{}, err @@ -175,9 +181,20 @@ func loadFromDisk(f util.Factory, path string) ([]*unstructured.Unstructured, kp return nil, kptfilev1.Inventory{}, err } + absPkgPath, _, err := pathutil.ResolveAbsAndRelPaths(path) + if err != nil { + return nil, kptfilev1.Inventory{}, err + } + pr := printer.FromContextOrDie(ctx) + scanResult, err := krmvalidate.ScanFlatAndDecide(filesys.FileSystemOrOnDisk{}, absPkgPath, pr.Printf) + if err != nil { + return nil, kptfilev1.Inventory{}, err + } + objs, err := (&ResourceGroupPathManifestReader{ PkgPath: path, ReaderOptions: ro, + FileSkipFunc: scanResult.SkipFunc(absPkgPath), }).Read() if err != nil { return nil, kptfilev1.Inventory{}, err diff --git a/pkg/live/load_test.go b/pkg/live/load_test.go index c50aa3427d..a157317641 100644 --- a/pkg/live/load_test.go +++ b/pkg/live/load_test.go @@ -26,6 +26,7 @@ import ( rgfilev1alpha1 "github.com/kptdev/kpt/api/resourcegroup/v1alpha1" "github.com/kptdev/kpt/internal/testutil" "github.com/kptdev/kpt/internal/testutil/pkgbuilder" + "github.com/kptdev/kpt/pkg/printer/fake" "github.com/stretchr/testify/assert" "k8s.io/apimachinery/pkg/runtime/schema" cmdtesting "k8s.io/kubectl/pkg/cmd/testing" @@ -252,7 +253,7 @@ func TestLoad_LocalDisk(t *testing.T) { }() var buf bytes.Buffer - objs, inv, err := Load(tf, dir, &buf) + objs, inv, err := Load(fake.CtxWithDefaultPrinter(), tf, dir, &buf) if tc.expectedErrMsg != "" { if !assert.Error(t, err) { @@ -282,6 +283,113 @@ func TestLoad_LocalDisk(t *testing.T) { } } +// TestLoad_LocalDisk_NonKRM covers the non-KRM pre-scan in loadFromDisk. +func TestLoad_LocalDisk_NonKRM(t *testing.T) { + rgFile := pkgbuilder.NewRGFile().WithInventory(pkgbuilder.Inventory{ + Name: "foo", + Namespace: "bar", + ID: "foo-bar", + }) + + t.Run("non-KRM file with no annotation blocks Load with a consolidated, file-naming error", func(t *testing.T) { + tf := cmdtesting.NewTestFactory().WithNamespace("foo") + defer tf.Cleanup() + + rootPkg := pkgbuilder.NewRootPkg(). + WithKptfile(pkgbuilder.NewKptfile()). + WithRGFile(rgFile). + WithFile("deployment.yaml", deploymentA). + WithFile("values.yaml", "replicaCount: 3\n") + dir := rootPkg.ExpandPkg(t, nil) + defer func() { _ = os.RemoveAll(dir) }() + + var buf bytes.Buffer + _, _, err := Load(fake.CtxWithDefaultPrinter(), tf, dir, &buf) + + assert.Error(t, err) + assert.Contains(t, err.Error(), "values.yaml") + assert.Contains(t, err.Error(), "1 file(s)") + // Specifically NOT the old, file-less cli-utils error shape. + assert.NotContains(t, err.Error(), "/, Kind=") + }) + + t.Run("annotated package skips the non-KRM file in memory, leaves it untouched on disk, no .krmignore written", func(t *testing.T) { + tf := cmdtesting.NewTestFactory().WithNamespace("foo") + defer tf.Cleanup() + + rootPkg := pkgbuilder.NewRootPkg(). + WithFile("Kptfile", `apiVersion: kpt.dev/v1 +kind: Kptfile +metadata: + name: test-pkg + annotations: + kpt.dev/auto-ignore-non-krm: "true" +`). + WithRGFile(rgFile). + WithFile("deployment.yaml", deploymentA). + WithFile("values.yaml", "replicaCount: 3\n") + dir := rootPkg.ExpandPkg(t, nil) + defer func() { _ = os.RemoveAll(dir) }() + + var buf bytes.Buffer + objs, _, err := Load(fake.CtxWithDefaultPrinter(), tf, dir, &buf) + assert.NoError(t, err) + + objMetas := object.UnstructuredSetToObjMetadataSet(objs) + var names []string + for _, m := range objMetas { + names = append(names, m.Name) + } + assert.ElementsMatch(t, []string{"test-deployment"}, names, "values.yaml must not have reached the loaded object set") + + content, readErr := os.ReadFile(filepath.Join(dir, "values.yaml")) + assert.NoError(t, readErr) + assert.Equal(t, "replicaCount: 3\n", string(content), "non-KRM file must be left completely untouched on disk") + + assert.NoFileExists(t, filepath.Join(dir, ".krmignore"), ".krmignore must never be written by the annotation path") + }) + + t.Run("bare directory with no Kptfile at all is still pre-scanned, same as a Kptfile'd package", func(t *testing.T) { + tf := cmdtesting.NewTestFactory().WithNamespace("foo") + defer tf.Cleanup() + + rootPkg := pkgbuilder.NewRootPkg(). + WithRGFile(rgFile). + WithFile("deployment.yaml", deploymentA). + WithFile("values.yaml", "replicaCount: 3\n") + dir := rootPkg.ExpandPkg(t, nil) + defer func() { _ = os.RemoveAll(dir) }() + + var buf bytes.Buffer + _, _, err := Load(fake.CtxWithDefaultPrinter(), tf, dir, &buf) + + assert.Error(t, err) + assert.Contains(t, err.Error(), "values.yaml") + }) + + t.Run(".krmignore still works independently of the annotation, no non-KRM-file error at all", func(t *testing.T) { + tf := cmdtesting.NewTestFactory().WithNamespace("foo") + defer tf.Cleanup() + + rootPkg := pkgbuilder.NewRootPkg(). + WithKptfile(pkgbuilder.NewKptfile()). + WithRGFile(rgFile). + WithFile("deployment.yaml", deploymentA). + WithFile("values.yaml", "replicaCount: 3\n"). + WithFile(".krmignore", "values.yaml\n") + dir := rootPkg.ExpandPkg(t, nil) + defer func() { _ = os.RemoveAll(dir) }() + + var buf bytes.Buffer + objs, _, err := Load(fake.CtxWithDefaultPrinter(), tf, dir, &buf) + assert.NoError(t, err) + + objMetas := object.UnstructuredSetToObjMetadataSet(objs) + assert.Len(t, objMetas, 1) + assert.Equal(t, "test-deployment", objMetas[0].Name) + }) +} + func TestLoad_StdIn(t *testing.T) { testCases := map[string]struct { pkg *pkgbuilder.RootPkg @@ -533,7 +641,7 @@ func TestLoad_StdIn(t *testing.T) { os.Remove(filepath.Join(dir, kptfilev1.KptFileName)) os.Remove(filepath.Join(dir, tc.rgFile)) - objs, inv, err := Load(tf, "-", &buf) + objs, inv, err := Load(fake.CtxWithDefaultPrinter(), tf, "-", &buf) if tc.expectedErrMsg != "" { if !assert.Error(t, err) { diff --git a/pkg/live/rgpath.go b/pkg/live/rgpath.go index 60f8dfd8b4..55944c581a 100644 --- a/pkg/live/rgpath.go +++ b/pkg/live/rgpath.go @@ -35,6 +35,10 @@ type ResourceGroupPathManifestReader struct { PkgPath string manifestreader.ReaderOptions + + // FileSkipFunc, if set, is passed through to the underlying reader + // so pre-excused non-KRM files are skipped at read time. + FileSkipFunc kio.LocalPackageSkipFileFunc } // Read reads the manifests and returns them as Info objects. @@ -51,6 +55,7 @@ func (r *ResourceGroupPathManifestReader) Read() ([]*unstructured.Unstructured, nodes, err := (&kio.LocalPackageReader{ PackagePath: absPkgPath, WrapBareSeqNode: true, + FileSkipFunc: r.FileSkipFunc, }).Read() if err != nil { return objs, err