diff --git a/.dagger/build_publish.go b/.dagger/build_publish.go index 04ec20d..5f155c1 100644 --- a/.dagger/build_publish.go +++ b/.dagger/build_publish.go @@ -18,8 +18,8 @@ func (m *Cuestomize) Build( ctx context.Context, // +defaultPath=./ buildContext *dagger.Directory, - // +defaultPath="./.git" - git *dagger.GitRepository, + // +optional + ref string, // +default="" platform string, // +default="" @@ -29,9 +29,23 @@ func (m *Cuestomize) Build( ) *dagger.Container { ldflags = fmt.Sprintf("-X 'main.Version=%s' %s", version, ldflags) + git := FromDirectory(buildContext) + + ctx, sp := Tracer().Start(ctx, "building Cuestomize image") + defer sp.End() + + if ref != "" { + git.Stash(ctx) + + git.Checkout(ctx, ref) + + buildContext = git.Directory() + } + commit, err := git.Head().Commit(ctx) if err != nil { - panic("failed to get git commit: " + err.Error()) + sp.RecordError(fmt.Errorf("failed to get git commit: %w", err)) + return nil } container := buildContext.DockerBuild(dagger.DirectoryDockerBuildOpts{ @@ -63,8 +77,8 @@ func (m *Cuestomize) BuildAndPublish( password *dagger.Secret, // +defaultPath=./ buildContext *dagger.Directory, - // +defaultPath="./.git" - git *dagger.GitRepository, + // +optional + ref string, // +default="ghcr.io" registry string, repository string, @@ -87,7 +101,10 @@ func (m *Cuestomize) BuildAndPublish( platformVariants := make([]*dagger.Container, 0, len(platforms)) for _, platform := range platforms { - container := m.Build(ctx, buildContext, git, string(platform), ldflags, version) + container := m.Build(ctx, buildContext, ref, platform, ldflags, version) + if container == nil { + return fmt.Errorf("failed to build container for platform %s", platform) + } platformVariants = append(platformVariants, container) } diff --git a/.dagger/constants.go b/.dagger/constants.go index 0b54edc..8f0c314 100644 --- a/.dagger/constants.go +++ b/.dagger/constants.go @@ -17,6 +17,8 @@ const ( CuelangVersion = "v0.16.0" // GolangciLintImage is the GolangCI-Lint image used by default GolangciLintImage = "golangci/golangci-lint:v2.11.4-alpine" + // GitImage is the image used for Git operations in Dagger + GitImage = "alpine/git:2.52.0" ) const ( diff --git a/.dagger/git.go b/.dagger/git.go new file mode 100644 index 0000000..cc1f5c3 --- /dev/null +++ b/.dagger/git.go @@ -0,0 +1,53 @@ +package main + +import ( + "context" + "dagger/cuestomize/internal/dagger" +) + +type GitRepository struct { + dir *dagger.Directory +} + +func FromDirectory(dir *dagger.Directory) *GitRepository { + return &GitRepository{ + dir: dir, + } +} + +func (r *GitRepository) Directory() *dagger.Directory { + return r.dir +} + +func (r *GitRepository) AsGit() *dagger.GitRepository { + return r.Directory().AsGit() +} + +func (r *GitRepository) Head() *dagger.GitRef { + return r.AsGit().Head() +} + +func (r *GitRepository) Stash(ctx context.Context) { + r.Run(ctx, "stash") +} + +func (r *GitRepository) Checkout(ctx context.Context, ref string) { + r.Run(ctx, "checkout", ref) +} + +func (r *GitRepository) output(c *dagger.Container) { + r.dir = c.Directory("/git") +} + +func (r *GitRepository) Run(ctx context.Context, args ...string) *dagger.Container { + cmd := []string{"git"} + cmd = append(cmd, args...) + c := dag.Container().From(GitImage). + WithWorkdir("/git"). + WithDirectory("/git", r.dir). + WithExec(cmd) + + r.output(c) + + return c +} diff --git a/.dagger/testing_pipelines.go b/.dagger/testing_pipelines.go index 9d887f4..28f4c3c 100644 --- a/.dagger/testing_pipelines.go +++ b/.dagger/testing_pipelines.go @@ -40,7 +40,7 @@ func (m *Cuestomize) E2E_Test( git *dagger.GitRepository, ) error { // build cuestomize - cuestomize := m.Build(ctx, buildContext, git, "", "", "nightly") + cuestomize := m.Build(ctx, buildContext, "", "", "", "nightly") cuestomizeBinary := cuestomize.File("/usr/local/bin/cuestomize") diff --git a/pkg/cuestomize/checkers.go b/pkg/cuestomize/checkers.go index c497fd2..43c80a4 100644 --- a/pkg/cuestomize/checkers.go +++ b/pkg/cuestomize/checkers.go @@ -13,6 +13,11 @@ const ( ValidatorAnnotationKey = "config.cuestomize.io/validator" // ValidatorAnnotationValue is the value of the annotation that marks a CUE function as a validator. ValidatorAnnotationValue = "true" + + // EditStreamAnnotationKey is the annotation key that allows Cuestomize to edit resources in place. + EditStreamAnnotationKey = "config.cuestomize.io/edit-stream" + // EditStreamAnnotationValue is the value of the annotation that allows Cuestomize to edit resources in place. + EditStreamAnnotationValue = "true" ) // CheckInstances checks if any of the instances have an error and returns an error if so. @@ -32,3 +37,9 @@ func ShouldActAsValidator(config *api.KRMInput) bool { return config.Annotations != nil && config.Annotations[ValidatorAnnotationKey] == ValidatorAnnotationValue } + +// AllowEditResourcesInStream checks if the KRMInput configuration has the edit stream annotation set. +func AllowEditResourcesInStream(config *api.KRMInput) bool { + return config.Annotations != nil && + config.Annotations[EditStreamAnnotationKey] == EditStreamAnnotationValue +} diff --git a/pkg/cuestomize/cuestomize.go b/pkg/cuestomize/cuestomize.go index e81c4d5..8ca2178 100644 --- a/pkg/cuestomize/cuestomize.go +++ b/pkg/cuestomize/cuestomize.go @@ -79,5 +79,7 @@ func Cuestomize(ctx context.Context, items []*kyaml.RNode, config *api.KRMInput, log.V(4).Info("cuestomize is acting in validator mode.") return items, nil // if the function is a validator, return the original items without processing } - return ProcessOutputs(ctx, unified, items) + return ProcessOutputs(ctx, unified, items, OutputOptions{ + AllowEdit: AllowEditResourcesInStream(config), + }) } diff --git a/pkg/cuestomize/outputs.go b/pkg/cuestomize/outputs.go index 6b50761..9c2611b 100644 --- a/pkg/cuestomize/outputs.go +++ b/pkg/cuestomize/outputs.go @@ -7,15 +7,20 @@ import ( "cuelang.org/go/cue" "cuelang.org/go/encoding/yaml" "github.com/Workday/cuestomize/pkg/cuerrors" - "github.com/go-logr/logr" + "sigs.k8s.io/kustomize/kyaml/resid" kyaml "sigs.k8s.io/kustomize/kyaml/yaml" ) -// ProcessOutputs processes the outputs from the CUE model and appends them to the output slice. -func ProcessOutputs(ctx context.Context, unified cue.Value, items []*kyaml.RNode) ([]*kyaml.RNode, error) { - log := logr.FromContextOrDiscard(ctx) +type OutputOptions struct { + // AllowEdit allows output resources to replace existing items in the stream if a matching ResId is found. + AllowEdit bool +} +// ProcessOutputs processes the outputs from the CUE model and appends them to the output slice. +// When allowEdit is true, output resources that match an existing item by ResId (GVK + namespace + name) +// will replace the existing item in place; otherwise they are appended. +func ProcessOutputs(ctx context.Context, unified cue.Value, items []*kyaml.RNode, opts OutputOptions) ([]*kyaml.RNode, error) { detailer := cuerrors.FromContextOrEmpty(ctx) outputsValue := unified.LookupPath(cue.ParsePath(OutputsPath)) @@ -29,6 +34,32 @@ func ProcessOutputs(ctx context.Context, unified cue.Value, items []*kyaml.RNode return nil, fmt.Errorf("failed to get iterator over '%s' in unified CUE instance: %v", OutputsPath, err) } + if !opts.AllowEdit { + return appendOutputs(ctx, outputsIter, items) + } + return editOutputs(ctx, outputsIter, items) +} + +// appendOutputs appends all CUE outputs to the items slice (generate-only mode). +func appendOutputs(ctx context.Context, outputsIter *cue.Iterator, items []*kyaml.RNode) ([]*kyaml.RNode, error) { + for outputsIter.Next() { + item := outputsIter.Value() + + rNode, err := cueValueToRNode(&item) + if err != nil { + return nil, fmt.Errorf("failed to convert CUE value to kyaml.RNode: %w", err) + } + + items = append(items, rNode) + } + return items, nil +} + +// editOutputs replaces existing resources in the items stream if a matching +// resource (by ResId) is found, or appends new ones. +func editOutputs(ctx context.Context, outputsIter *cue.Iterator, items []*kyaml.RNode) ([]*kyaml.RNode, error) { + residMap := make(map[resid.ResId]*kyaml.RNode) + for outputsIter.Next() { item := outputsIter.Value() @@ -37,10 +68,26 @@ func ProcessOutputs(ctx context.Context, unified cue.Value, items []*kyaml.RNode return nil, fmt.Errorf("failed to convert CUE value to kyaml.RNode: %w", err) } - log.V(4).Info("adding item to output resources", - "kind", rNode.GetKind(), "apiVersion", rNode.GetApiVersion(), "namespace", rNode.GetNamespace(), "name", rNode.GetName()) + rid := residFromRNode(rNode) + + if _, found := residMap[rid]; found { + return nil, fmt.Errorf("duplicate output resource with ResId '%s' found in CUE model", rid) + } + } + + for i := range items { + itemRid := residFromRNode(items[i]) + if cueOutputRNode, found := residMap[itemRid]; found { + items[i] = cueOutputRNode + delete(residMap, itemRid) + } + } + + // append any remaining CUE output resource + for _, rNode := range residMap { items = append(items, rNode) } + return items, nil } @@ -74,3 +121,7 @@ func cueValueToRNode(value *cue.Value) (*kyaml.RNode, error) { return rNode, nil } + +func residFromRNode(rNode *kyaml.RNode) resid.ResId { + return resid.NewResIdWithNamespace(resid.GvkFromNode(rNode), rNode.GetName(), rNode.GetNamespace()) +} diff --git a/pkg/cuestomize/outputs_test.go b/pkg/cuestomize/outputs_test.go new file mode 100644 index 0000000..9bbca4d --- /dev/null +++ b/pkg/cuestomize/outputs_test.go @@ -0,0 +1,438 @@ +package cuestomize + +import ( + "context" + "testing" + + "cuelang.org/go/cue" + "cuelang.org/go/cue/cuecontext" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "sigs.k8s.io/kustomize/kyaml/resid" + kyaml "sigs.k8s.io/kustomize/kyaml/yaml" +) + +// helper to create an RNode from a map. +func createTestRNode(t *testing.T, apiVersion, kind, namespace, name string) *kyaml.RNode { + t.Helper() + yamlObj := map[string]interface{}{ + "apiVersion": apiVersion, + "kind": kind, + "metadata": map[string]interface{}{ + "name": name, + "namespace": namespace, + }, + } + node, err := kyaml.FromMap(yamlObj) + require.NoError(t, err) + return node +} + +// helper to compile a CUE string and return the value. +func compileCUE(t *testing.T, src string) cue.Value { + t.Helper() + ctx := cuecontext.New() + v := ctx.CompileString(src) + require.NoError(t, v.Err()) + return v +} + +func TestProcessOutputs_AppendMode(t *testing.T) { + tests := []struct { + name string + cueSrc string + existingItems []*kyaml.RNode + expectedCount int + expectedError bool + errorSubstring string + }{ + { + name: "append single output to empty items", + cueSrc: `{ + outputs: [{ + apiVersion: "v1" + kind: "ConfigMap" + metadata: { + name: "my-cm" + namespace: "default" + } + data: { + key: "value" + } + }] + }`, + existingItems: nil, + expectedCount: 1, + }, + { + name: "append multiple outputs to empty items", + cueSrc: `{ + outputs: [{ + apiVersion: "v1" + kind: "ConfigMap" + metadata: { + name: "cm1" + namespace: "default" + } + }, { + apiVersion: "v1" + kind: "ConfigMap" + metadata: { + name: "cm2" + namespace: "default" + } + }] + }`, + existingItems: nil, + expectedCount: 2, + }, + { + name: "append outputs to existing items", + cueSrc: `{ + outputs: [{ + apiVersion: "v1" + kind: "ConfigMap" + metadata: { + name: "new-cm" + namespace: "default" + } + }] + }`, + existingItems: []*kyaml.RNode{ + createTestRNode(t, "v1", "ConfigMap", "default", "existing-cm"), + }, + expectedCount: 2, + }, + { + name: "empty outputs list", + cueSrc: `{ + outputs: [] + }`, + existingItems: []*kyaml.RNode{ + createTestRNode(t, "v1", "ConfigMap", "default", "existing-cm"), + }, + expectedCount: 1, + }, + { + name: "outputs path missing", + cueSrc: `{ + something: "else" + }`, + existingItems: nil, + expectedError: true, + errorSubstring: "not found in unified CUE instance", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + unified := compileCUE(t, tt.cueSrc) + + result, err := ProcessOutputs(t.Context(), unified, tt.existingItems, OutputOptions{AllowEdit: false}) + + if tt.expectedError { + require.Error(t, err) + if tt.errorSubstring != "" { + assert.Contains(t, err.Error(), tt.errorSubstring) + } + } else { + require.NoError(t, err) + assert.Len(t, result, tt.expectedCount) + } + }) + } +} + +func TestProcessOutputs_AppendPreservesContent(t *testing.T) { + cueSrc := `{ + outputs: [{ + apiVersion: "v1" + kind: "ConfigMap" + metadata: { + name: "my-cm" + namespace: "test-ns" + } + data: { + foo: "bar" + } + }] + }` + unified := compileCUE(t, cueSrc) + + result, err := ProcessOutputs(t.Context(), unified, nil, OutputOptions{AllowEdit: false}) + require.NoError(t, err) + require.Len(t, result, 1) + + rNode := result[0] + assert.Equal(t, "my-cm", rNode.GetName()) + assert.Equal(t, "test-ns", rNode.GetNamespace()) + assert.Equal(t, "ConfigMap", rNode.GetKind()) + assert.Equal(t, "v1", rNode.GetApiVersion()) +} + +func TestProcessOutputs_EditMode_ReplacesExisting(t *testing.T) { + cueSrc := `{ + outputs: [{ + apiVersion: "v1" + kind: "ConfigMap" + metadata: { + name: "my-cm" + namespace: "default" + } + data: { + key: "new-value" + } + }] + }` + unified := compileCUE(t, cueSrc) + + existingItems := []*kyaml.RNode{ + createTestRNode(t, "v1", "ConfigMap", "default", "my-cm"), + createTestRNode(t, "apps/v1", "Deployment", "default", "my-deploy"), + } + + result, err := ProcessOutputs(t.Context(), unified, existingItems, OutputOptions{AllowEdit: true}) + require.NoError(t, err) + + // Should still have 2 items – the existing ConfigMap replaced in-place, deployment unchanged + require.Len(t, result, 2) + + // First item should be the replaced ConfigMap with new data + assert.Equal(t, "my-cm", result[0].GetName()) + + // Second item should be the untouched deployment + assert.Equal(t, "my-deploy", result[1].GetName()) +} + +func TestProcessOutputs_EditMode_AppendsNew(t *testing.T) { + cueSrc := `{ + outputs: [{ + apiVersion: "v1" + kind: "ConfigMap" + metadata: { + name: "brand-new" + namespace: "default" + } + }] + }` + unified := compileCUE(t, cueSrc) + + existingItems := []*kyaml.RNode{ + createTestRNode(t, "apps/v1", "Deployment", "default", "my-deploy"), + } + + result, err := ProcessOutputs(t.Context(), unified, existingItems, OutputOptions{AllowEdit: true}) + require.NoError(t, err) + + // Should have 2 items – existing deployment + appended ConfigMap + require.Len(t, result, 2) + assert.Equal(t, "my-deploy", result[0].GetName()) + assert.Equal(t, "brand-new", result[1].GetName()) +} + +func TestProcessOutputs_EditMode_DuplicateOutputError(t *testing.T) { + cueSrc := `{ + outputs: [{ + apiVersion: "v1" + kind: "ConfigMap" + metadata: { + name: "my-cm" + namespace: "default" + } + }, { + apiVersion: "v1" + kind: "ConfigMap" + metadata: { + name: "my-cm" + namespace: "default" + } + }] + }` + unified := compileCUE(t, cueSrc) + + result, err := ProcessOutputs(t.Context(), unified, nil, OutputOptions{AllowEdit: true}) + require.Error(t, err) + assert.Contains(t, err.Error(), "duplicate output resource") + assert.Nil(t, result) +} + +func TestProcessOutputs_StructOutputs(t *testing.T) { + cueSrc := `{ + outputs: { + cm1: { + apiVersion: "v1" + kind: "ConfigMap" + metadata: { + name: "cm1" + namespace: "default" + } + } + cm2: { + apiVersion: "v1" + kind: "ConfigMap" + metadata: { + name: "cm2" + namespace: "default" + } + } + } + }` + unified := compileCUE(t, cueSrc) + + result, err := ProcessOutputs(t.Context(), unified, nil, OutputOptions{AllowEdit: false}) + require.NoError(t, err) + assert.Len(t, result, 2) +} + +func TestGetIter(t *testing.T) { + ctx := cuecontext.New() + + tests := []struct { + name string + cueSrc string + expectedError bool + }{ + { + name: "list kind", + cueSrc: `[1, 2, 3]`, + expectedError: false, + }, + { + name: "struct kind", + cueSrc: `{a: 1, b: 2}`, + expectedError: false, + }, + { + name: "string kind", + cueSrc: `"hello"`, + expectedError: true, + }, + { + name: "int kind", + cueSrc: `42`, + expectedError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + v := ctx.CompileString(tt.cueSrc) + require.NoError(t, v.Err()) + + iter, err := getIter(v) + if tt.expectedError { + assert.Error(t, err) + assert.Nil(t, iter) + } else { + assert.NoError(t, err) + assert.NotNil(t, iter) + } + }) + } +} + +func TestCueValueToRNode(t *testing.T) { + tests := []struct { + name string + cueSrc string + expectedName string + expectedError bool + }{ + { + name: "valid k8s resource", + cueSrc: `{ + apiVersion: "v1" + kind: "ConfigMap" + metadata: { + name: "my-cm" + namespace: "default" + } + data: { + key: "value" + } + }`, + expectedName: "my-cm", + }, + { + name: "empty struct", + cueSrc: `{}`, + }, + } + + ctx := cuecontext.New() + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + v := ctx.CompileString(tt.cueSrc) + require.NoError(t, v.Err()) + + rNode, err := cueValueToRNode(&v) + if tt.expectedError { + assert.Error(t, err) + } else { + require.NoError(t, err) + assert.NotNil(t, rNode) + if tt.expectedName != "" { + assert.Equal(t, tt.expectedName, rNode.GetName()) + } + } + }) + } +} + +func TestResidFromRNode(t *testing.T) { + rNode := createTestRNode(t, "apps/v1", "Deployment", "my-ns", "my-deploy") + rid := residFromRNode(rNode) + + expected := resid.NewResIdWithNamespace( + resid.Gvk{Group: "apps", Version: "v1", Kind: "Deployment"}, + "my-deploy", + "my-ns", + ) + assert.Equal(t, expected, rid) +} + +func TestResidFromRNode_CoreResource(t *testing.T) { + rNode := createTestRNode(t, "v1", "ConfigMap", "default", "my-cm") + rid := residFromRNode(rNode) + + assert.Equal(t, "ConfigMap", rid.Kind) + assert.Equal(t, "my-cm", rid.Name) + assert.Equal(t, "default", rid.Namespace) +} + +func TestProcessOutputs_EditMode_EmptyOutputs(t *testing.T) { + cueSrc := `{ + outputs: [] + }` + unified := compileCUE(t, cueSrc) + + existingItems := []*kyaml.RNode{ + createTestRNode(t, "v1", "ConfigMap", "default", "existing"), + } + + result, err := ProcessOutputs(t.Context(), unified, existingItems, OutputOptions{AllowEdit: true}) + require.NoError(t, err) + + // Existing items should remain unchanged + require.Len(t, result, 1) + assert.Equal(t, "existing", result[0].GetName()) +} + +func TestProcessOutputs_NilContext(t *testing.T) { + cueSrc := `{ + outputs: [{ + apiVersion: "v1" + kind: "ConfigMap" + metadata: { + name: "my-cm" + namespace: "default" + } + }] + }` + unified := compileCUE(t, cueSrc) + + //nolint:staticcheck // intentionally testing with nil context + result, err := ProcessOutputs(context.TODO(), unified, nil, OutputOptions{AllowEdit: false}) + require.NoError(t, err) + assert.Len(t, result, 1) +} diff --git a/renovate.json5 b/renovate.json5 index 56bc6db..2ad2996 100644 --- a/renovate.json5 +++ b/renovate.json5 @@ -11,7 +11,8 @@ "RegistryImage\\s*=\\s*\"(?[\\w/-]+)[:@](?.*?)\"", "DistrolessStaticImage\\s*=\\s*\"(?[\\w/-]+)[:@](?.*?)\"", "KustomizeImage\\s*=\\s*\"(?[\\w/-]+)[:@](?.*?)\"", - "GolangciLintImage\\s*=\\s*\"(?[\\w/-]+)[:@](?.*?)\"" + "GolangciLintImage\\s*=\\s*\"(?[\\w/-]+)[:@](?.*?)\"", + "GitImage\\s*=\\s*\"(?[\\w/-]+)[:@](?.*?)\"" ], "datasourceTemplate": "docker", "versioningTemplate": "docker" diff --git a/semver b/semver index 5e265ff..7e8d209 100644 --- a/semver +++ b/semver @@ -1 +1 @@ -v0.5.0-alpha.1 +v0.5.0-alpha.2 \ No newline at end of file diff --git a/testdata/function/cue-modules/generate-resources-model/cue.mod/module.cue b/testdata/function/cue-modules/generate-resources-model/cue.mod/module.cue new file mode 100644 index 0000000..979b90a --- /dev/null +++ b/testdata/function/cue-modules/generate-resources-model/cue.mod/module.cue @@ -0,0 +1,4 @@ +module: "cue.k8s.example" +language: { + version: "v0.16.0" +} diff --git a/testdata/function/cue-modules/generate-resources-model/main.cue b/testdata/function/cue-modules/generate-resources-model/main.cue new file mode 100644 index 0000000..fbe5de0 --- /dev/null +++ b/testdata/function/cue-modules/generate-resources-model/main.cue @@ -0,0 +1,75 @@ +package main + +import ( + api "k8s.io/api/core/v1" +) + +input: {} + +includes: { + "v1": "Namespace": "": _: api.#Namespace +} + +// helper to extract a single namespace resource from includes, with error handling +_#ExtractNamespace: { + i: [string]: [string]: [string]: [string]: [string]: _ + + _resources: list.FlattenN([ + for k, rs in i["v1"]["Namespace"][""] {[ + for kk, r in rs {r}, + ]}, + ], 1) + + if len(_resources) != 1 { + error("Expected 1 resource of kind \(i.kind) in namespace \(i.namespace), but found \(len(_resources))") + } + out: _resources[0] +} + +inputNs: _#ExtractNamespace & {i: includes} + +editedNs: inputNs & { + metadata: { + annotations: { + "edited": "true" + } + } +} + +outputs: { + ns: editedNs + ksa: { + apiVersion: "v1" + kind: "ServiceAccount" + metadata: { + name: "test-serviceaccount" + namespace: editedNs.metadata.name + } + } + deploy: { + apiVersion: "apps/v1" + kind: "Deployment" + metadata: { + name: "test-deployment" + namespace: editedNs.metadata.name + } + spec: { + serviceAccountName: ksa.metadata.name + replicas: 1 + selector: { + matchLabels: {app: "test-app"} + } + template: { + metadata: { + labels: {app: "test-app"} + } + spec: { + containers: [{ + name: "test-container" + image: "nginx:latest" + }] + } + } + } + } +}