From 88ce5a8185aa321130a63536e5a9541c33055277 Mon Sep 17 00:00:00 2001 From: arewm Date: Fri, 13 Dec 2024 15:55:10 -0500 Subject: [PATCH] Enable the fetching of only the OCI descriptor Some OCI artifacts need to have a specific media type (i.e. they should only be an OCI image index or an OCI image manifest). In order to determine that with a single remote call, we can just get the descriptor and then check the mediaType of that object. Signed-off-by: arewm --- .../modules/ROOT/pages/ec_oci_descriptor.adoc | 36 ++ docs/modules/ROOT/pages/rego_builtins.adoc | 2 + docs/modules/ROOT/partials/rego_nav.adoc | 1 + internal/rego/oci/__snapshots__/oci_test.snap | 353 ++++++++++++++++++ internal/rego/oci/oci.go | 84 +++++ internal/rego/oci/oci_test.go | 117 ++++++ 6 files changed, 593 insertions(+) create mode 100644 docs/modules/ROOT/pages/ec_oci_descriptor.adoc diff --git a/docs/modules/ROOT/pages/ec_oci_descriptor.adoc b/docs/modules/ROOT/pages/ec_oci_descriptor.adoc new file mode 100644 index 000000000..4fd2e5c2c --- /dev/null +++ b/docs/modules/ROOT/pages/ec_oci_descriptor.adoc @@ -0,0 +1,36 @@ += ec.oci.descriptor + +Fetch a raw Image from an OCI registry. + +== Usage + + object = ec.oci.descriptor(ref: string) + +== Parameters + +* `ref` (`string`): OCI descriptor reference + +== Return + +`object` (`object`): the OCI descriptor object + +The object contains the following attributes: + +* `annotations` (`object`) +** (`string`): (`string`) +* `artifactType` (`string`) +* `data` (`string`) +* `digest` (`string`) +* `mediaType` (`string`) +* `platform` (`object`) +** `architecture` (`string`) +** `features`(`array`) +*** (`string`) +** `os` (`string`) +** `os.features`(`array`) +*** (`string`) +** `os.version` (`string`) +** `variant` (`string`) +* `size` (`number`) +* `urls`(`array`) +** (`string`) diff --git a/docs/modules/ROOT/pages/rego_builtins.adoc b/docs/modules/ROOT/pages/rego_builtins.adoc index 4067b78fe..427803b96 100644 --- a/docs/modules/ROOT/pages/rego_builtins.adoc +++ b/docs/modules/ROOT/pages/rego_builtins.adoc @@ -10,6 +10,8 @@ information. |=== |xref:ec_oci_blob.adoc[ec.oci.blob] |Fetch a blob from an OCI registry. +|xref:ec_oci_descriptor.adoc[ec.oci.descriptor] +|Fetch a raw Image from an OCI registry. |xref:ec_oci_image_files.adoc[ec.oci.image_files] |Fetch structured files (YAML or JSON) from within an image. |xref:ec_oci_image_manifest.adoc[ec.oci.image_manifest] diff --git a/docs/modules/ROOT/partials/rego_nav.adoc b/docs/modules/ROOT/partials/rego_nav.adoc index 70f0cf88e..a60e223a5 100644 --- a/docs/modules/ROOT/partials/rego_nav.adoc +++ b/docs/modules/ROOT/partials/rego_nav.adoc @@ -1,5 +1,6 @@ * xref:rego_builtins.adoc[Rego Reference] ** xref:ec_oci_blob.adoc[ec.oci.blob] +** xref:ec_oci_descriptor.adoc[ec.oci.descriptor] ** xref:ec_oci_image_files.adoc[ec.oci.image_files] ** xref:ec_oci_image_manifest.adoc[ec.oci.image_manifest] ** xref:ec_purl_is_valid.adoc[ec.purl.is_valid] diff --git a/internal/rego/oci/__snapshots__/oci_test.snap b/internal/rego/oci/__snapshots__/oci_test.snap index a94b1e97c..91c7baa0e 100755 --- a/internal/rego/oci/__snapshots__/oci_test.snap +++ b/internal/rego/oci/__snapshots__/oci_test.snap @@ -867,3 +867,356 @@ ] } --- + +[TestOCIDescriptorManifest/complete_image_manifest - 1] +{ + "type": "object", + "value": [ + [ + { + "type": "string", + "value": "annotations" + }, + { + "type": "object", + "value": [ + [ + { + "type": "string", + "value": "config.annotation.1" + }, + { + "type": "string", + "value": "config.annotation.value.1" + } + ], + [ + { + "type": "string", + "value": "config.annotation.2" + }, + { + "type": "string", + "value": "config.annotation.value.2" + } + ] + ] + } + ], + [ + { + "type": "string", + "value": "artifactType" + }, + { + "type": "string", + "value": "artifact-type" + } + ], + [ + { + "type": "string", + "value": "data" + }, + { + "type": "string", + "value": "{\"data\": \"config\"}" + } + ], + [ + { + "type": "string", + "value": "digest" + }, + { + "type": "string", + "value": "sha256:4e388ab32b10dc8dbc7e28144f552830adc74787c1e2c0824032078a79f227fb" + } + ], + [ + { + "type": "string", + "value": "mediaType" + }, + { + "type": "string", + "value": "application/vnd.oci.image.manifest.v1+json" + } + ], + [ + { + "type": "string", + "value": "platform" + }, + { + "type": "object", + "value": [ + [ + { + "type": "string", + "value": "architecture" + }, + { + "type": "string", + "value": "arch" + } + ], + [ + { + "type": "string", + "value": "features" + }, + { + "type": "array", + "value": [ + { + "type": "string", + "value": "feature-1" + }, + { + "type": "string", + "value": "feature-2" + } + ] + } + ], + [ + { + "type": "string", + "value": "os" + }, + { + "type": "string", + "value": "os" + } + ], + [ + { + "type": "string", + "value": "os.features" + }, + { + "type": "array", + "value": [ + { + "type": "string", + "value": "os-feature-1" + }, + { + "type": "string", + "value": "os-feature-2" + } + ] + } + ], + [ + { + "type": "string", + "value": "os.version" + }, + { + "type": "string", + "value": "os-version" + } + ], + [ + { + "type": "string", + "value": "variant" + }, + { + "type": "string", + "value": "variant" + } + ] + ] + } + ], + [ + { + "type": "string", + "value": "size" + }, + { + "type": "number", + "value": 123 + } + ], + [ + { + "type": "string", + "value": "urls" + }, + { + "type": "array", + "value": [ + { + "type": "string", + "value": "https://config-1.local/spam" + }, + { + "type": "string", + "value": "https://config-2.local/spam" + } + ] + } + ] + ] +} +--- + +[TestOCIDescriptorManifest/minimal_image_manifest - 1] +{ + "type": "object", + "value": [ + [ + { + "type": "string", + "value": "annotations" + }, + { + "type": "object", + "value": [] + } + ], + [ + { + "type": "string", + "value": "artifactType" + }, + { + "type": "string", + "value": "" + } + ], + [ + { + "type": "string", + "value": "data" + }, + { + "type": "string", + "value": "" + } + ], + [ + { + "type": "string", + "value": "digest" + }, + { + "type": "string", + "value": "sha256:4e388ab32b10dc8dbc7e28144f552830adc74787c1e2c0824032078a79f227fb" + } + ], + [ + { + "type": "string", + "value": "mediaType" + }, + { + "type": "string", + "value": "application/vnd.oci.image.manifest.v1+json" + } + ], + [ + { + "type": "string", + "value": "size" + }, + { + "type": "number", + "value": 123 + } + ], + [ + { + "type": "string", + "value": "urls" + }, + { + "type": "array", + "value": [] + } + ] + ] +} +--- + +[TestOCIDescriptorManifest/minimal_image_index - 1] +{ + "type": "object", + "value": [ + [ + { + "type": "string", + "value": "annotations" + }, + { + "type": "object", + "value": [] + } + ], + [ + { + "type": "string", + "value": "artifactType" + }, + { + "type": "string", + "value": "" + } + ], + [ + { + "type": "string", + "value": "data" + }, + { + "type": "string", + "value": "" + } + ], + [ + { + "type": "string", + "value": "digest" + }, + { + "type": "string", + "value": "sha256:4e388ab32b10dc8dbc7e28144f552830adc74787c1e2c0824032078a79f227fb" + } + ], + [ + { + "type": "string", + "value": "mediaType" + }, + { + "type": "string", + "value": "application/vnd.oci.image.index.v1+json" + } + ], + [ + { + "type": "string", + "value": "size" + }, + { + "type": "number", + "value": 123 + } + ], + [ + { + "type": "string", + "value": "urls" + }, + { + "type": "array", + "value": [] + } + ] + ] +} +--- diff --git a/internal/rego/oci/oci.go b/internal/rego/oci/oci.go index f66142799..6dcb4f060 100644 --- a/internal/rego/oci/oci.go +++ b/internal/rego/oci/oci.go @@ -41,6 +41,7 @@ import ( const ( ociBlobName = "ec.oci.blob" + ociDescriptorName = "ec.oci.descriptor" ociImageManifestName = "ec.oci.image_manifest" ociImageFilesName = "ec.oci.image_files" ) @@ -73,6 +74,66 @@ func registerOCIBlob() { }) } +func registerOCIDescriptor() { + platform := types.NewObject( + []*types.StaticProperty{ + {Key: "architecture", Value: types.S}, + {Key: "os", Value: types.S}, + {Key: "os.version", Value: types.S}, + {Key: "os.features", Value: types.NewArray([]types.Type{types.S}, nil)}, + {Key: "variant", Value: types.S}, + {Key: "features", Value: types.NewArray([]types.Type{types.S}, nil)}, + }, + nil, + ) + + // annotations represents the map[string]string rego type + annotations := types.NewObject(nil, types.NewDynamicProperty(types.S, types.S)) + manifest := types.NewObject( + []*types.StaticProperty{ + // Specifying the properties like this ensure the compiler catches typos when + // evaluating rego functions. + {Key: "mediaType", Value: types.S}, + {Key: "size", Value: types.N}, + {Key: "digest", Value: types.S}, + {Key: "data", Value: types.S}, + {Key: "urls", Value: types.NewArray( + []types.Type{types.S}, nil, + )}, + {Key: "annotations", Value: annotations}, + {Key: "platform", Value: platform}, + {Key: "artifactType", Value: types.S}, + }, + nil, + ) + + decl := rego.Function{ + Name: ociDescriptorName, + Decl: types.NewFunction( + types.Args( + types.Named("ref", types.S).Description("OCI descriptor reference"), + ), + types.Named("object", manifest).Description("the OCI descriptor object"), + ), + // As per the documentation, enable memoization to ensure function evaluation is + // deterministic. But also mark it as non-deterministic because it does rely on external + // entities, i.e. OCI registry. https://www.openpolicyagent.org/docs/latest/extensions/ + Memoize: true, + Nondeterministic: true, + } + + rego.RegisterBuiltin1(&decl, ociDescriptor) + // Due to https://github.com/open-policy-agent/opa/issues/6449, we cannot set a description for + // the custom function through the call above. As a workaround we re-register the function with + // a declaration that does include the description. + ast.RegisterBuiltin(&ast.Builtin{ + Name: decl.Name, + Description: "Fetch a raw Image from an OCI registry.", + Decl: decl.Decl, + Nondeterministic: decl.Nondeterministic, + }) +} + func registerOCIImageManifest() { platform := types.NewObject( []*types.StaticProperty{ @@ -225,6 +286,28 @@ func ociBlob(bctx rego.BuiltinContext, a *ast.Term) (*ast.Term, error) { return ast.StringTerm(blob.String()), nil } +func ociDescriptor(bctx rego.BuiltinContext, a *ast.Term) (*ast.Term, error) { + log := log.WithField("rego", ociDescriptor) + uri, ok := a.Value.(ast.String) + if !ok { + return nil, nil + } + + ref, err := name.NewDigest(string(uri)) + if err != nil { + log.Errorf("new digest: %s", err) + return nil, nil + } + + descriptor, err := oci.NewClient(bctx.Context).Head(ref) + if err != nil { + log.Errorf("fetch image: %s", err) + return nil, nil + } + + return newDescriptorTerm(*descriptor), nil +} + func ociImageManifest(bctx rego.BuiltinContext, a *ast.Term) (*ast.Term, error) { log := log.WithField("rego", ociImageManifestName) uri, ok := a.Value.(ast.String) @@ -378,6 +461,7 @@ func newAnnotationsTerm(annotations map[string]string) *ast.Term { func init() { registerOCIBlob() + registerOCIDescriptor() registerOCIImageFiles() registerOCIImageManifest() } diff --git a/internal/rego/oci/oci_test.go b/internal/rego/oci/oci_test.go index dbf303369..a4a561a69 100644 --- a/internal/rego/oci/oci_test.go +++ b/internal/rego/oci/oci_test.go @@ -110,6 +110,122 @@ func TestOCIBlob(t *testing.T) { } } +func TestOCIDescriptorManifest(t *testing.T) { + cases := []struct { + name string + ref *ast.Term + descriptor *v1.Descriptor + err error + }{ + { + name: "complete image manifest", + ref: ast.StringTerm("registry.local/spam:latest@sha256:01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b"), + descriptor: &v1.Descriptor{ + MediaType: types.OCIManifestSchema1, + Size: 123, + Digest: v1.Hash{ + Algorithm: "sha256", + Hex: "4e388ab32b10dc8dbc7e28144f552830adc74787c1e2c0824032078a79f227fb", + }, + Data: []byte(`{"data": "config"}`), + URLs: []string{"https://config-1.local/spam", "https://config-2.local/spam"}, + Annotations: map[string]string{ + "config.annotation.1": "config.annotation.value.1", + "config.annotation.2": "config.annotation.value.2", + }, + Platform: &v1.Platform{ + Architecture: "arch", + OS: "os", + OSVersion: "os-version", + OSFeatures: []string{"os-feature-1", "os-feature-2"}, + Variant: "variant", + Features: []string{"feature-1", "feature-2"}, + }, + ArtifactType: "artifact-type", + }, + }, + { + name: "minimal image manifest", + ref: ast.StringTerm("registry.local/spam:latest@sha256:01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b"), + descriptor: &v1.Descriptor{ + MediaType: types.OCIManifestSchema1, + Size: 123, + Digest: v1.Hash{ + Algorithm: "sha256", + Hex: "4e388ab32b10dc8dbc7e28144f552830adc74787c1e2c0824032078a79f227fb", + }, + }, + }, + { + name: "minimal image index", + ref: ast.StringTerm("registry.local/spam:latest@sha256:01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b"), + descriptor: &v1.Descriptor{ + MediaType: types.OCIImageIndex, + Size: 123, + Digest: v1.Hash{ + Algorithm: "sha256", + Hex: "4e388ab32b10dc8dbc7e28144f552830adc74787c1e2c0824032078a79f227fb", + }, + }, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + client := fake.FakeClient{} + if c.err != nil { + client.On("Head", mock.Anything).Return(nil, c.err) + } else { + client.On("Head", mock.Anything).Return(c.descriptor, nil) + } + ctx := oci.WithClient(context.Background(), &client) + bctx := rego.BuiltinContext{Context: ctx} + + got, err := ociDescriptor(bctx, c.ref) + require.NoError(t, err) + if c.err != nil { + require.Nil(t, got) + } else { + require.NotNil(t, got) + snaps.MatchJSON(t, got) + } + }) + } +} + +func TestOCIDescriptorErrors(t *testing.T) { + cases := []struct { + name string + ref *ast.Term + }{ + { + name: "missing digest", + ref: ast.StringTerm("registry.local/spam:latest"), + }, + { + name: "bad image ref", + ref: ast.StringTerm("......registry.local/spam:latest@sha256:01ba4719c80b6fe911b091a7c05124b64eeece964e09c058ef8f9805daca546b"), + }, + { + name: "invalid ref type", + ref: ast.IntNumberTerm(42), + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + client := fake.FakeClient{} + client.On("Head", mock.Anything, mock.Anything).Return(nil, errors.New("expected")) + ctx := oci.WithClient(context.Background(), &client) + bctx := rego.BuiltinContext{Context: ctx} + + got, err := ociDescriptor(bctx, c.ref) + require.NoError(t, err) + require.Nil(t, got) + }) + } +} + func TestOCIImageManifest(t *testing.T) { cases := []struct { name string @@ -360,6 +476,7 @@ func TestOCIImageFiles(t *testing.T) { func TestFunctionsRegistered(t *testing.T) { names := []string{ ociBlobName, + ociDescriptorName, ociImageFilesName, ociImageManifestName, }