From 5f47666896305b764925b944905c90f98364bb43 Mon Sep 17 00:00:00 2001 From: robnester-rh Date: Wed, 20 Nov 2024 09:25:13 -0500 Subject: [PATCH] Revert "Use given source URLs as keys in download cache" This reverts commit 087f4659b10ada36b092d3f599e024d5c79ac955. Ref: EC-1023 Signed-off-by: robnester-rh --- cmd/fetch/fetch_policy.go | 5 +- cmd/inspect/inspect_policy.go | 3 +- cmd/inspect/inspect_policy_data.go | 3 +- cmd/validate/image.go | 90 +++++++++---------- cmd/validate/image_integration_test.go | 5 +- cmd/validate/image_test.go | 11 +-- features/__snapshots__/validate_image.snap | 8 +- features/__snapshots__/validate_input.snap | 6 +- features/validate_image.feature | 39 ++++---- internal/evaluation_target/input/input.go | 6 +- internal/evaluator/conftest_evaluator_test.go | 7 +- internal/input/validate.go | 4 +- internal/mutate/mutate.go | 65 -------------- internal/mutate/mutate_test.go | 63 ------------- internal/policy/policy.go | 65 ++++++++++++++ internal/policy/policy_test.go | 11 ++- internal/policy/source/git_config.go | 10 +-- internal/policy/source/source.go | 21 +++-- internal/policy/source/source_test.go | 34 +++---- 19 files changed, 186 insertions(+), 270 deletions(-) delete mode 100644 internal/mutate/mutate.go delete mode 100644 internal/mutate/mutate_test.go diff --git a/cmd/fetch/fetch_policy.go b/cmd/fetch/fetch_policy.go index 7a04fdc1f..9b8bd6a2c 100644 --- a/cmd/fetch/fetch_policy.go +++ b/cmd/fetch/fetch_policy.go @@ -23,7 +23,6 @@ import ( "github.com/spf13/afero" "github.com/spf13/cobra" - "github.com/enterprise-contract/ec-cli/internal/mutate" "github.com/enterprise-contract/ec-cli/internal/policy/source" "github.com/enterprise-contract/ec-cli/internal/utils" ) @@ -109,11 +108,11 @@ func fetchPolicyCmd() *cobra.Command { sources := make([]*source.PolicyUrl, 0, len(sourceUrls)+len(dataSourceUrls)) for _, url := range sourceUrls { - sources = append(sources, &source.PolicyUrl{Url: mutate.Const(url), Kind: source.PolicyKind}) + sources = append(sources, &source.PolicyUrl{Url: url, Kind: source.PolicyKind}) } for _, url := range dataSourceUrls { - sources = append(sources, &source.PolicyUrl{Url: mutate.Const(url), Kind: source.DataKind}) + sources = append(sources, &source.PolicyUrl{Url: url, Kind: source.DataKind}) } for _, s := range sources { diff --git a/cmd/inspect/inspect_policy.go b/cmd/inspect/inspect_policy.go index c59a55b83..70b53bb2c 100644 --- a/cmd/inspect/inspect_policy.go +++ b/cmd/inspect/inspect_policy.go @@ -28,7 +28,6 @@ import ( "github.com/spf13/cobra" "golang.org/x/exp/slices" - "github.com/enterprise-contract/ec-cli/internal/mutate" "github.com/enterprise-contract/ec-cli/internal/opa" opaRule "github.com/enterprise-contract/ec-cli/internal/opa/rule" "github.com/enterprise-contract/ec-cli/internal/policy" @@ -119,7 +118,7 @@ func inspectPolicyCmd() *cobra.Command { allResults := make(map[string][]*ast.AnnotationsRef) for _, url := range sourceUrls { - s := &source.PolicyUrl{Url: mutate.Const(url), Kind: source.PolicyKind} + s := &source.PolicyUrl{Url: url, Kind: source.PolicyKind} // Download policyDir, err := s.GetPolicy(ctx, destDir, false) diff --git a/cmd/inspect/inspect_policy_data.go b/cmd/inspect/inspect_policy_data.go index a3237d8b4..f51df593d 100644 --- a/cmd/inspect/inspect_policy_data.go +++ b/cmd/inspect/inspect_policy_data.go @@ -31,7 +31,6 @@ import ( "golang.org/x/exp/slices" "sigs.k8s.io/yaml" - "github.com/enterprise-contract/ec-cli/internal/mutate" "github.com/enterprise-contract/ec-cli/internal/policy/source" "github.com/enterprise-contract/ec-cli/internal/utils" ) @@ -89,7 +88,7 @@ func inspectPolicyDataCmd() *cobra.Command { allData := make(map[string]interface{}) for _, url := range sourceUrls { - s := &source.PolicyUrl{Url: mutate.Const(url), Kind: source.PolicyKind} + s := &source.PolicyUrl{Url: url, Kind: source.PolicyKind} // Download policyDir, err := s.GetPolicy(ctx, destDir, false) diff --git a/cmd/validate/image.go b/cmd/validate/image.go index 2e19d99b2..2538ada80 100644 --- a/cmd/validate/image.go +++ b/cmd/validate/image.go @@ -230,63 +230,63 @@ func validateImageCmd(validate imageValidationFunc) *cobra.Command { RekorURL: data.rekorURL, } - p, err := policy.NewPolicy(ctx, policyOptions) - if err != nil { + // We're not currently using the policyCache returned from PreProcessPolicy, but we could + // use it to cache the policy for future use. + if p, _, err := policy.PreProcessPolicy(ctx, policyOptions); err != nil { allErrors = errors.Join(allErrors, err) - return - } + } else { + // inject extra variables into rule data per source + if len(data.extraRuleData) > 0 { + policySpec := p.Spec() + sources := policySpec.Sources + for i := range sources { + src := sources[i] + var rule_data_raw []byte + unmarshaled := make(map[string]interface{}) + + if src.RuleData != nil { + rule_data_raw, err = src.RuleData.MarshalJSON() + if err != nil { + log.Errorf("Unable to parse ruledata to raw data") + } + err = json.Unmarshal(rule_data_raw, &unmarshaled) + if err != nil { + log.Errorf("Unable to parse ruledata into standard JSON object") + } + } else { + sources[i].RuleData = new(extv1.JSON) + } - // inject extra variables into rule data per source - if len(data.extraRuleData) > 0 { - policySpec := p.Spec() - sources := policySpec.Sources - for i := range sources { - src := sources[i] - var rule_data_raw []byte - unmarshaled := make(map[string]interface{}) - - if src.RuleData != nil { - rule_data_raw, err = src.RuleData.MarshalJSON() - if err != nil { - log.Errorf("Unable to parse ruledata to raw data") + for j := range data.extraRuleData { + parts := strings.SplitN(data.extraRuleData[j], "=", 2) + if len(parts) < 2 { + log.Errorf("Incorrect syntax for --extra-rule-data") + } + extraRuleDataPolicyConfig, err := validate_utils.GetPolicyConfig(ctx, parts[1]) + if err != nil { + log.Errorf("Unable to load data from extraRuleData: %s", err.Error()) + } + unmarshaled[parts[0]] = extraRuleDataPolicyConfig } - err = json.Unmarshal(rule_data_raw, &unmarshaled) + rule_data_raw, err = json.Marshal(unmarshaled) if err != nil { - log.Errorf("Unable to parse ruledata into standard JSON object") + log.Errorf("Unable to parse updated ruledata: %s", err.Error()) } - } else { - sources[i].RuleData = new(extv1.JSON) - } - for j := range data.extraRuleData { - parts := strings.SplitN(data.extraRuleData[j], "=", 2) - if len(parts) < 2 { - log.Errorf("Incorrect syntax for --extra-rule-data") + if rule_data_raw == nil { + log.Errorf("Invalid rule data JSON") } - extraRuleDataPolicyConfig, err := validate_utils.GetPolicyConfig(ctx, parts[1]) + + err = sources[i].RuleData.UnmarshalJSON(rule_data_raw) if err != nil { - log.Errorf("Unable to load data from extraRuleData: %s", err.Error()) + log.Errorf("Unable to marshal updated JSON: %s", err.Error()) } - unmarshaled[parts[0]] = extraRuleDataPolicyConfig - } - rule_data_raw, err = json.Marshal(unmarshaled) - if err != nil { - log.Errorf("Unable to parse updated ruledata: %s", err.Error()) - } - - if rule_data_raw == nil { - log.Errorf("Invalid rule data JSON") - } - - err = sources[i].RuleData.UnmarshalJSON(rule_data_raw) - if err != nil { - log.Errorf("Unable to marshal updated JSON: %s", err.Error()) } + policySpec.Sources = sources + p = p.WithSpec(policySpec) } - policySpec.Sources = sources - p = p.WithSpec(policySpec) + data.policy = p } - data.policy = p return }, diff --git a/cmd/validate/image_integration_test.go b/cmd/validate/image_integration_test.go index 666a5300c..c530904e8 100644 --- a/cmd/validate/image_integration_test.go +++ b/cmd/validate/image_integration_test.go @@ -28,6 +28,7 @@ import ( "time" "github.com/enterprise-contract/enterprise-contract-controller/api/v1alpha1" + ociMetadata "github.com/enterprise-contract/go-gather/metadata/oci" app "github.com/konflux-ci/application-api/api/v1alpha1" "github.com/spf13/afero" "github.com/stretchr/testify/assert" @@ -51,10 +52,12 @@ func TestEvaluatorLifecycle(t *testing.T) { commonMockClient(&client) ctx = oci.WithClient(ctx, &client) mdl := MockDownloader{} + downloaderCall := mdl.On("Download", mock.Anything, mock.Anything, false).Return(&ociMetadata.OCIMetadata{Digest: "sha256:da54bca5477bf4e3449bc37de1822888fa0fbb8d89c640218cb31b987374d357"}, nil).Times(noEvaluators) ctx = context.WithValue(ctx, source.DownloaderFuncKey, &mdl) evaluators := make([]*mockEvaluator, 0, noEvaluators) expectations := make([]*mock.Call, 0, noEvaluators+1) + expectations = append(expectations, downloaderCall) for i := 0; i < noEvaluators; i++ { e := mockEvaluator{} @@ -70,7 +73,7 @@ func TestEvaluatorLifecycle(t *testing.T) { newConftestEvaluator = func(_ context.Context, s []source.PolicySource, _ evaluator.ConfigProvider, _ v1alpha1.Source) (evaluator.Evaluator, error) { // We are splitting this url to get to the index of the evaluator. - idx, err := strconv.Atoi(s[0].PolicyUrl()) + idx, err := strconv.Atoi(strings.Split(strings.Split(s[0].PolicyUrl(), "@")[0], "::")[1]) require.NoError(t, err) return evaluators[idx], nil diff --git a/cmd/validate/image_test.go b/cmd/validate/image_test.go index 2b0209a2f..be44dada2 100644 --- a/cmd/validate/image_test.go +++ b/cmd/validate/image_test.go @@ -64,16 +64,7 @@ var rootArgs = []string{ } func happyValidator() imageValidationFunc { - return func(ctx context.Context, component app.SnapshotComponent, _ *app.SnapshotSpec, p policy.Policy, _ []evaluator.Evaluator, _ bool) (*output.Output, error) { - // simulate fetching of sources - for _, src := range p.Spec().Sources { - for _, url := range source.PolicySourcesFrom(src) { - if _, err := url.GetPolicy(ctx, "dest", false); err != nil { - return nil, err - } - } - } - + return func(_ context.Context, component app.SnapshotComponent, _ *app.SnapshotSpec, _ policy.Policy, _ []evaluator.Evaluator, _ bool) (*output.Output, error) { return &output.Output{ ImageSignatureCheck: output.VerificationStatus{ Passed: true, diff --git a/features/__snapshots__/validate_image.snap b/features/__snapshots__/validate_image.snap index c9b458d41..dfb60cc31 100755 --- a/features/__snapshots__/validate_image.snap +++ b/features/__snapshots__/validate_image.snap @@ -1122,7 +1122,7 @@ Error: success criteria not met "sources": [ { "policy": [ - "git::https://${GITHOST}/git/unexpected-keyless-cert.git" + "git::${GITHOST}/git/unexpected-keyless-cert.git?ref=${LATEST_COMMIT}" ] } ] @@ -1167,7 +1167,7 @@ Error: success criteria not met "sources": [ { "policy": [ - "git::https://${GITHOST}/git/invalid-image-signature.git" + "git::${GITHOST}/git/invalid-image-signature.git?ref=${LATEST_COMMIT}" ] } ], @@ -1598,7 +1598,7 @@ Error: success criteria not met "sources": [ { "policy": [ - "git::https://${GITHOST}/git/mismatched-image-digest.git" + "git::${GITHOST}/git/mismatched-image-digest.git?ref=${LATEST_COMMIT}" ] } ], @@ -2744,7 +2744,7 @@ ${__________known_PUBLIC_KEY} "sources": [ { "policy": [ - "git::https://${GITHOST}/git/rekor-by-default.git" + "git::${GITHOST}/git/rekor-by-default.git?ref=${LATEST_COMMIT}" ] } ], diff --git a/features/__snapshots__/validate_input.snap b/features/__snapshots__/validate_input.snap index b59fc2f9c..7f301a3e0 100755 --- a/features/__snapshots__/validate_input.snap +++ b/features/__snapshots__/validate_input.snap @@ -15,7 +15,7 @@ "sources": [ { "policy": [ - "git::${GITHOST}/git/happy-day-policy.git?ref=${LATEST_COMMIT}" + "git::https://${GITHOST}/git/happy-day-policy.git" ] } ] @@ -68,12 +68,12 @@ Error: error validating file pipeline_definition.yaml: evaluating policy: no reg "sources": [ { "policy": [ - "git::${GITHOST}/git/ham-policy?ref=${LATEST_COMMIT}" + "git::https://${GITHOST}/git/ham-policy" ] }, { "policy": [ - "git::${GITHOST}/git/spam-policy?ref=4707d251d08b466389705c121d84efa2683114cf" + "git::https://${GITHOST}/git/spam-policy" ] } ] diff --git a/features/validate_image.feature b/features/validate_image.feature index 93cded8e2..00be89757 100644 --- a/features/validate_image.feature +++ b/features/validate_image.feature @@ -1121,25 +1121,26 @@ Feature: evaluate enterprise contract Then the exit status should be 0 Then the output should match the snapshot - Scenario: many components and sources - Given a key pair named "known" - And a git repository named "multitude-policy" with - | main.rego | examples/happy_day.rego | - And policy configuration named "ec-policy" with 10 policy sources from "git::https://${GITHOST}/git/multitude-policy.git", patched with - | [{"op": "add", "path": "/sources/0/ruleData", "value": {"key": "value"}}] | - | [{"op": "add", "path": "/sources/1/ruleData", "value": {"something": "here"}}] | - | [{"op": "add", "path": "/sources/2/ruleData", "value": {"key": "different"}}] | - | [{"op": "add", "path": "/sources/3/ruleData", "value": {"hello": "world"}}] | - | [{"op": "add", "path": "/sources/4/ruleData", "value": {"foo": "bar"}}] | - | [{"op": "add", "path": "/sources/5/ruleData", "value": {"peek": "poke"}}] | - | [{"op": "add", "path": "/sources/6/ruleData", "value": {"hide": "seek"}}] | - | [{"op": "add", "path": "/sources/7/ruleData", "value": {"hokus": "pokus"}}] | - | [{"op": "add", "path": "/sources/8/ruleData", "value": {"mr": "mxyzptlk"}}] | - | [{"op": "add", "path": "/sources/9/ruleData", "value": {"more": "data"}}] | - And an Snapshot named "multitude" with 10 components signed with "known" key - When ec command is run with "validate image --snapshot acceptance/multitude --policy acceptance/ec-policy --public-key ${known_PUBLIC_KEY} --rekor-url ${REKOR} --show-successes --output json" - Then the exit status should be 0 - And the output should match the snapshot + # Commented out as part of EC-1023. This will be enabled once the issue is resolved. + # Scenario: many components and sources + # Given a key pair named "known" + # And a git repository named "multitude-policy" with + # | main.rego | examples/happy_day.rego | + # And policy configuration named "ec-policy" with 10 policy sources from "git::https://${GITHOST}/git/multitude-policy.git", patched with + # | [{"op": "add", "path": "/sources/0/ruleData", "value": {"key": "value"}}] | + # | [{"op": "add", "path": "/sources/1/ruleData", "value": {"something": "here"}}] | + # | [{"op": "add", "path": "/sources/2/ruleData", "value": {"key": "different"}}] | + # | [{"op": "add", "path": "/sources/3/ruleData", "value": {"hello": "world"}}] | + # | [{"op": "add", "path": "/sources/4/ruleData", "value": {"foo": "bar"}}] | + # | [{"op": "add", "path": "/sources/5/ruleData", "value": {"peek": "poke"}}] | + # | [{"op": "add", "path": "/sources/6/ruleData", "value": {"hide": "seek"}}] | + # | [{"op": "add", "path": "/sources/7/ruleData", "value": {"hokus": "pokus"}}] | + # | [{"op": "add", "path": "/sources/8/ruleData", "value": {"mr": "mxyzptlk"}}] | + # | [{"op": "add", "path": "/sources/9/ruleData", "value": {"more": "data"}}] | + # And an Snapshot named "multitude" with 10 components signed with "known" key + # When ec command is run with "validate image --snapshot acceptance/multitude --policy acceptance/ec-policy --public-key ${known_PUBLIC_KEY} --rekor-url ${REKOR} --show-successes --output json" + # Then the exit status should be 0 + # And the output should match the snapshot Scenario: Format options Given a key pair named "known" diff --git a/internal/evaluation_target/input/input.go b/internal/evaluation_target/input/input.go index 79eddfe62..4e980dd0c 100644 --- a/internal/evaluation_target/input/input.go +++ b/internal/evaluation_target/input/input.go @@ -36,7 +36,7 @@ type Input struct { // NewInput returns a Input struct with FPath and evaluator ready to use func NewInput(ctx context.Context, paths []string, p policy.Policy) (*Input, error) { - in := &Input{ + i := &Input{ Paths: paths, } @@ -55,8 +55,8 @@ func NewInput(ctx context.Context, paths []string, p policy.Policy) (*Input, err } log.Debug("Conftest evaluator initialized") - in.Evaluators = append(in.Evaluators, c) + i.Evaluators = append(i.Evaluators, c) } - return in, nil + return i, nil } diff --git a/internal/evaluator/conftest_evaluator_test.go b/internal/evaluator/conftest_evaluator_test.go index 908186c6c..6202d92b7 100644 --- a/internal/evaluator/conftest_evaluator_test.go +++ b/internal/evaluator/conftest_evaluator_test.go @@ -45,7 +45,6 @@ import ( "k8s.io/kube-openapi/pkg/util/sets" "github.com/enterprise-contract/ec-cli/internal/downloader" - "github.com/enterprise-contract/ec-cli/internal/mutate" "github.com/enterprise-contract/ec-cli/internal/opa/rule" "github.com/enterprise-contract/ec-cli/internal/policy" "github.com/enterprise-contract/ec-cli/internal/policy/source" @@ -1820,7 +1819,7 @@ func TestConftestEvaluatorEvaluate(t *testing.T) { evaluator, err := NewConftestEvaluator(ctx, []source.PolicySource{ &source.PolicyUrl{ - Url: mutate.Const(rules), + Url: rules, Kind: source.PolicyKind, }, }, config, ecc.Source{}) @@ -1883,7 +1882,7 @@ func TestUnconformingRule(t *testing.T) { evaluator, err := NewConftestEvaluator(ctx, []source.PolicySource{ &source.PolicyUrl{ - Url: mutate.Const(rules), + Url: rules, Kind: source.PolicyKind, }, }, p, ecc.Source{}) @@ -2099,7 +2098,7 @@ func TestNewConftestEvaluatorComputeIncludeExclude(t *testing.T) { evaluator, err := NewConftestEvaluator(ctx, []source.PolicySource{ &source.PolicyUrl{ - Url: mutate.Const(path.Join(dir, "policy", "rules.tar")), + Url: path.Join(dir, "policy", "rules.tar"), Kind: source.PolicyKind, }, }, p, tt.source) diff --git a/internal/input/validate.go b/internal/input/validate.go index e3ab3135f..00e8ac04d 100644 --- a/internal/input/validate.go +++ b/internal/input/validate.go @@ -47,14 +47,14 @@ func ValidateInput(ctx context.Context, fpath string, policy policy.Policy, deta return nil, err } - in, err := inputFile(ctx, inputFiles, policy) + p, err := inputFile(ctx, inputFiles, policy) if err != nil { log.Debug("Failed to create input!") return nil, err } var allResults []evaluator.Outcome - for _, e := range in.Evaluators { + for _, e := range p.Evaluators { results, _, err := e.Evaluate(ctx, evaluator.EvaluationTarget{Inputs: inputFiles}) if err != nil { return nil, fmt.Errorf("evaluating policy: %w", err) diff --git a/internal/mutate/mutate.go b/internal/mutate/mutate.go deleted file mode 100644 index faffed037..000000000 --- a/internal/mutate/mutate.go +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright The Enterprise Contract Contributors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.` -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -package mutate - -import "fmt" - -type Mut[T any] interface { - Value() T - Set(T) -} - -type mutableValue[T any] struct { - v *T -} - -func (m *mutableValue[T]) Value() T { - return *m.v -} - -func (m *mutableValue[T]) Set(value T) { - *m.v = value -} - -func (m *mutableValue[T]) String() string { - return fmt.Sprint("%", *m.v) -} - -func Value[T any](value *T) Mut[T] { - return &mutableValue[T]{value} -} - -func Const[T any](value T) Mut[T] { - return &mutableValue[T]{&value} -} - -type mutableSlice[T any] struct { - s []T - idx int -} - -func (m *mutableSlice[T]) Value() T { - return m.s[m.idx] -} - -func (m *mutableSlice[T]) Set(value T) { - m.s[m.idx] = value -} - -func Slice[T any](s []T, i int) Mut[T] { - return &mutableSlice[T]{s, i} -} diff --git a/internal/mutate/mutate_test.go b/internal/mutate/mutate_test.go deleted file mode 100644 index cf75af44b..000000000 --- a/internal/mutate/mutate_test.go +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright The Enterprise Contract Contributors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.` -// See the License for the specific language governing permissions and -// limitations under the License. -// -// SPDX-License-Identifier: Apache-2.0 - -package mutate - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestMutatesStructFields(t *testing.T) { - s := struct { - some string - }{ - "hello", - } - - m := Value(&s.some) - - assert.Equal(t, "hello", s.some) - assert.Equal(t, "hello", m.Value()) - - m.Set("hi") - assert.Equal(t, "hi", s.some) - assert.Equal(t, "hi", m.Value()) -} - -func TestMutatesSliceItems(t *testing.T) { - s := []string{ - "hello", - "world", - } - - m := Slice(s, 1) - - assert.Equal(t, []string{"hello", "world"}, s) - assert.Equal(t, "world", m.Value()) - - m.Set("universe") - assert.Equal(t, []string{"hello", "universe"}, s) - assert.Equal(t, "universe", m.Value()) -} - -func TestEquality(t *testing.T) { - x := "value" - - assert.Equal(t, Value(&x), Value(&x)) - assert.Equal(t, Value(&x), Const("value")) -} diff --git a/internal/policy/policy.go b/internal/policy/policy.go index d9a1fcece..c12807ed5 100644 --- a/internal/policy/policy.go +++ b/internal/policy/policy.go @@ -27,6 +27,7 @@ import ( "strings" "time" + "github.com/enterprise-contract/enterprise-contract-controller/api/v1alpha1" ecc "github.com/enterprise-contract/enterprise-contract-controller/api/v1alpha1" schemaExporter "github.com/invopop/jsonschema" "github.com/santhosh-tekuri/jsonschema/v5" @@ -581,6 +582,70 @@ func validatePolicyConfig(policyConfig string) error { return nil } +// PreProcessPolicy fetches policy sources and returns a policy object with +// pinned SHA/image digest URL where applicable, along with a policy cache object. +func PreProcessPolicy(ctx context.Context, policyOptions Options) (Policy, *cache.PolicyCache, error) { + var policyCache *cache.PolicyCache + pinnedPolicyUrls := map[string][]string{} + policyCache, err := cache.NewPolicyCache(ctx) + if err != nil { + return nil, nil, err + } + + p, err := NewPolicy(ctx, policyOptions) + if err != nil { + return nil, nil, err + } + + sources := p.Spec().Sources + for i, sourceGroup := range sources { + log.Debugf("Fetching policy source group '%+v'\n", sourceGroup.Name) + policySources := PolicySourcesFrom(sourceGroup) + + fs := utils.FS(ctx) + dir, err := utils.CreateWorkDir(fs) + if err != nil { + log.Debug("Failed to create work dir!") + return nil, nil, err + } + + for _, policySource := range policySources { + if strings.HasPrefix(policySource.PolicyUrl(), "data:") { + continue + } + + destDir, err := policySource.GetPolicy(ctx, dir, false) + if err != nil { + log.Debugf("Unable to download source from %s!", policySource.PolicyUrl()) + return nil, nil, err + } + log.Debugf("Downloaded policy source from %s to %s\n", policySource.PolicyUrl(), destDir) + + url := policySource.PolicyUrl() + + if _, found := policyCache.Get(policySource.PolicyUrl()); !found { + log.Debugf("Cache miss for: %s, adding to cache", url) + policyCache.Set(url, destDir, nil) + pinnedPolicyUrls[policySource.Subdir()] = append(pinnedPolicyUrls[policySource.Subdir()], url) + log.Debugf("Added %s to the pinnedPolicyUrls in \"%s\"", url, policySource.Subdir()) + } else { + log.Debugf("Cache hit for: %s", url) + } + } + + sources[i] = v1alpha1.Source{ + Name: sourceGroup.Name, + Policy: urls(policySources, source.PolicyKind), + Data: urls(policySources, source.DataKind), + RuleData: sourceGroup.RuleData, + Config: sourceGroup.Config, + VolatileConfig: sourceGroup.VolatileConfig, + } + } + + return p, policyCache, err +} + func urls(s []source.PolicySource, kind source.PolicyType) []string { ret := make([]string, 0, len(s)) for _, u := range s { diff --git a/internal/policy/policy_test.go b/internal/policy/policy_test.go index fa2b2776a..acb7a456b 100644 --- a/internal/policy/policy_test.go +++ b/internal/policy/policy_test.go @@ -39,7 +39,6 @@ import ( "sigs.k8s.io/yaml" "github.com/enterprise-contract/ec-cli/internal/kubernetes" - "github.com/enterprise-contract/ec-cli/internal/mutate" "github.com/enterprise-contract/ec-cli/internal/policy/source" "github.com/enterprise-contract/ec-cli/internal/utils" ) @@ -832,9 +831,9 @@ func TestUrls(t *testing.T) { { name: "Returns URLs of the specified kind", s: []source.PolicySource{ - &source.PolicyUrl{Url: mutate.Const("http://example.com/policy1"), Kind: source.PolicyKind}, - &source.PolicyUrl{Url: mutate.Const("http://example.com/data1"), Kind: source.DataKind}, - &source.PolicyUrl{Url: mutate.Const("http://example.com/policy2"), Kind: source.PolicyKind}, + &source.PolicyUrl{Url: "http://example.com/policy1", Kind: source.PolicyKind}, + &source.PolicyUrl{Url: "http://example.com/data1", Kind: source.DataKind}, + &source.PolicyUrl{Url: "http://example.com/policy2", Kind: source.PolicyKind}, }, kind: source.PolicyKind, want: []string{"http://example.com/policy1", "http://example.com/policy2"}, @@ -842,8 +841,8 @@ func TestUrls(t *testing.T) { { name: "Returns empty slice when no URLs of the specified kind", s: []source.PolicySource{ - &source.PolicyUrl{Url: mutate.Const("http://example.com/data1"), Kind: source.PolicyType("data")}, - &source.PolicyUrl{Url: mutate.Const("http://example.com/data2"), Kind: source.PolicyType("data")}, + &source.PolicyUrl{Url: "http://example.com/data1", Kind: source.PolicyType("data")}, + &source.PolicyUrl{Url: "http://example.com/data2", Kind: source.PolicyType("data")}, }, kind: source.PolicyKind, want: []string{}, diff --git a/internal/policy/source/git_config.go b/internal/policy/source/git_config.go index d534b52df..d615a2f50 100644 --- a/internal/policy/source/git_config.go +++ b/internal/policy/source/git_config.go @@ -28,8 +28,6 @@ import ( getter "github.com/hashicorp/go-getter" log "github.com/sirupsen/logrus" - - "github.com/enterprise-contract/ec-cli/internal/mutate" ) // SourceIsFile returns true if go-getter thinks the src looks like a file path. @@ -59,21 +57,21 @@ func SourceIsHttp(src string) bool { func GoGetterDownload(ctx context.Context, tmpDir, src string) (string, error) { // Download the config from a url c := PolicyUrl{ - Url: mutate.Value(&src), + Url: src, Kind: ConfigKind, } configDir, err := c.GetPolicy(ctx, tmpDir, false) if err != nil { - log.Debugf("Failed to download policy config from %s", c.Url.Value()) + log.Debugf("Failed to download policy config from %s", c.Url) return "", err } - log.Debugf("Downloaded policy config from %s to %s", c.Url.Value(), configDir) + log.Debugf("Downloaded policy config from %s to %s", c.Url, configDir) // Look for a suitable file to use for the config configFile, err := choosePolicyFile(ctx, configDir) if err != nil { // A more useful error message: - return "", fmt.Errorf("no suitable config file found at %s", c.Url.Value()) + return "", fmt.Errorf("no suitable config file found at %s", c.Url) } log.Debugf("Chose file %s to use for the policy config", configFile) return configFile, nil diff --git a/internal/policy/source/source.go b/internal/policy/source/source.go index 4971cad30..e49a8fccc 100644 --- a/internal/policy/source/source.go +++ b/internal/policy/source/source.go @@ -40,7 +40,6 @@ import ( "github.com/spf13/afero" "github.com/enterprise-contract/ec-cli/internal/downloader" - "github.com/enterprise-contract/ec-cli/internal/mutate" "github.com/enterprise-contract/ec-cli/internal/utils" ) @@ -71,7 +70,8 @@ type PolicySource interface { } type PolicyUrl struct { - Url mutate.Mut[string] + // A string containing a go-getter style source url compatible with conftest pull + Url string Kind PolicyType } @@ -144,7 +144,7 @@ func (p *PolicyUrl) GetPolicy(ctx context.Context, workDir string, showMsg bool) if trace.IsEnabled() { region := trace.StartRegion(ctx, "ec:get-policy") defer region.End() - trace.Logf(ctx, "", "policy=%q", p.Url.Value()) + trace.Logf(ctx, "", "policy=%q", p.Url) } dl := func(source string, dest string) (metadata.Metadata, error) { @@ -160,18 +160,17 @@ func (p *PolicyUrl) GetPolicy(ctx context.Context, workDir string, showMsg bool) return "", err } - pinned, err := metadata.GetPinnedURL(p.Url.Value()) + p.Url, err = metadata.GetPinnedURL(p.Url) + log.Debug("Pinned URL: ", p.Url) if err != nil { return "", err } - log.Debug("Pinned URL: ", pinned) - p.Url.Set(pinned) return dest, err } func (p *PolicyUrl) PolicyUrl() string { - return p.Url.Value() + return p.Url } func (p *PolicyUrl) Subdir() string { @@ -254,13 +253,13 @@ func (inlineData) Type() PolicyType { func PolicySourcesFrom(s ecc.Source) []PolicySource { policySources := make([]PolicySource, 0, len(s.Policy)+len(s.Data)) - for i := range s.Policy { - url := PolicyUrl{Url: mutate.Slice(s.Policy, i), Kind: PolicyKind} + for _, policySourceUrl := range s.Policy { + url := PolicyUrl{Url: policySourceUrl, Kind: PolicyKind} policySources = append(policySources, &url) } - for i := range s.Data { - url := PolicyUrl{Url: mutate.Slice(s.Data, i), Kind: DataKind} + for _, dataSourceUrl := range s.Data { + url := PolicyUrl{Url: dataSourceUrl, Kind: DataKind} policySources = append(policySources, &url) } diff --git a/internal/policy/source/source_test.go b/internal/policy/source/source_test.go index 768388ffc..87bc36127 100644 --- a/internal/policy/source/source_test.go +++ b/internal/policy/source/source_test.go @@ -37,7 +37,6 @@ import ( "github.com/stretchr/testify/require" extv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" - "github.com/enterprise-contract/ec-cli/internal/mutate" "github.com/enterprise-contract/ec-cli/internal/utils" ) @@ -87,7 +86,7 @@ func TestGetPolicy(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - p := PolicyUrl{Url: mutate.Value(&tt.sourceUrl), Kind: PolicyKind} + p := PolicyUrl{Url: tt.sourceUrl, Kind: PolicyKind} dl := mockDownloader{} dl.On("Download", mock.MatchedBy(func(dest string) bool { @@ -138,13 +137,6 @@ func TestInlineDataSource(t *testing.T) { } func TestPolicySourcesFrom(t *testing.T) { - - policies1 := []string{"github.com/org/repo1//policy/", "github.com/org/repo2//policy/", "github.com/org/repo3//policy/"} - data1 := []string{"github.com/org/repo1//data/", "github.com/org/repo2//data/", "github.com/org/repo3//data/"} - - policies2 := []string{"github.com/org/repo1//policy/"} - data2 := []string{"github.com/org/repo1//data/"} - // var ruleData = &extv1.JSON{Raw: []byte("foo")} tests := []struct { name string @@ -155,29 +147,29 @@ func TestPolicySourcesFrom(t *testing.T) { name: "fetches policy configs", source: ecc.Source{ Name: "policy1", - Policy: policies1, - Data: data1, + Policy: []string{"github.com/org/repo1//policy/", "github.com/org/repo2//policy/", "github.com/org/repo3//policy/"}, + Data: []string{"github.com/org/repo1//data/", "github.com/org/repo2//data/", "github.com/org/repo3//data/"}, }, expected: []PolicySource{ - &PolicyUrl{Url: mutate.Slice(policies1, 0), Kind: PolicyKind}, - &PolicyUrl{Url: mutate.Slice(policies1, 1), Kind: PolicyKind}, - &PolicyUrl{Url: mutate.Slice(policies1, 2), Kind: PolicyKind}, - &PolicyUrl{Url: mutate.Slice(data1, 0), Kind: DataKind}, - &PolicyUrl{Url: mutate.Slice(data1, 1), Kind: DataKind}, - &PolicyUrl{Url: mutate.Slice(data1, 2), Kind: DataKind}, + &PolicyUrl{Url: "github.com/org/repo1//policy/", Kind: PolicyKind}, + &PolicyUrl{Url: "github.com/org/repo2//policy/", Kind: PolicyKind}, + &PolicyUrl{Url: "github.com/org/repo3//policy/", Kind: PolicyKind}, + &PolicyUrl{Url: "github.com/org/repo1//data/", Kind: DataKind}, + &PolicyUrl{Url: "github.com/org/repo2//data/", Kind: DataKind}, + &PolicyUrl{Url: "github.com/org/repo3//data/", Kind: DataKind}, }, }, { name: "handles rule data", source: ecc.Source{ Name: "policy2", - Policy: policies2, - Data: data2, + Policy: []string{"github.com/org/repo1//policy/"}, + Data: []string{"github.com/org/repo1//data/"}, RuleData: &extv1.JSON{Raw: []byte(`"foo":"bar"`)}, }, expected: []PolicySource{ - &PolicyUrl{Url: mutate.Slice(policies2, 0), Kind: PolicyKind}, - &PolicyUrl{Url: mutate.Slice(data2, 0), Kind: DataKind}, + &PolicyUrl{Url: "github.com/org/repo1//policy/", Kind: PolicyKind}, + &PolicyUrl{Url: "github.com/org/repo1//data/", Kind: DataKind}, inlineData{source: []byte("{\"rule_data__configuration__\":\"foo\":\"bar\"}")}, }, },