From 558136851b4079998e2a7af583b7336f59d1840d Mon Sep 17 00:00:00 2001 From: Luca Sepe Date: Thu, 29 Feb 2024 16:03:34 +0100 Subject: [PATCH] feat: form template data api and merge (#16) --- apis/ui/formtemplates/v1alpha1/types.go | 8 + go.mod | 4 +- .../formtemplates/evaluator/evaluator.go | 67 +- .../formtemplates/evaluator/evaluator_test.go | 39 +- .../formtemplates/formtemplates_test.go | 17 +- .../widgets/formtemplates/merger/merger.go | 27 + .../formtemplates/merger/merger_test.go | 78 ++ internal/strvals/literal_parser.go | 244 ++++++ internal/strvals/literal_parser_test.go | 480 ++++++++++ internal/strvals/parser.go | 559 ++++++++++++ internal/strvals/parser_test.go | 818 ++++++++++++++++++ testdata/fireworksapp.sample.yaml | 6 +- testdata/formdefinition.sample.yaml | 2 +- testdata/formtemplate.api.sample.yaml | 39 + testdata/formtemplate.sample.yaml | 10 +- 15 files changed, 2310 insertions(+), 88 deletions(-) create mode 100644 internal/kubernetes/widgets/formtemplates/merger/merger.go create mode 100644 internal/kubernetes/widgets/formtemplates/merger/merger_test.go create mode 100644 internal/strvals/literal_parser.go create mode 100644 internal/strvals/literal_parser_test.go create mode 100644 internal/strvals/parser.go create mode 100644 internal/strvals/parser_test.go create mode 100644 testdata/formtemplate.api.sample.yaml diff --git a/apis/ui/formtemplates/v1alpha1/types.go b/apis/ui/formtemplates/v1alpha1/types.go index fde4e3d..2e50cf3 100644 --- a/apis/ui/formtemplates/v1alpha1/types.go +++ b/apis/ui/formtemplates/v1alpha1/types.go @@ -1,6 +1,9 @@ package v1alpha1 import ( + "fmt" + "strings" + "github.com/krateoplatformops/krateo-bff/apis/core" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -11,6 +14,11 @@ type DataItem struct { Value string `json:"value"` } +func (di *DataItem) String() string { + return fmt.Sprintf("%s=%s", + strings.TrimSpace(di.Path), strings.TrimSpace(di.Value)) +} + type FormTemplateSpec struct { // DefinitionRef: reference to FormDefintion DefinitionRef *core.Reference `json:"definitionRef"` diff --git a/go.mod b/go.mod index 4d3e3bb..773f171 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/google/go-cmp v0.6.0 github.com/itchyny/gojq v0.12.13 github.com/krateoplatformops/provider-runtime v0.7.0 + github.com/pkg/errors v0.9.1 github.com/rs/zerolog v1.31.0 k8s.io/api v0.29.2 k8s.io/apimachinery v0.29.2 @@ -16,6 +17,7 @@ require ( k8s.io/utils v0.0.0-20240102154912-e7106e64919e sigs.k8s.io/controller-runtime v0.17.0 sigs.k8s.io/controller-tools v0.13.0 + sigs.k8s.io/yaml v1.4.0 ) require ( @@ -43,7 +45,6 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/pkg/errors v0.9.1 // indirect github.com/spf13/cobra v1.7.0 // indirect github.com/spf13/pflag v1.0.5 // indirect golang.org/x/mod v0.14.0 // indirect @@ -64,5 +65,4 @@ require ( k8s.io/kube-openapi v0.0.0-20240126223410-2919ad4fcfec // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect - sigs.k8s.io/yaml v1.4.0 // indirect ) diff --git a/internal/kubernetes/widgets/formtemplates/evaluator/evaluator.go b/internal/kubernetes/widgets/formtemplates/evaluator/evaluator.go index 836dfe8..ef337c2 100644 --- a/internal/kubernetes/widgets/formtemplates/evaluator/evaluator.go +++ b/internal/kubernetes/widgets/formtemplates/evaluator/evaluator.go @@ -2,17 +2,10 @@ package evaluator import ( "context" - "fmt" "github.com/krateoplatformops/krateo-bff/apis/ui/formtemplates/v1alpha1" - "github.com/krateoplatformops/krateo-bff/internal/kubernetes/dynamic" - "github.com/krateoplatformops/krateo-bff/internal/kubernetes/formdefinitions" - formdefinitionsutil "github.com/krateoplatformops/krateo-bff/internal/kubernetes/formdefinitions/util" "github.com/krateoplatformops/krateo-bff/internal/tmpl" - "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/rest" ) @@ -24,7 +17,6 @@ type EvalOptions struct { RESTConfig *rest.Config AuthnNS string Subject string - Groups []string } func Eval(ctx context.Context, in *v1alpha1.FormTemplate, opts EvalOptions) error { @@ -40,61 +32,12 @@ func Eval(ctx context.Context, in *v1alpha1.FormTemplate, opts EvalOptions) erro tpl: tpl, apiList: in.Spec.APIList, }) - _ = ds // TODO - formdefinitionsClient, err := formdefinitions.NewClient(opts.RESTConfig) - if err != nil { - return err - } - - ref, err := formdefinitionsClient.Namespace(in.Spec.DefinitionRef.Namespace). - Get(ctx, in.Spec.DefinitionRef.Name) - if err != nil { - return err - } - - dyn, err := dynamic.NewGetter(opts.RESTConfig) - if err != nil { - return err - } - - src, err := dyn.Get(ctx, dynamic.GetOptions{ - GVK: schema.GroupVersionKind{ - Group: ref.Spec.Schema.Group, - Version: ref.Spec.Schema.Version, - Kind: ref.Spec.Schema.Kind, - }, - Namespace: ref.Namespace, - Name: ref.Name, - }) - if err != nil { - return err - } - - gr := formdefinitionsutil.InferGroupResource(ref) - - crd, err := dyn.Get(ctx, dynamic.GetOptions{ - GVK: schema.GroupVersionKind{ - Group: "apiextensions.k8s.io", - Version: "v1", - Kind: "CustomResourceDefinition", - }, - Name: gr.String(), - }) - if err != nil { - return err - } - - sch, err := dynamic.Extract(ctx, crd, fmt.Sprintf(filter, ref.Spec.Schema.Version)) - if err != nil { - return err - } - - in.Status.Content = &v1alpha1.FormTemplateStatusContent{ - Instance: &runtime.RawExtension{Object: src}, - Schema: &runtime.RawExtension{Object: &unstructured.Unstructured{ - Object: sch.(map[string]any), - }}, + for _, el := range in.Spec.Data { + el.Value, err = tpl.Execute(el.Value, ds) + if err != nil { + return err + } } return nil diff --git a/internal/kubernetes/widgets/formtemplates/evaluator/evaluator_test.go b/internal/kubernetes/widgets/formtemplates/evaluator/evaluator_test.go index 572c84b..c4b44e8 100644 --- a/internal/kubernetes/widgets/formtemplates/evaluator/evaluator_test.go +++ b/internal/kubernetes/widgets/formtemplates/evaluator/evaluator_test.go @@ -1,16 +1,53 @@ +//go:build integration +// +build integration + package evaluator import ( + "context" + "encoding/json" "os" "path/filepath" "testing" + "github.com/krateoplatformops/krateo-bff/internal/kubernetes/widgets/formtemplates" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" ) -func TestIteratorCount(t *testing.T) { +func TestEval(t *testing.T) { + cfg, err := newRestConfig() + if err != nil { + t.Fatal(err) + } + + cli, err := formtemplates.NewClient(cfg) + if err != nil { + t.Fatal(err) + } + + namespace := "demo-system" + name := "fireworksapp-with-api" + + res, err := cli.Namespace(namespace).Get(context.TODO(), name) + if err != nil { + t.Fatal(err) + } + + err = Eval(context.TODO(), res, EvalOptions{ + RESTConfig: cfg, + AuthnNS: namespace, + Subject: "", + }) + if err != nil { + t.Fatal(err) + } + //fin, _ := os.Create("ppp.json") + //defer fin.Close() + enc := json.NewEncoder(os.Stdout) + enc.SetIndent("", " ") + enc.Encode(res) } func newRestConfig() (*rest.Config, error) { diff --git a/internal/kubernetes/widgets/formtemplates/formtemplates_test.go b/internal/kubernetes/widgets/formtemplates/formtemplates_test.go index 5f84b03..91ff79b 100644 --- a/internal/kubernetes/widgets/formtemplates/formtemplates_test.go +++ b/internal/kubernetes/widgets/formtemplates/formtemplates_test.go @@ -11,7 +11,6 @@ import ( "testing" "github.com/krateoplatformops/krateo-bff/internal/kubernetes/widgets/formtemplates" - "github.com/krateoplatformops/krateo-bff/internal/kubernetes/widgets/formtemplates/evaluator" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" @@ -84,21 +83,11 @@ func TestFormTemplatePlain(t *testing.T) { t.Fatal(err) } - err = evaluator.Eval(context.TODO(), res, evaluator.EvalOptions{ - RESTConfig: cfg, - AuthnNS: namespace, - Subject: "", - }) - if err != nil { - t.Fatal(err) - } - - fin, _ := os.Create("ppp.json") - defer fin.Close() - enc := json.NewEncoder(fin) + //fin, _ := os.Create("ppp.json") + //defer fin.Close() + enc := json.NewEncoder(os.Stdout) enc.SetIndent("", " ") enc.Encode(res) - } func newRestConfig() (*rest.Config, error) { diff --git a/internal/kubernetes/widgets/formtemplates/merger/merger.go b/internal/kubernetes/widgets/formtemplates/merger/merger.go new file mode 100644 index 0000000..0ea608c --- /dev/null +++ b/internal/kubernetes/widgets/formtemplates/merger/merger.go @@ -0,0 +1,27 @@ +package merger + +import ( + "fmt" + "strings" + + formtemplatesv1alpha1 "github.com/krateoplatformops/krateo-bff/apis/ui/formtemplates/v1alpha1" + "github.com/krateoplatformops/krateo-bff/internal/strvals" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +func Merge(src *formtemplatesv1alpha1.FormTemplate, dst *unstructured.Unstructured) error { + lines := make([]string, len(src.Spec.Data)) + for i, di := range src.Spec.Data { + lines[i] = di.String() + } + + values := strings.Join(lines, ",") + fmt.Println(values) + + err := strvals.ParseInto(values, dst.UnstructuredContent()) + if err != nil { + return err + } + + return nil //unstructured.SetNestedMap(dst.Object, spec, "spec") +} diff --git a/internal/kubernetes/widgets/formtemplates/merger/merger_test.go b/internal/kubernetes/widgets/formtemplates/merger/merger_test.go new file mode 100644 index 0000000..8104beb --- /dev/null +++ b/internal/kubernetes/widgets/formtemplates/merger/merger_test.go @@ -0,0 +1,78 @@ +package merger + +import ( + "context" + "encoding/json" + "io" + "os" + "path/filepath" + "testing" + + "github.com/krateoplatformops/krateo-bff/internal/kubernetes/dynamic" + "github.com/krateoplatformops/krateo-bff/internal/kubernetes/widgets/formtemplates" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" +) + +func TestMerge(t *testing.T) { + ctx := context.TODO() + namespace := "demo-system" + name := "fireworksapp" + + cfg, err := newRestConfig() + if err != nil { + t.Fatal(err) + } + + dyn, err := dynamic.NewGetter(cfg) + if err != nil { + t.Fatal(err) + } + + dst, err := dyn.Get(ctx, dynamic.GetOptions{ + GVK: schema.GroupVersionKind{ + Group: "apps.krateo.io", + Version: "v1alpha1", + Kind: "FireworksappForm", + }, + Namespace: namespace, + Name: name, + }) + if err != nil { + t.Fatal(err) + } + + cli, err := formtemplates.NewClient(cfg) + if err != nil { + t.Fatal(err) + } + + src, err := cli.Namespace(namespace).Get(ctx, name) + if err != nil { + t.Fatal(err) + } + + if err := Merge(src, dst); err != nil { + t.Fatal(err) + } + + dump(os.Stdout, dst) +} + +func newRestConfig() (*rest.Config, error) { + home, err := os.UserHomeDir() + if err != nil { + return nil, err + } + + return clientcmd.BuildConfigFromFlags("", filepath.Join(home, ".kube", "config")) +} + +func dump(w io.Writer, v any) { + //fin, _ := os.Create("ppp.json") + //defer fin.Close() + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + enc.Encode(v) +} diff --git a/internal/strvals/literal_parser.go b/internal/strvals/literal_parser.go new file mode 100644 index 0000000..f756558 --- /dev/null +++ b/internal/strvals/literal_parser.go @@ -0,0 +1,244 @@ +/* +Copyright The Helm Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package strvals + +import ( + "bytes" + "fmt" + "io" + "strconv" + + "github.com/pkg/errors" +) + +// ParseLiteral parses a set line interpreting the value as a literal string. +// +// A set line is of the form name1=value1 +func ParseLiteral(s string) (map[string]interface{}, error) { + vals := map[string]interface{}{} + scanner := bytes.NewBufferString(s) + t := newLiteralParser(scanner, vals) + err := t.parse() + return vals, err +} + +// ParseLiteralInto parses a strvals line and merges the result into dest. +// The value is interpreted as a literal string. +// +// If the strval string has a key that exists in dest, it overwrites the +// dest version. +func ParseLiteralInto(s string, dest map[string]interface{}) error { + scanner := bytes.NewBufferString(s) + t := newLiteralParser(scanner, dest) + return t.parse() +} + +// literalParser is a simple parser that takes a strvals line and parses +// it into a map representation. +// +// Values are interpreted as a literal string. +// +// where sc is the source of the original data being parsed +// where data is the final parsed data from the parses with correct types +type literalParser struct { + sc *bytes.Buffer + data map[string]interface{} +} + +func newLiteralParser(sc *bytes.Buffer, data map[string]interface{}) *literalParser { + return &literalParser{sc: sc, data: data} +} + +func (t *literalParser) parse() error { + for { + err := t.key(t.data, 0) + if err == nil { + continue + } + if err == io.EOF { + return nil + } + return err + } +} + +func runesUntilLiteral(in io.RuneReader, stop map[rune]bool) ([]rune, rune, error) { + v := []rune{} + for { + switch r, _, e := in.ReadRune(); { + case e != nil: + return v, r, e + case inMap(r, stop): + return v, r, nil + default: + v = append(v, r) + } + } +} + +func (t *literalParser) key(data map[string]interface{}, nestedNameLevel int) (reterr error) { + defer func() { + if r := recover(); r != nil { + reterr = fmt.Errorf("unable to parse key: %s", r) + } + }() + stop := runeSet([]rune{'=', '[', '.'}) + for { + switch key, lastRune, err := runesUntilLiteral(t.sc, stop); { + case err != nil: + if len(key) == 0 { + return err + } + return errors.Errorf("key %q has no value", string(key)) + + case lastRune == '=': + // found end of key: swallow the '=' and get the value + value, err := t.val() + if err == nil && err != io.EOF { + return err + } + set(data, string(key), string(value)) + return nil + + case lastRune == '.': + // Check value name is within the maximum nested name level + nestedNameLevel++ + if nestedNameLevel > MaxNestedNameLevel { + return fmt.Errorf("value name nested level is greater than maximum supported nested level of %d", MaxNestedNameLevel) + } + + // first, create or find the target map in the given data + inner := map[string]interface{}{} + if _, ok := data[string(key)]; ok { + inner = data[string(key)].(map[string]interface{}) + } + + // recurse on sub-tree with remaining data + err := t.key(inner, nestedNameLevel) + if err == nil && len(inner) == 0 { + return errors.Errorf("key map %q has no value", string(key)) + } + if len(inner) != 0 { + set(data, string(key), inner) + } + return err + + case lastRune == '[': + // We are in a list index context, so we need to set an index. + i, err := t.keyIndex() + if err != nil { + return errors.Wrap(err, "error parsing index") + } + kk := string(key) + + // find or create target list + list := []interface{}{} + if _, ok := data[kk]; ok { + list = data[kk].([]interface{}) + } + + // now we need to get the value after the ] + list, err = t.listItem(list, i, nestedNameLevel) + set(data, kk, list) + return err + } + } +} + +func (t *literalParser) keyIndex() (int, error) { + // First, get the key. + stop := runeSet([]rune{']'}) + v, _, err := runesUntilLiteral(t.sc, stop) + if err != nil { + return 0, err + } + + // v should be the index + return strconv.Atoi(string(v)) +} + +func (t *literalParser) listItem(list []interface{}, i, nestedNameLevel int) ([]interface{}, error) { + if i < 0 { + return list, fmt.Errorf("negative %d index not allowed", i) + } + stop := runeSet([]rune{'[', '.', '='}) + + switch key, lastRune, err := runesUntilLiteral(t.sc, stop); { + case len(key) > 0: + return list, errors.Errorf("unexpected data at end of array index: %q", key) + + case err != nil: + return list, err + + case lastRune == '=': + value, err := t.val() + if err != nil && err != io.EOF { + return list, err + } + return setIndex(list, i, string(value)) + + case lastRune == '.': + // we have a nested object. Send to t.key + inner := map[string]interface{}{} + if len(list) > i { + var ok bool + inner, ok = list[i].(map[string]interface{}) + if !ok { + // We have indices out of order. Initialize empty value. + list[i] = map[string]interface{}{} + inner = list[i].(map[string]interface{}) + } + } + + // recurse + err := t.key(inner, nestedNameLevel) + if err != nil { + return list, err + } + return setIndex(list, i, inner) + + case lastRune == '[': + // now we have a nested list. Read the index and handle. + nextI, err := t.keyIndex() + if err != nil { + return list, errors.Wrap(err, "error parsing index") + } + var crtList []interface{} + if len(list) > i { + // If nested list already exists, take the value of list to next cycle. + existed := list[i] + if existed != nil { + crtList = list[i].([]interface{}) + } + } + + // Now we need to get the value after the ]. + list2, err := t.listItem(crtList, nextI, nestedNameLevel) + if err != nil { + return list, err + } + return setIndex(list, i, list2) + + default: + return nil, errors.Errorf("parse error: unexpected token %v", lastRune) + } +} + +func (t *literalParser) val() ([]rune, error) { + stop := runeSet([]rune{}) + v, _, err := runesUntilLiteral(t.sc, stop) + return v, err +} diff --git a/internal/strvals/literal_parser_test.go b/internal/strvals/literal_parser_test.go new file mode 100644 index 0000000..4e74423 --- /dev/null +++ b/internal/strvals/literal_parser_test.go @@ -0,0 +1,480 @@ +/* +Copyright The Helm Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package strvals + +import ( + "fmt" + "testing" + + "sigs.k8s.io/yaml" +) + +func TestParseLiteral(t *testing.T) { + cases := []struct { + str string + expect map[string]interface{} + err bool + }{ + { + str: "name", + err: true, + }, + { + str: "name=", + expect: map[string]interface{}{"name": ""}, + }, + { + str: "name=value", + expect: map[string]interface{}{"name": "value"}, + err: false, + }, + { + str: "long_int_string=1234567890", + expect: map[string]interface{}{"long_int_string": "1234567890"}, + err: false, + }, + { + str: "boolean=true", + expect: map[string]interface{}{"boolean": "true"}, + err: false, + }, + { + str: "is_null=null", + expect: map[string]interface{}{"is_null": "null"}, + err: false, + }, + { + str: "zero=0", + expect: map[string]interface{}{"zero": "0"}, + err: false, + }, + { + str: "name1=null,name2=value2", + expect: map[string]interface{}{"name1": "null,name2=value2"}, + err: false, + }, + { + str: "name1=value,,,tail", + expect: map[string]interface{}{"name1": "value,,,tail"}, + err: false, + }, + { + str: "leading_zeros=00009", + expect: map[string]interface{}{"leading_zeros": "00009"}, + err: false, + }, + { + str: "name=one two three", + expect: map[string]interface{}{"name": "one two three"}, + err: false, + }, + { + str: "outer.inner=value", + expect: map[string]interface{}{"outer": map[string]interface{}{"inner": "value"}}, + err: false, + }, + { + str: "outer.middle.inner=value", + expect: map[string]interface{}{"outer": map[string]interface{}{"middle": map[string]interface{}{"inner": "value"}}}, + err: false, + }, + { + str: "name1.name2", + err: true, + }, + { + str: "name1.name2=", + expect: map[string]interface{}{"name1": map[string]interface{}{"name2": ""}}, + err: false, + }, + { + str: "name1.=name2", + err: true, + }, + { + str: "name1.,name2", + err: true, + }, + { + str: "name1={value1,value2}", + expect: map[string]interface{}{"name1": "{value1,value2}"}, + }, + + // List support + { + str: "list[0]=foo", + expect: map[string]interface{}{"list": []string{"foo"}}, + err: false, + }, + { + str: "list[0].foo=bar", + expect: map[string]interface{}{ + "list": []interface{}{ + map[string]interface{}{"foo": "bar"}, + }, + }, + err: false, + }, + { + str: "list[-30].hello=world", + err: true, + }, + { + str: "list[3]=bar", + expect: map[string]interface{}{"list": []interface{}{nil, nil, nil, "bar"}}, + err: false, + }, + { + str: "illegal[0]name.foo=bar", + err: true, + }, + { + str: "noval[0]", + expect: map[string]interface{}{"noval": []interface{}{}}, + err: false, + }, + { + str: "noval[0]=", + expect: map[string]interface{}{"noval": []interface{}{""}}, + err: false, + }, + { + str: "nested[0][0]=1", + expect: map[string]interface{}{"nested": []interface{}{[]interface{}{"1"}}}, + err: false, + }, + { + str: "nested[1][1]=1", + expect: map[string]interface{}{"nested": []interface{}{nil, []interface{}{nil, "1"}}}, + err: false, + }, + { + str: "name1.name2[0].foo=bar", + expect: map[string]interface{}{ + "name1": map[string]interface{}{ + "name2": []map[string]interface{}{{"foo": "bar"}}, + }, + }, + }, + { + str: "name1.name2[1].foo=bar", + expect: map[string]interface{}{ + "name1": map[string]interface{}{ + "name2": []map[string]interface{}{nil, {"foo": "bar"}}, + }, + }, + }, + { + str: "name1.name2[1].foo=bar", + expect: map[string]interface{}{ + "name1": map[string]interface{}{ + "name2": []map[string]interface{}{nil, {"foo": "bar"}}, + }, + }, + }, + { + str: "]={}].", + expect: map[string]interface{}{"]": "{}]."}, + err: false, + }, + + // issue test cases: , = $ ( ) { } . \ \\ + { + str: "name=val,val", + expect: map[string]interface{}{"name": "val,val"}, + err: false, + }, + { + str: "name=val.val", + expect: map[string]interface{}{"name": "val.val"}, + err: false, + }, + { + str: "name=val=val", + expect: map[string]interface{}{"name": "val=val"}, + err: false, + }, + { + str: "name=val$val", + expect: map[string]interface{}{"name": "val$val"}, + err: false, + }, + { + str: "name=(value", + expect: map[string]interface{}{"name": "(value"}, + err: false, + }, + { + str: "name=value)", + expect: map[string]interface{}{"name": "value)"}, + err: false, + }, + { + str: "name=(value)", + expect: map[string]interface{}{"name": "(value)"}, + err: false, + }, + { + str: "name={value", + expect: map[string]interface{}{"name": "{value"}, + err: false, + }, + { + str: "name=value}", + expect: map[string]interface{}{"name": "value}"}, + err: false, + }, + { + str: "name={value}", + expect: map[string]interface{}{"name": "{value}"}, + err: false, + }, + { + str: "name={value1,value2}", + expect: map[string]interface{}{"name": "{value1,value2}"}, + err: false, + }, + { + str: `name=val\val`, + expect: map[string]interface{}{"name": `val\val`}, + err: false, + }, + { + str: `name=val\\val`, + expect: map[string]interface{}{"name": `val\\val`}, + err: false, + }, + { + str: `name=val\\\val`, + expect: map[string]interface{}{"name": `val\\\val`}, + err: false, + }, + { + str: `name={val,.?*v\0a!l)some`, + expect: map[string]interface{}{"name": `{val,.?*v\0a!l)some`}, + err: false, + }, + { + str: `name=em%GT)tqUDqz,i-\h+Mbqs-!:.m\\rE=mkbM#rR}@{-k@`, + expect: map[string]interface{}{"name": `em%GT)tqUDqz,i-\h+Mbqs-!:.m\\rE=mkbM#rR}@{-k@`}, + }, + } + + for _, tt := range cases { + got, err := ParseLiteral(tt.str) + if err != nil { + if !tt.err { + t.Fatalf("%s: %s", tt.str, err) + } + continue + } + + if tt.err { + t.Errorf("%s: Expected error. Got nil", tt.str) + } + + y1, err := yaml.Marshal(tt.expect) + if err != nil { + t.Fatal(err) + } + + y2, err := yaml.Marshal(got) + if err != nil { + t.Fatalf("Error serializing parsed value: %s", err) + } + + if string(y1) != string(y2) { + t.Errorf("%s: Expected:\n%s\nGot:\n%s", tt.str, y1, y2) + } + } +} + +func TestParseLiteralInto(t *testing.T) { + tests := []struct { + input string + input2 string + got map[string]interface{} + expect map[string]interface{} + err bool + }{ + { + input: "outer.inner1=value1,outer.inner3=value3,outer.inner4=4", + got: map[string]interface{}{ + "outer": map[string]interface{}{ + "inner1": "overwrite", + "inner2": "value2", + }, + }, + expect: map[string]interface{}{ + "outer": map[string]interface{}{ + "inner1": "value1,outer.inner3=value3,outer.inner4=4", + "inner2": "value2", + }}, + err: false, + }, + { + input: "listOuter[0][0].type=listValue", + input2: "listOuter[0][0].status=alive", + got: map[string]interface{}{}, + expect: map[string]interface{}{ + "listOuter": [][]interface{}{{map[string]string{ + "type": "listValue", + "status": "alive", + }}}, + }, + err: false, + }, + { + input: "listOuter[0][0].type=listValue", + input2: "listOuter[1][0].status=alive", + got: map[string]interface{}{}, + expect: map[string]interface{}{ + "listOuter": [][]interface{}{ + { + map[string]string{"type": "listValue"}, + }, + { + map[string]string{"status": "alive"}, + }, + }, + }, + err: false, + }, + { + input: "listOuter[0][1][0].type=listValue", + input2: "listOuter[0][0][1].status=alive", + got: map[string]interface{}{ + "listOuter": []interface{}{ + []interface{}{ + []interface{}{ + map[string]string{"exited": "old"}, + }, + }, + }, + }, + expect: map[string]interface{}{ + "listOuter": [][][]interface{}{ + { + { + map[string]string{"exited": "old"}, + map[string]string{"status": "alive"}, + }, + { + map[string]string{"type": "listValue"}, + }, + }, + }, + }, + err: false, + }, + } + + for _, tt := range tests { + if err := ParseLiteralInto(tt.input, tt.got); err != nil { + t.Fatal(err) + } + if tt.err { + t.Errorf("%s: Expected error. Got nil", tt.input) + } + + if tt.input2 != "" { + if err := ParseLiteralInto(tt.input2, tt.got); err != nil { + t.Fatal(err) + } + if tt.err { + t.Errorf("%s: Expected error. Got nil", tt.input2) + } + } + + y1, err := yaml.Marshal(tt.expect) + if err != nil { + t.Fatal(err) + } + + y2, err := yaml.Marshal(tt.got) + if err != nil { + t.Fatalf("Error serializing parsed value: %s", err) + } + + if string(y1) != string(y2) { + t.Errorf("%s: Expected:\n%s\nGot:\n%s", tt.input, y1, y2) + } + } +} + +func TestParseLiteralNestedLevels(t *testing.T) { + var keyMultipleNestedLevels string + + for i := 1; i <= MaxNestedNameLevel+2; i++ { + tmpStr := fmt.Sprintf("name%d", i) + if i <= MaxNestedNameLevel+1 { + tmpStr = tmpStr + "." + } + keyMultipleNestedLevels += tmpStr + } + + tests := []struct { + str string + expect map[string]interface{} + err bool + errStr string + }{ + { + "outer.middle.inner=value", + map[string]interface{}{"outer": map[string]interface{}{"middle": map[string]interface{}{"inner": "value"}}}, + false, + "", + }, + { + str: keyMultipleNestedLevels + "=value", + err: true, + errStr: fmt.Sprintf("value name nested level is greater than maximum supported nested level of %d", MaxNestedNameLevel), + }, + } + + for _, tt := range tests { + got, err := ParseLiteral(tt.str) + if err != nil { + if tt.err { + if tt.errStr != "" { + if err.Error() != tt.errStr { + t.Errorf("Expected error: %s. Got error: %s", tt.errStr, err.Error()) + } + } + continue + } + t.Fatalf("%s: %s", tt.str, err) + } + + if tt.err { + t.Errorf("%s: Expected error. Got nil", tt.str) + } + + y1, err := yaml.Marshal(tt.expect) + if err != nil { + t.Fatal(err) + } + + y2, err := yaml.Marshal(got) + if err != nil { + t.Fatalf("Error serializing parsed value: %s", err) + } + + if string(y1) != string(y2) { + t.Errorf("%s: Expected:\n%s\nGot:\n%s", tt.str, y1, y2) + } + } +} diff --git a/internal/strvals/parser.go b/internal/strvals/parser.go new file mode 100644 index 0000000..2828f20 --- /dev/null +++ b/internal/strvals/parser.go @@ -0,0 +1,559 @@ +/* +Copyright The Helm Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package strvals + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "strconv" + "strings" + "unicode" + + "github.com/pkg/errors" + "sigs.k8s.io/yaml" +) + +// ErrNotList indicates that a non-list was treated as a list. +var ErrNotList = errors.New("not a list") + +// MaxIndex is the maximum index that will be allowed by setIndex. +// The default value 65536 = 1024 * 64 +var MaxIndex = 65536 + +// MaxNestedNameLevel is the maximum level of nesting for a value name that +// will be allowed. +var MaxNestedNameLevel = 30 + +// ToYAML takes a string of arguments and converts to a YAML document. +func ToYAML(s string) (string, error) { + m, err := Parse(s) + if err != nil { + return "", err + } + d, err := yaml.Marshal(m) + return strings.TrimSuffix(string(d), "\n"), err +} + +// Parse parses a set line. +// +// A set line is of the form name1=value1,name2=value2 +func Parse(s string) (map[string]interface{}, error) { + vals := map[string]interface{}{} + scanner := bytes.NewBufferString(s) + t := newParser(scanner, vals, false) + err := t.parse() + return vals, err +} + +// ParseString parses a set line and forces a string value. +// +// A set line is of the form name1=value1,name2=value2 +func ParseString(s string) (map[string]interface{}, error) { + vals := map[string]interface{}{} + scanner := bytes.NewBufferString(s) + t := newParser(scanner, vals, true) + err := t.parse() + return vals, err +} + +// ParseInto parses a strvals line and merges the result into dest. +// +// If the strval string has a key that exists in dest, it overwrites the +// dest version. +func ParseInto(s string, dest map[string]interface{}) error { + scanner := bytes.NewBufferString(s) + t := newParser(scanner, dest, false) + return t.parse() +} + +// ParseFile parses a set line, but its final value is loaded from the file at the path specified by the original value. +// +// A set line is of the form name1=path1,name2=path2 +// +// When the files at path1 and path2 contained "val1" and "val2" respectively, the set line is consumed as +// name1=val1,name2=val2 +func ParseFile(s string, reader RunesValueReader) (map[string]interface{}, error) { + vals := map[string]interface{}{} + scanner := bytes.NewBufferString(s) + t := newFileParser(scanner, vals, reader) + err := t.parse() + return vals, err +} + +// ParseIntoString parses a strvals line and merges the result into dest. +// +// This method always returns a string as the value. +func ParseIntoString(s string, dest map[string]interface{}) error { + scanner := bytes.NewBufferString(s) + t := newParser(scanner, dest, true) + return t.parse() +} + +// ParseJSON parses a string with format key1=val1, key2=val2, ... +// where values are json strings (null, or scalars, or arrays, or objects). +// An empty val is treated as null. +// +// If a key exists in dest, the new value overwrites the dest version. +func ParseJSON(s string, dest map[string]interface{}) error { + scanner := bytes.NewBufferString(s) + t := newJSONParser(scanner, dest) + return t.parse() +} + +// ParseIntoFile parses a filevals line and merges the result into dest. +// +// This method always returns a string as the value. +func ParseIntoFile(s string, dest map[string]interface{}, reader RunesValueReader) error { + scanner := bytes.NewBufferString(s) + t := newFileParser(scanner, dest, reader) + return t.parse() +} + +// RunesValueReader is a function that takes the given value (a slice of runes) +// and returns the parsed value +type RunesValueReader func([]rune) (interface{}, error) + +// parser is a simple parser that takes a strvals line and parses it into a +// map representation. +// +// where sc is the source of the original data being parsed +// where data is the final parsed data from the parses with correct types +type parser struct { + sc *bytes.Buffer + data map[string]interface{} + reader RunesValueReader + isjsonval bool +} + +func newParser(sc *bytes.Buffer, data map[string]interface{}, stringBool bool) *parser { + stringConverter := func(rs []rune) (interface{}, error) { + return typedVal(rs, stringBool), nil + } + return &parser{sc: sc, data: data, reader: stringConverter} +} + +func newJSONParser(sc *bytes.Buffer, data map[string]interface{}) *parser { + return &parser{sc: sc, data: data, reader: nil, isjsonval: true} +} + +func newFileParser(sc *bytes.Buffer, data map[string]interface{}, reader RunesValueReader) *parser { + return &parser{sc: sc, data: data, reader: reader} +} + +func (t *parser) parse() error { + for { + err := t.key(t.data, 0) + if err == nil { + continue + } + if err == io.EOF { + return nil + } + return err + } +} + +func runeSet(r []rune) map[rune]bool { + s := make(map[rune]bool, len(r)) + for _, rr := range r { + s[rr] = true + } + return s +} + +func (t *parser) key(data map[string]interface{}, nestedNameLevel int) (reterr error) { + defer func() { + if r := recover(); r != nil { + reterr = fmt.Errorf("unable to parse key: %s", r) + } + }() + stop := runeSet([]rune{'=', '[', ',', '.'}) + for { + switch k, last, err := runesUntil(t.sc, stop); { + case err != nil: + if len(k) == 0 { + return err + } + return errors.Errorf("key %q has no value", string(k)) + //set(data, string(k), "") + //return err + case last == '[': + // We are in a list index context, so we need to set an index. + i, err := t.keyIndex() + if err != nil { + return errors.Wrap(err, "error parsing index") + } + kk := string(k) + // Find or create target list + list := []interface{}{} + if _, ok := data[kk]; ok { + list = data[kk].([]interface{}) + } + + // Now we need to get the value after the ]. + list, err = t.listItem(list, i, nestedNameLevel) + set(data, kk, list) + return err + case last == '=': + if t.isjsonval { + empval, err := t.emptyVal() + if err != nil { + return err + } + if empval { + set(data, string(k), nil) + return nil + } + // parse jsonvals by using Go’s JSON standard library + // Decode is preferred to Unmarshal in order to parse just the json parts of the list key1=jsonval1,key2=jsonval2,... + // Since Decode has its own buffer that consumes more characters (from underlying t.sc) than the ones actually decoded, + // we invoke Decode on a separate reader built with a copy of what is left in t.sc. After Decode is executed, we + // discard in t.sc the chars of the decoded json value (the number of those characters is returned by InputOffset). + var jsonval interface{} + dec := json.NewDecoder(strings.NewReader(t.sc.String())) + if err = dec.Decode(&jsonval); err != nil { + return err + } + set(data, string(k), jsonval) + if _, err = io.CopyN(io.Discard, t.sc, dec.InputOffset()); err != nil { + return err + } + // skip possible blanks and comma + _, err = t.emptyVal() + return err + } + //End of key. Consume =, Get value. + // FIXME: Get value list first + vl, e := t.valList() + switch e { + case nil: + set(data, string(k), vl) + return nil + case io.EOF: + set(data, string(k), "") + return e + case ErrNotList: + rs, e := t.val() + if e != nil && e != io.EOF { + return e + } + v, e := t.reader(rs) + set(data, string(k), v) + return e + default: + return e + } + case last == ',': + // No value given. Set the value to empty string. Return error. + set(data, string(k), "") + return errors.Errorf("key %q has no value (cannot end with ,)", string(k)) + case last == '.': + // Check value name is within the maximum nested name level + nestedNameLevel++ + if nestedNameLevel > MaxNestedNameLevel { + return fmt.Errorf("value name nested level is greater than maximum supported nested level of %d", MaxNestedNameLevel) + } + + // First, create or find the target map. + inner := map[string]interface{}{} + if _, ok := data[string(k)]; ok { + inner = data[string(k)].(map[string]interface{}) + } + + // Recurse + e := t.key(inner, nestedNameLevel) + if e == nil && len(inner) == 0 { + return errors.Errorf("key map %q has no value", string(k)) + } + if len(inner) != 0 { + set(data, string(k), inner) + } + return e + } + } +} + +func set(data map[string]interface{}, key string, val interface{}) { + // If key is empty, don't set it. + if len(key) == 0 { + return + } + data[key] = val +} + +func setIndex(list []interface{}, index int, val interface{}) (l2 []interface{}, err error) { + // There are possible index values that are out of range on a target system + // causing a panic. This will catch the panic and return an error instead. + // The value of the index that causes a panic varies from system to system. + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("error processing index %d: %s", index, r) + } + }() + + if index < 0 { + return list, fmt.Errorf("negative %d index not allowed", index) + } + if index > MaxIndex { + return list, fmt.Errorf("index of %d is greater than maximum supported index of %d", index, MaxIndex) + } + if len(list) <= index { + newlist := make([]interface{}, index+1) + copy(newlist, list) + list = newlist + } + list[index] = val + return list, nil +} + +func (t *parser) keyIndex() (int, error) { + // First, get the key. + stop := runeSet([]rune{']'}) + v, _, err := runesUntil(t.sc, stop) + if err != nil { + return 0, err + } + // v should be the index + return strconv.Atoi(string(v)) + +} +func (t *parser) listItem(list []interface{}, i, nestedNameLevel int) ([]interface{}, error) { + if i < 0 { + return list, fmt.Errorf("negative %d index not allowed", i) + } + stop := runeSet([]rune{'[', '.', '='}) + switch k, last, err := runesUntil(t.sc, stop); { + case len(k) > 0: + return list, errors.Errorf("unexpected data at end of array index: %q", k) + case err != nil: + return list, err + case last == '=': + if t.isjsonval { + empval, err := t.emptyVal() + if err != nil { + return list, err + } + if empval { + return setIndex(list, i, nil) + } + // parse jsonvals by using Go’s JSON standard library + // Decode is preferred to Unmarshal in order to parse just the json parts of the list key1=jsonval1,key2=jsonval2,... + // Since Decode has its own buffer that consumes more characters (from underlying t.sc) than the ones actually decoded, + // we invoke Decode on a separate reader built with a copy of what is left in t.sc. After Decode is executed, we + // discard in t.sc the chars of the decoded json value (the number of those characters is returned by InputOffset). + var jsonval interface{} + dec := json.NewDecoder(strings.NewReader(t.sc.String())) + if err = dec.Decode(&jsonval); err != nil { + return list, err + } + if list, err = setIndex(list, i, jsonval); err != nil { + return list, err + } + if _, err = io.CopyN(io.Discard, t.sc, dec.InputOffset()); err != nil { + return list, err + } + // skip possible blanks and comma + _, err = t.emptyVal() + return list, err + } + vl, e := t.valList() + switch e { + case nil: + return setIndex(list, i, vl) + case io.EOF: + return setIndex(list, i, "") + case ErrNotList: + rs, e := t.val() + if e != nil && e != io.EOF { + return list, e + } + v, e := t.reader(rs) + if e != nil { + return list, e + } + return setIndex(list, i, v) + default: + return list, e + } + case last == '[': + // now we have a nested list. Read the index and handle. + nextI, err := t.keyIndex() + if err != nil { + return list, errors.Wrap(err, "error parsing index") + } + var crtList []interface{} + if len(list) > i { + // If nested list already exists, take the value of list to next cycle. + existed := list[i] + if existed != nil { + crtList = list[i].([]interface{}) + } + } + // Now we need to get the value after the ]. + list2, err := t.listItem(crtList, nextI, nestedNameLevel) + if err != nil { + return list, err + } + return setIndex(list, i, list2) + case last == '.': + // We have a nested object. Send to t.key + inner := map[string]interface{}{} + if len(list) > i { + var ok bool + inner, ok = list[i].(map[string]interface{}) + if !ok { + // We have indices out of order. Initialize empty value. + list[i] = map[string]interface{}{} + inner = list[i].(map[string]interface{}) + } + } + + // Recurse + e := t.key(inner, nestedNameLevel) + if e != nil { + return list, e + } + return setIndex(list, i, inner) + default: + return nil, errors.Errorf("parse error: unexpected token %v", last) + } +} + +// check for an empty value +// read and consume optional spaces until comma or EOF (empty val) or any other char (not empty val) +// comma and spaces are consumed, while any other char is not cosumed +func (t *parser) emptyVal() (bool, error) { + for { + r, _, e := t.sc.ReadRune() + if e == io.EOF { + return true, nil + } + if e != nil { + return false, e + } + if r == ',' { + return true, nil + } + if !unicode.IsSpace(r) { + t.sc.UnreadRune() + return false, nil + } + } +} + +func (t *parser) val() ([]rune, error) { + stop := runeSet([]rune{','}) + v, _, err := runesUntil(t.sc, stop) + return v, err +} + +func (t *parser) valList() ([]interface{}, error) { + r, _, e := t.sc.ReadRune() + if e != nil { + return []interface{}{}, e + } + + if r != '{' { + t.sc.UnreadRune() + return []interface{}{}, ErrNotList + } + + list := []interface{}{} + stop := runeSet([]rune{',', '}'}) + for { + switch rs, last, err := runesUntil(t.sc, stop); { + case err != nil: + if err == io.EOF { + err = errors.New("list must terminate with '}'") + } + return list, err + case last == '}': + // If this is followed by ',', consume it. + if r, _, e := t.sc.ReadRune(); e == nil && r != ',' { + t.sc.UnreadRune() + } + v, e := t.reader(rs) + list = append(list, v) + return list, e + case last == ',': + v, e := t.reader(rs) + if e != nil { + return list, e + } + list = append(list, v) + } + } +} + +func runesUntil(in io.RuneReader, stop map[rune]bool) ([]rune, rune, error) { + v := []rune{} + for { + switch r, _, e := in.ReadRune(); { + case e != nil: + return v, r, e + case inMap(r, stop): + return v, r, nil + case r == '\\': + next, _, e := in.ReadRune() + if e != nil { + return v, next, e + } + v = append(v, next) + default: + v = append(v, r) + } + } +} + +func inMap(k rune, m map[rune]bool) bool { + _, ok := m[k] + return ok +} + +func typedVal(v []rune, st bool) interface{} { + val := string(v) + + if st { + return val + } + + if strings.EqualFold(val, "true") { + return true + } + + if strings.EqualFold(val, "false") { + return false + } + + if strings.EqualFold(val, "null") { + return nil + } + + if strings.EqualFold(val, "0") { + return int64(0) + } + + // If this value does not start with zero, try parsing it to an int + if len(val) != 0 && val[0] != '0' { + if iv, err := strconv.ParseInt(val, 10, 64); err == nil { + return iv + } + } + + return val +} diff --git a/internal/strvals/parser_test.go b/internal/strvals/parser_test.go new file mode 100644 index 0000000..925aa97 --- /dev/null +++ b/internal/strvals/parser_test.go @@ -0,0 +1,818 @@ +/* +Copyright The Helm Authors. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package strvals + +import ( + "fmt" + "testing" + + "sigs.k8s.io/yaml" +) + +func TestSetIndex(t *testing.T) { + tests := []struct { + name string + initial []interface{} + expect []interface{} + add int + val int + err bool + }{ + { + name: "short", + initial: []interface{}{0, 1}, + expect: []interface{}{0, 1, 2}, + add: 2, + val: 2, + err: false, + }, + { + name: "equal", + initial: []interface{}{0, 1}, + expect: []interface{}{0, 2}, + add: 1, + val: 2, + err: false, + }, + { + name: "long", + initial: []interface{}{0, 1, 2, 3, 4, 5}, + expect: []interface{}{0, 1, 2, 4, 4, 5}, + add: 3, + val: 4, + err: false, + }, + { + name: "negative", + initial: []interface{}{0, 1, 2, 3, 4, 5}, + expect: []interface{}{0, 1, 2, 3, 4, 5}, + add: -1, + val: 4, + err: true, + }, + { + name: "large", + initial: []interface{}{0, 1, 2, 3, 4, 5}, + expect: []interface{}{0, 1, 2, 3, 4, 5}, + add: MaxIndex + 1, + val: 4, + err: true, + }, + } + + for _, tt := range tests { + got, err := setIndex(tt.initial, tt.add, tt.val) + + if err != nil && tt.err == false { + t.Fatalf("%s: Expected no error but error returned", tt.name) + } else if err == nil && tt.err == true { + t.Fatalf("%s: Expected error but no error returned", tt.name) + } + + if len(got) != len(tt.expect) { + t.Fatalf("%s: Expected length %d, got %d", tt.name, len(tt.expect), len(got)) + } + + if !tt.err { + if gg := got[tt.add].(int); gg != tt.val { + t.Errorf("%s, Expected value %d, got %d", tt.name, tt.val, gg) + } + } + + for k, v := range got { + if v != tt.expect[k] { + t.Errorf("%s, Expected value %d, got %d", tt.name, tt.expect[k], v) + } + } + } +} + +func TestParseSet(t *testing.T) { + testsString := []struct { + str string + expect map[string]interface{} + err bool + }{ + { + str: "long_int_string=1234567890", + expect: map[string]interface{}{"long_int_string": "1234567890"}, + err: false, + }, + { + str: "boolean=true", + expect: map[string]interface{}{"boolean": "true"}, + err: false, + }, + { + str: "is_null=null", + expect: map[string]interface{}{"is_null": "null"}, + err: false, + }, + { + str: "zero=0", + expect: map[string]interface{}{"zero": "0"}, + err: false, + }, + } + tests := []struct { + str string + expect map[string]interface{} + err bool + }{ + { + "name1=null,f=false,t=true", + map[string]interface{}{"name1": nil, "f": false, "t": true}, + false, + }, + { + "name1=value1", + map[string]interface{}{"name1": "value1"}, + false, + }, + { + "name1=value1,name2=value2", + map[string]interface{}{"name1": "value1", "name2": "value2"}, + false, + }, + { + "name1=value1,name2=value2,", + map[string]interface{}{"name1": "value1", "name2": "value2"}, + false, + }, + { + str: "name1=value1,,,,name2=value2,", + err: true, + }, + { + str: "name1=,name2=value2", + expect: map[string]interface{}{"name1": "", "name2": "value2"}, + }, + { + str: "leading_zeros=00009", + expect: map[string]interface{}{"leading_zeros": "00009"}, + }, + { + str: "zero_int=0", + expect: map[string]interface{}{"zero_int": 0}, + }, + { + str: "long_int=1234567890", + expect: map[string]interface{}{"long_int": 1234567890}, + }, + { + str: "boolean=true", + expect: map[string]interface{}{"boolean": true}, + }, + { + str: "is_null=null", + expect: map[string]interface{}{"is_null": nil}, + err: false, + }, + { + str: "name1,name2=", + err: true, + }, + { + str: "name1,name2=value2", + err: true, + }, + { + str: "name1,name2=value2\\", + err: true, + }, + { + str: "name1,name2", + err: true, + }, + { + "name1=one\\,two,name2=three\\,four", + map[string]interface{}{"name1": "one,two", "name2": "three,four"}, + false, + }, + { + "name1=one\\=two,name2=three\\=four", + map[string]interface{}{"name1": "one=two", "name2": "three=four"}, + false, + }, + { + "name1=one two three,name2=three two one", + map[string]interface{}{"name1": "one two three", "name2": "three two one"}, + false, + }, + { + "outer.inner=value", + map[string]interface{}{"outer": map[string]interface{}{"inner": "value"}}, + false, + }, + { + "outer.middle.inner=value", + map[string]interface{}{"outer": map[string]interface{}{"middle": map[string]interface{}{"inner": "value"}}}, + false, + }, + { + "outer.inner1=value,outer.inner2=value2", + map[string]interface{}{"outer": map[string]interface{}{"inner1": "value", "inner2": "value2"}}, + false, + }, + { + "outer.inner1=value,outer.middle.inner=value", + map[string]interface{}{ + "outer": map[string]interface{}{ + "inner1": "value", + "middle": map[string]interface{}{ + "inner": "value", + }, + }, + }, + false, + }, + { + str: "name1.name2", + err: true, + }, + { + str: "name1.name2,name1.name3", + err: true, + }, + { + str: "name1.name2=", + expect: map[string]interface{}{"name1": map[string]interface{}{"name2": ""}}, + }, + { + str: "name1.=name2", + err: true, + }, + { + str: "name1.,name2", + err: true, + }, + { + "name1={value1,value2}", + map[string]interface{}{"name1": []string{"value1", "value2"}}, + false, + }, + { + "name1={value1,value2},name2={value1,value2}", + map[string]interface{}{ + "name1": []string{"value1", "value2"}, + "name2": []string{"value1", "value2"}, + }, + false, + }, + { + "name1={1021,902}", + map[string]interface{}{"name1": []int{1021, 902}}, + false, + }, + { + "name1.name2={value1,value2}", + map[string]interface{}{"name1": map[string]interface{}{"name2": []string{"value1", "value2"}}}, + false, + }, + { + str: "name1={1021,902", + err: true, + }, + // List support + { + str: "list[0]=foo", + expect: map[string]interface{}{"list": []string{"foo"}}, + }, + { + str: "list[0].foo=bar", + expect: map[string]interface{}{ + "list": []interface{}{ + map[string]interface{}{"foo": "bar"}, + }, + }, + }, + { + str: "list[0].foo=bar,list[0].hello=world", + expect: map[string]interface{}{ + "list": []interface{}{ + map[string]interface{}{"foo": "bar", "hello": "world"}, + }, + }, + }, + { + str: "list[0].foo=bar,list[-30].hello=world", + err: true, + }, + { + str: "list[0]=foo,list[1]=bar", + expect: map[string]interface{}{"list": []string{"foo", "bar"}}, + }, + { + str: "list[0]=foo,list[1]=bar,", + expect: map[string]interface{}{"list": []string{"foo", "bar"}}, + }, + { + str: "list[0]=foo,list[3]=bar", + expect: map[string]interface{}{"list": []interface{}{"foo", nil, nil, "bar"}}, + }, + { + str: "list[0]=foo,list[-20]=bar", + err: true, + }, + { + str: "illegal[0]name.foo=bar", + err: true, + }, + { + str: "noval[0]", + expect: map[string]interface{}{"noval": []interface{}{}}, + }, + { + str: "noval[0]=", + expect: map[string]interface{}{"noval": []interface{}{""}}, + }, + { + str: "nested[0][0]=1", + expect: map[string]interface{}{"nested": []interface{}{[]interface{}{1}}}, + }, + { + str: "nested[1][1]=1", + expect: map[string]interface{}{"nested": []interface{}{nil, []interface{}{nil, 1}}}, + }, + { + str: "name1.name2[0].foo=bar,name1.name2[1].foo=bar", + expect: map[string]interface{}{ + "name1": map[string]interface{}{ + "name2": []map[string]interface{}{{"foo": "bar"}, {"foo": "bar"}}, + }, + }, + }, + { + str: "name1.name2[1].foo=bar,name1.name2[0].foo=bar", + expect: map[string]interface{}{ + "name1": map[string]interface{}{ + "name2": []map[string]interface{}{{"foo": "bar"}, {"foo": "bar"}}, + }, + }, + }, + { + str: "name1.name2[1].foo=bar", + expect: map[string]interface{}{ + "name1": map[string]interface{}{ + "name2": []map[string]interface{}{nil, {"foo": "bar"}}, + }, + }, + }, + { + str: "]={}].", + err: true, + }, + } + + for _, tt := range tests { + got, err := Parse(tt.str) + if err != nil { + if tt.err { + continue + } + t.Fatalf("%s: %s", tt.str, err) + } + if tt.err { + t.Errorf("%s: Expected error. Got nil", tt.str) + } + + y1, err := yaml.Marshal(tt.expect) + if err != nil { + t.Fatal(err) + } + y2, err := yaml.Marshal(got) + if err != nil { + t.Fatalf("Error serializing parsed value: %s", err) + } + + if string(y1) != string(y2) { + t.Errorf("%s: Expected:\n%s\nGot:\n%s", tt.str, y1, y2) + } + } + for _, tt := range testsString { + got, err := ParseString(tt.str) + if err != nil { + if tt.err { + continue + } + t.Fatalf("%s: %s", tt.str, err) + } + if tt.err { + t.Errorf("%s: Expected error. Got nil", tt.str) + } + + y1, err := yaml.Marshal(tt.expect) + if err != nil { + t.Fatal(err) + } + y2, err := yaml.Marshal(got) + if err != nil { + t.Fatalf("Error serializing parsed value: %s", err) + } + + if string(y1) != string(y2) { + t.Errorf("%s: Expected:\n%s\nGot:\n%s", tt.str, y1, y2) + } + } +} + +func TestParseInto(t *testing.T) { + tests := []struct { + input string + input2 string + got map[string]interface{} + expect map[string]interface{} + err bool + }{ + { + input: "outer.inner1=value1,outer.inner3=value3,outer.inner4=4", + got: map[string]interface{}{ + "outer": map[string]interface{}{ + "inner1": "overwrite", + "inner2": "value2", + }, + }, + expect: map[string]interface{}{ + "outer": map[string]interface{}{ + "inner1": "value1", + "inner2": "value2", + "inner3": "value3", + "inner4": 4, + }}, + err: false, + }, + { + input: "listOuter[0][0].type=listValue", + input2: "listOuter[0][0].status=alive", + got: map[string]interface{}{}, + expect: map[string]interface{}{ + "listOuter": [][]interface{}{{map[string]string{ + "type": "listValue", + "status": "alive", + }}}, + }, + err: false, + }, + { + input: "listOuter[0][0].type=listValue", + input2: "listOuter[1][0].status=alive", + got: map[string]interface{}{}, + expect: map[string]interface{}{ + "listOuter": [][]interface{}{ + { + map[string]string{"type": "listValue"}, + }, + { + map[string]string{"status": "alive"}, + }, + }, + }, + err: false, + }, + { + input: "listOuter[0][1][0].type=listValue", + input2: "listOuter[0][0][1].status=alive", + got: map[string]interface{}{ + "listOuter": []interface{}{ + []interface{}{ + []interface{}{ + map[string]string{"exited": "old"}, + }, + }, + }, + }, + expect: map[string]interface{}{ + "listOuter": [][][]interface{}{ + { + { + map[string]string{"exited": "old"}, + map[string]string{"status": "alive"}, + }, + { + map[string]string{"type": "listValue"}, + }, + }, + }, + }, + err: false, + }, + } + for _, tt := range tests { + if err := ParseInto(tt.input, tt.got); err != nil { + t.Fatal(err) + } + if tt.err { + t.Errorf("%s: Expected error. Got nil", tt.input) + } + + if tt.input2 != "" { + if err := ParseInto(tt.input2, tt.got); err != nil { + t.Fatal(err) + } + if tt.err { + t.Errorf("%s: Expected error. Got nil", tt.input2) + } + } + + y1, err := yaml.Marshal(tt.expect) + if err != nil { + t.Fatal(err) + } + y2, err := yaml.Marshal(tt.got) + if err != nil { + t.Fatalf("Error serializing parsed value: %s", err) + } + + if string(y1) != string(y2) { + t.Errorf("%s: Expected:\n%s\nGot:\n%s", tt.input, y1, y2) + } + } +} + +func TestParseIntoString(t *testing.T) { + got := map[string]interface{}{ + "outer": map[string]interface{}{ + "inner1": "overwrite", + "inner2": "value2", + }, + } + input := "outer.inner1=1,outer.inner3=3" + expect := map[string]interface{}{ + "outer": map[string]interface{}{ + "inner1": "1", + "inner2": "value2", + "inner3": "3", + }, + } + + if err := ParseIntoString(input, got); err != nil { + t.Fatal(err) + } + + y1, err := yaml.Marshal(expect) + if err != nil { + t.Fatal(err) + } + y2, err := yaml.Marshal(got) + if err != nil { + t.Fatalf("Error serializing parsed value: %s", err) + } + + if string(y1) != string(y2) { + t.Errorf("%s: Expected:\n%s\nGot:\n%s", input, y1, y2) + } +} + +func TestParseJSON(t *testing.T) { + tests := []struct { + input string + got map[string]interface{} + expect map[string]interface{} + err bool + }{ + { // set json scalars values, and replace one existing key + input: "outer.inner1=\"1\",outer.inner3=3,outer.inner4=true,outer.inner5=\"true\"", + got: map[string]interface{}{ + "outer": map[string]interface{}{ + "inner1": "overwrite", + "inner2": "value2", + }, + }, + expect: map[string]interface{}{ + "outer": map[string]interface{}{ + "inner1": "1", + "inner2": "value2", + "inner3": 3, + "inner4": true, + "inner5": "true", + }, + }, + err: false, + }, + { // set json objects and arrays, and replace one existing key + input: "outer.inner1={\"a\":\"1\",\"b\":2,\"c\":[1,2,3]},outer.inner3=[\"new value 1\",\"new value 2\"],outer.inner4={\"aa\":\"1\",\"bb\":2,\"cc\":[1,2,3]},outer.inner5=[{\"A\":\"1\",\"B\":2,\"C\":[1,2,3]}]", + got: map[string]interface{}{ + "outer": map[string]interface{}{ + "inner1": map[string]interface{}{ + "x": "overwrite", + }, + "inner2": "value2", + "inner3": []interface{}{ + "overwrite", + }, + }, + }, + expect: map[string]interface{}{ + "outer": map[string]interface{}{ + "inner1": map[string]interface{}{"a": "1", "b": 2, "c": []interface{}{1, 2, 3}}, + "inner2": "value2", + "inner3": []interface{}{"new value 1", "new value 2"}, + "inner4": map[string]interface{}{"aa": "1", "bb": 2, "cc": []interface{}{1, 2, 3}}, + "inner5": []interface{}{map[string]interface{}{"A": "1", "B": 2, "C": []interface{}{1, 2, 3}}}, + }, + }, + err: false, + }, + { // null assigment, and no value assigned (equivalent to null) + input: "outer.inner1=,outer.inner3={\"aa\":\"1\",\"bb\":2,\"cc\":[1,2,3]},outer.inner3.cc[1]=null", + got: map[string]interface{}{ + "outer": map[string]interface{}{ + "inner1": map[string]interface{}{ + "x": "overwrite", + }, + "inner2": "value2", + }, + }, + expect: map[string]interface{}{ + "outer": map[string]interface{}{ + "inner1": nil, + "inner2": "value2", + "inner3": map[string]interface{}{"aa": "1", "bb": 2, "cc": []interface{}{1, nil, 3}}, + }, + }, + err: false, + }, + { // syntax error + input: "outer.inner1={\"a\":\"1\",\"b\":2,\"c\":[1,2,3]},outer.inner3=[\"new value 1\",\"new value 2\"],outer.inner4={\"aa\":\"1\",\"bb\":2,\"cc\":[1,2,3]},outer.inner5={\"A\":\"1\",\"B\":2,\"C\":[1,2,3]}]", + got: nil, + expect: nil, + err: true, + }, + } + for _, tt := range tests { + if err := ParseJSON(tt.input, tt.got); err != nil { + if tt.err { + continue + } + t.Fatalf("%s: %s", tt.input, err) + } + if tt.err { + t.Fatalf("%s: Expected error. Got nil", tt.input) + } + y1, err := yaml.Marshal(tt.expect) + if err != nil { + t.Fatalf("Error serializing expected value: %s", err) + } + y2, err := yaml.Marshal(tt.got) + if err != nil { + t.Fatalf("Error serializing parsed value: %s", err) + } + + if string(y1) != string(y2) { + t.Errorf("%s: Expected:\n%s\nGot:\n%s", tt.input, y1, y2) + } + } +} + +func TestParseFile(t *testing.T) { + input := "name1=path1" + expect := map[string]interface{}{ + "name1": "value1", + } + rs2v := func(rs []rune) (interface{}, error) { + v := string(rs) + if v != "path1" { + t.Errorf("%s: runesToVal: Expected value path1, got %s", input, v) + return "", nil + } + return "value1", nil + } + + got, err := ParseFile(input, rs2v) + if err != nil { + t.Fatal(err) + } + + y1, err := yaml.Marshal(expect) + if err != nil { + t.Fatal(err) + } + y2, err := yaml.Marshal(got) + if err != nil { + t.Fatalf("Error serializing parsed value: %s", err) + } + + if string(y1) != string(y2) { + t.Errorf("%s: Expected:\n%s\nGot:\n%s", input, y1, y2) + } +} + +func TestParseIntoFile(t *testing.T) { + got := map[string]interface{}{} + input := "name1=path1" + expect := map[string]interface{}{ + "name1": "value1", + } + rs2v := func(rs []rune) (interface{}, error) { + v := string(rs) + if v != "path1" { + t.Errorf("%s: runesToVal: Expected value path1, got %s", input, v) + return "", nil + } + return "value1", nil + } + + if err := ParseIntoFile(input, got, rs2v); err != nil { + t.Fatal(err) + } + + y1, err := yaml.Marshal(expect) + if err != nil { + t.Fatal(err) + } + y2, err := yaml.Marshal(got) + if err != nil { + t.Fatalf("Error serializing parsed value: %s", err) + } + + if string(y1) != string(y2) { + t.Errorf("%s: Expected:\n%s\nGot:\n%s", input, y1, y2) + } +} + +func TestToYAML(t *testing.T) { + // The TestParse does the hard part. We just verify that YAML formatting is + // happening. + o, err := ToYAML("name=value") + if err != nil { + t.Fatal(err) + } + expect := "name: value" + if o != expect { + t.Errorf("Expected %q, got %q", expect, o) + } +} + +func TestParseSetNestedLevels(t *testing.T) { + var keyMultipleNestedLevels string + for i := 1; i <= MaxNestedNameLevel+2; i++ { + tmpStr := fmt.Sprintf("name%d", i) + if i <= MaxNestedNameLevel+1 { + tmpStr = tmpStr + "." + } + keyMultipleNestedLevels += tmpStr + } + tests := []struct { + str string + expect map[string]interface{} + err bool + errStr string + }{ + { + "outer.middle.inner=value", + map[string]interface{}{"outer": map[string]interface{}{"middle": map[string]interface{}{"inner": "value"}}}, + false, + "", + }, + { + str: keyMultipleNestedLevels + "=value", + err: true, + errStr: fmt.Sprintf("value name nested level is greater than maximum supported nested level of %d", + MaxNestedNameLevel), + }, + } + + for _, tt := range tests { + got, err := Parse(tt.str) + if err != nil { + if tt.err { + if tt.errStr != "" { + if err.Error() != tt.errStr { + t.Errorf("Expected error: %s. Got error: %s", tt.errStr, err.Error()) + } + } + continue + } + t.Fatalf("%s: %s", tt.str, err) + } + if tt.err { + t.Errorf("%s: Expected error. Got nil", tt.str) + } + + y1, err := yaml.Marshal(tt.expect) + if err != nil { + t.Fatal(err) + } + y2, err := yaml.Marshal(got) + if err != nil { + t.Fatalf("Error serializing parsed value: %s", err) + } + + if string(y1) != string(y2) { + t.Errorf("%s: Expected:\n%s\nGot:\n%s", tt.str, y1, y2) + } + } +} diff --git a/testdata/fireworksapp.sample.yaml b/testdata/fireworksapp.sample.yaml index b87242b..1c4d4b6 100644 --- a/testdata/fireworksapp.sample.yaml +++ b/testdata/fireworksapp.sample.yaml @@ -55,6 +55,6 @@ spec: initialize: true deletionPolicy: Delete replaceValues: - key1: value1 - key2: 5 - key3: true + key1: ciao + key2: 1 + key3: false diff --git a/testdata/formdefinition.sample.yaml b/testdata/formdefinition.sample.yaml index 0144108..c8a85f8 100644 --- a/testdata/formdefinition.sample.yaml +++ b/testdata/formdefinition.sample.yaml @@ -7,7 +7,7 @@ metadata: namespace: demo-system spec: schema: - group: apps.krateo.io + #group: apps.krateo.io version: v1alpha1 kind: FireworksappForm url: https://raw.githubusercontent.com/krateoplatformops/krateo-v2-template-fireworksapp/main/chart/values.schema.json diff --git a/testdata/formtemplate.api.sample.yaml b/testdata/formtemplate.api.sample.yaml new file mode 100644 index 0000000..0219af3 --- /dev/null +++ b/testdata/formtemplate.api.sample.yaml @@ -0,0 +1,39 @@ +--- +apiVersion: v1 +kind: Secret +type: Opaque +metadata: + name: dummyjson-endpoint + namespace: demo-system +stringData: + #debug: "true" + server-url: https://dummyjson.com +--- +apiVersion: widgets.ui.krateo.io/v1alpha1 +kind: FormTemplate +metadata: + name: fireworksapp-with-api + namespace: demo-system +spec: + definitionRef: + name: fireworksapp + namespace: demo-system + resourceRef: + name: fireworksapp + namespace: demo-system + data: + - path: git.replaceValues.key1 + value: ${ .product1.title } + - path: git.replaceValues.key2 + value: ${ .product1.stock } + - path: git.replaceValues.key2 + value: ${ .product1.category } + api: + - name: product1 + path: "/products/1" + endpointRef: + name: dummyjson-endpoint + namespace: demo-system + verb: GET + headers: + - 'Accept: application/json' diff --git a/testdata/formtemplate.sample.yaml b/testdata/formtemplate.sample.yaml index 35c4dbd..36664cf 100644 --- a/testdata/formtemplate.sample.yaml +++ b/testdata/formtemplate.sample.yaml @@ -12,9 +12,9 @@ spec: name: fireworksapp namespace: demo-system data: - - path: .git.replaceValues.key1 - value: value1 - - path: .git.replaceValues.key2 + - path: spec.git.replaceValues.key1 + value: "Hello" + - path: spec.git.replaceValues.key2 value: "32" - - path: .git.replaceValues.key2 - value: "false" + - path: spec.git.replaceValues.key3 + value: "true"