From e9e2346fdd919bdff97300a62093814c1249ccac Mon Sep 17 00:00:00 2001 From: "James J. Goodhouse" Date: Thu, 2 Sep 2021 16:13:03 -0700 Subject: [PATCH 01/10] Add configuration for stores this will allow for setting of parameters specific to each store, such as indentation level for YAML Co-authored-by: Bastien Wermeille Signed-off-by: James J. Goodhouse --- cmd/sops/common/common.go | 40 +++++++++++--------- cmd/sops/main.go | 22 ++++++++++- cmd/sops/subcommand/updatekeys/updatekeys.go | 6 ++- config/config.go | 29 ++++++++++++++ stores/dotenv/store.go | 9 +++++ stores/ini/store.go | 9 +++++ stores/json/store.go | 19 +++++++++- stores/yaml/store.go | 9 +++++ 8 files changed, 122 insertions(+), 21 deletions(-) diff --git a/cmd/sops/common/common.go b/cmd/sops/common/common.go index fa54ecc7d..03d6f340a 100644 --- a/cmd/sops/common/common.go +++ b/cmd/sops/common/common.go @@ -10,6 +10,7 @@ import ( "github.com/getsops/sops/v3" "github.com/getsops/sops/v3/cmd/sops/codes" . "github.com/getsops/sops/v3/cmd/sops/formats" + "github.com/getsops/sops/v3/config" "github.com/getsops/sops/v3/keys" "github.com/getsops/sops/v3/keyservice" "github.com/getsops/sops/v3/kms" @@ -29,32 +30,37 @@ type ExampleFileEmitter interface { EmitExample() []byte } +type Configurable interface { + Configure(*config.Config) +} + // Store handles marshaling and unmarshaling from SOPS files type Store interface { sops.Store ExampleFileEmitter + Configurable } -type storeConstructor = func() Store +type storeConstructor = func(*config.StoresConfig) Store -func newBinaryStore() Store { - return &json.BinaryStore{} +func newBinaryStore(c *config.StoresConfig) Store { + return json.NewBinaryStore(&c.JSONBinary) } -func newDotenvStore() Store { - return &dotenv.Store{} +func newDotenvStore(c *config.StoresConfig) Store { + return dotenv.NewStore(&c.Dotenv) } -func newIniStore() Store { - return &ini.Store{} +func newIniStore(c *config.StoresConfig) Store { + return ini.NewStore(&c.INI) } -func newJsonStore() Store { - return &json.Store{} +func newJsonStore(c *config.StoresConfig) Store { + return json.NewStore(&c.JSON) } -func newYamlStore() Store { - return &yaml.Store{} +func newYamlStore(c *config.StoresConfig) Store { + return yaml.NewStore(&c.YAML) } var storeConstructors = map[Format]storeConstructor{ @@ -153,27 +159,27 @@ func NewExitError(i interface{}, exitCode int) *cli.ExitError { // StoreForFormat returns the correct format-specific implementation // of the Store interface given the format. -func StoreForFormat(format Format) Store { +func StoreForFormat(format Format, c *config.StoresConfig) Store { storeConst, found := storeConstructors[format] if !found { storeConst = storeConstructors[Binary] // default } - return storeConst() + return storeConst(c) } // DefaultStoreForPath returns the correct format-specific implementation // of the Store interface given the path to a file -func DefaultStoreForPath(path string) Store { +func DefaultStoreForPath(c *config.StoresConfig, path string) Store { format := FormatForPath(path) - return StoreForFormat(format) + return StoreForFormat(format, c) } // DefaultStoreForPathOrFormat returns the correct format-specific implementation // of the Store interface given the formatString if specified, or the path to a file. // This is to support the cli, where both are provided. -func DefaultStoreForPathOrFormat(path, format string) Store { +func DefaultStoreForPathOrFormat(c *config.StoresConfig, path string, format string) Store { formatFmt := FormatForPathOrString(path, format) - return StoreForFormat(formatFmt) + return StoreForFormat(formatFmt, c) } // KMS_ENC_CTX_BUG_FIXED_VERSION represents the SOPS version in which the diff --git a/cmd/sops/main.go b/cmd/sops/main.go index c78b51478..177f7af48 100644 --- a/cmd/sops/main.go +++ b/cmd/sops/main.go @@ -1066,11 +1066,29 @@ func keyservices(c *cli.Context) (svcs []keyservice.KeyServiceClient) { } func inputStore(context *cli.Context, path string) common.Store { - return common.DefaultStoreForPathOrFormat(path, context.String("input-type")) + var configPath string + if context.String("config") != "" { + configPath = context.String("config") + } else { + // Ignore config not found errors returned from FindConfigFile since the config file is not mandatory + configPath, _ = config.FindConfigFile(".") + } + storesConf, _ := config.LoadStoresConfig(configPath) + + return common.DefaultStoreForPathOrFormat(storesConf, path, context.String("input-type")) } func outputStore(context *cli.Context, path string) common.Store { - return common.DefaultStoreForPathOrFormat(path, context.String("output-type")) + var configPath string + if context.String("config") != "" { + configPath = context.String("config") + } else { + // Ignore config not found errors returned from FindConfigFile since the config file is not mandatory + configPath, _ = config.FindConfigFile(".") + } + storesConf, _ := config.LoadStoresConfig(configPath) + + return common.DefaultStoreForPathOrFormat(storesConf, path, context.String("output-type")) } func parseTreePath(arg string) ([]interface{}, error) { diff --git a/cmd/sops/subcommand/updatekeys/updatekeys.go b/cmd/sops/subcommand/updatekeys/updatekeys.go index 6bb105864..66ea01e22 100644 --- a/cmd/sops/subcommand/updatekeys/updatekeys.go +++ b/cmd/sops/subcommand/updatekeys/updatekeys.go @@ -40,7 +40,11 @@ func UpdateKeys(opts Opts) error { } func updateFile(opts Opts) error { - store := common.DefaultStoreForPathOrFormat(opts.InputPath, opts.InputType) + sc, err := config.LoadStoresConfig(opts.ConfigPath) + if err != nil { + return err + } + store := common.DefaultStoreForPath(sc, opts.InputPath) log.Printf("Syncing keys for file %s", opts.InputPath) tree, err := common.LoadEncryptedFile(store, opts.InputPath) if err != nil { diff --git a/config/config.go b/config/config.go index 67ddea1bb..113a549f1 100644 --- a/config/config.go +++ b/config/config.go @@ -63,9 +63,30 @@ func FindConfigFile(start string) (string, error) { return "", fmt.Errorf("Config file not found") } +type DotenvStoreConfig struct{} + +type INIStoreConfig struct{} + +type JSONStoreConfig struct{} + +type JSONBinaryStoreConfig struct{} + +type YAMLStoreConfig struct { + Indent int `yaml:"indent"` +} + +type StoresConfig struct { + Dotenv DotenvStoreConfig `yaml:"dotenv"` + INI INIStoreConfig `yaml:"ini"` + JSONBinary JSONBinaryStoreConfig `yaml:"json_binary"` + JSON JSONStoreConfig `yaml:"json"` + YAML YAMLStoreConfig `yaml:"yaml"` +} + type configFile struct { CreationRules []creationRule `yaml:"creation_rules"` DestinationRules []destinationRule `yaml:"destination_rules"` + Stores StoresConfig `yaml:"stores"` } type keyGroup struct { @@ -386,3 +407,11 @@ func LoadDestinationRuleForFile(confPath string, filePath string, kmsEncryptionC } return parseDestinationRuleForFile(conf, filePath, kmsEncryptionContext) } + +func LoadStoresConfig(confPath string) (*StoresConfig, error) { + conf, err := loadConfigFile(confPath) + if err != nil { + return nil, err + } + return &conf.Stores, nil +} diff --git a/stores/dotenv/store.go b/stores/dotenv/store.go index fad0f3494..e930d3960 100644 --- a/stores/dotenv/store.go +++ b/stores/dotenv/store.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/getsops/sops/v3" + "github.com/getsops/sops/v3/config" "github.com/getsops/sops/v3/stores" ) @@ -16,6 +17,11 @@ const SopsPrefix = "sops_" // Store handles storage of dotenv data type Store struct { + config config.DotenvStoreConfig +} + +func NewStore(c *config.DotenvStoreConfig) *Store { + return &Store{config: *c} } // LoadEncryptedFile loads an encrypted file's bytes onto a sops.Tree runtime object @@ -196,3 +202,6 @@ func isComplexValue(v interface{}) bool { } return false } + +func (store *Store) Configure(c *config.Config) { +} diff --git a/stores/ini/store.go b/stores/ini/store.go index 6485467f1..b74705e11 100644 --- a/stores/ini/store.go +++ b/stores/ini/store.go @@ -9,12 +9,18 @@ import ( "strings" "github.com/getsops/sops/v3" + "github.com/getsops/sops/v3/config" "github.com/getsops/sops/v3/stores" "gopkg.in/ini.v1" ) // Store handles storage of ini data. type Store struct { + config *config.INIStoreConfig +} + +func NewStore(c *config.INIStoreConfig) *Store { + return &Store{config: c} } func (store Store) encodeTree(branches sops.TreeBranches) ([]byte, error) { @@ -282,3 +288,6 @@ func (store *Store) EmitExample() []byte { } return bytes } + +func (store *Store) Configure(c *config.Config) { +} diff --git a/stores/json/store.go b/stores/json/store.go index 16a4b5d05..bcd8bdba9 100644 --- a/stores/json/store.go +++ b/stores/json/store.go @@ -8,16 +8,27 @@ import ( "io" "github.com/getsops/sops/v3" + "github.com/getsops/sops/v3/config" "github.com/getsops/sops/v3/stores" ) // Store handles storage of JSON data. type Store struct { + config config.JSONStoreConfig +} + +func NewStore(c *config.JSONStoreConfig) *Store { + return &Store{config: *c} } // BinaryStore handles storage of binary data in a JSON envelope. type BinaryStore struct { - store Store + store Store + config config.JSONBinaryStoreConfig +} + +func NewBinaryStore(c *config.JSONBinaryStoreConfig) *BinaryStore { + return &BinaryStore{config: *c} } // LoadEncryptedFile loads an encrypted json file onto a sops.Tree object @@ -337,3 +348,9 @@ func (store *Store) EmitExample() []byte { } return bytes } + +func (store *Store) Configure(c *config.Config) { +} + +func (store *BinaryStore) Configure(c *config.Config) { +} diff --git a/stores/yaml/store.go b/stores/yaml/store.go index 29fe2652a..74138d468 100644 --- a/stores/yaml/store.go +++ b/stores/yaml/store.go @@ -7,12 +7,18 @@ import ( "strings" "github.com/getsops/sops/v3" + "github.com/getsops/sops/v3/config" "github.com/getsops/sops/v3/stores" "gopkg.in/yaml.v3" ) // Store handles storage of YAML data type Store struct { + config config.YAMLStoreConfig +} + +func NewStore(c *config.YAMLStoreConfig) *Store { + return &Store{config: *c} } func (store Store) appendCommentToList(comment string, list []interface{}) []interface{} { @@ -391,3 +397,6 @@ func (store *Store) EmitExample() []byte { } return bytes } + +func (store *Store) Configure(c *config.Config) { +} From 303fdd8f370012fe7fd4ddbe14a605952498625b Mon Sep 17 00:00:00 2001 From: Bastien Wermeille Date: Fri, 8 Sep 2023 23:31:20 +0200 Subject: [PATCH 02/10] Add cli indent option for yaml store Signed-off-by: Bastien Wermeille --- cmd/sops/main.go | 7 +++++++ decrypt/decrypt.go | 3 ++- stores/yaml/store.go | 14 +++++++++++-- stores/yaml/store_test.go | 41 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 62 insertions(+), 3 deletions(-) diff --git a/cmd/sops/main.go b/cmd/sops/main.go index 177f7af48..3cdc7e933 100644 --- a/cmd/sops/main.go +++ b/cmd/sops/main.go @@ -704,6 +704,10 @@ func main() { Name: "shamir-secret-sharing-threshold", Usage: "the number of master keys required to retrieve the data key with shamir", }, + cli.IntFlag{ + Name: "indent", + Usage: "the number of space to indent yaml encoded file for encryption", + }, cli.BoolFlag{ Name: "verbose", Usage: "Enable verbose logging output", @@ -1087,6 +1091,9 @@ func outputStore(context *cli.Context, path string) common.Store { configPath, _ = config.FindConfigFile(".") } storesConf, _ := config.LoadStoresConfig(configPath) + if context.Int("indent") != 0 { + storesConf.YAML.Indent = context.Int("indent") + } return common.DefaultStoreForPathOrFormat(storesConf, path, context.String("output-type")) } diff --git a/decrypt/decrypt.go b/decrypt/decrypt.go index c3b2ba64b..dc213fd36 100644 --- a/decrypt/decrypt.go +++ b/decrypt/decrypt.go @@ -12,6 +12,7 @@ import ( "github.com/getsops/sops/v3/aes" "github.com/getsops/sops/v3/cmd/sops/common" . "github.com/getsops/sops/v3/cmd/sops/formats" // Re-export + "github.com/getsops/sops/v3/config" ) // File is a wrapper around Data that reads a local encrypted @@ -32,7 +33,7 @@ func File(path, format string) (cleartext []byte, err error) { // decrypts the data and returns its cleartext in an []byte. func DataWithFormat(data []byte, format Format) (cleartext []byte, err error) { - store := common.StoreForFormat(format) + store := common.StoreForFormat(format, &config.StoresConfig{}) // Load SOPS file and access the data key tree, err := store.LoadEncryptedFile(data) diff --git a/stores/yaml/store.go b/stores/yaml/store.go index 74138d468..50850bb10 100644 --- a/stores/yaml/store.go +++ b/stores/yaml/store.go @@ -12,6 +12,8 @@ import ( "gopkg.in/yaml.v3" ) +const IndentDefault = 4 + // Store handles storage of YAML data type Store struct { config config.YAMLStoreConfig @@ -329,7 +331,11 @@ func (store *Store) LoadPlainFile(in []byte) (sops.TreeBranches, error) { func (store *Store) EmitEncryptedFile(in sops.Tree) ([]byte, error) { var b bytes.Buffer e := yaml.NewEncoder(io.Writer(&b)) - e.SetIndent(4) + indent := IndentDefault + if store.config.Indent != 0 { + indent = store.config.Indent + } + e.SetIndent(indent) for _, branch := range in.Branches { // Document root var doc = yaml.Node{} @@ -361,7 +367,11 @@ func (store *Store) EmitEncryptedFile(in sops.Tree) ([]byte, error) { func (store *Store) EmitPlainFile(branches sops.TreeBranches) ([]byte, error) { var b bytes.Buffer e := yaml.NewEncoder(io.Writer(&b)) - e.SetIndent(4) + indent := IndentDefault + if store.config.Indent != 0 { + indent = store.config.Indent + } + e.SetIndent(indent) for _, branch := range branches { // Document root var doc = yaml.Node{} diff --git a/stores/yaml/store_test.go b/stores/yaml/store_test.go index 4851068a3..41b3004d0 100644 --- a/stores/yaml/store_test.go +++ b/stores/yaml/store_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/getsops/sops/v3" + "github.com/getsops/sops/v3/config" "github.com/stretchr/testify/assert" ) @@ -211,6 +212,32 @@ e: - f `) +var INDENT_1_IN = []byte(`## Configuration for prometheus-node-exporter subchart +## +prometheus-node-exporter: + podLabels: + ## Add the 'node-exporter' label to be used by serviceMonitor to match standard common usage in rules and grafana dashboards + ## + + jobLabel: node-exporter + extraArgs: + - --collector.filesystem.ignored-mount-points=^/(dev|proc|sys|var/lib/docker/.+)($|/) + - --collector.filesystem.ignored-fs-types=^(autofs|binfmt_misc|cgroup|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|mqueue|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|sysfs|tracefs)$ +`) + +var INDENT_1_OUT = []byte(`## Configuration for prometheus-node-exporter subchart +## +prometheus-node-exporter: + podLabels: + ## Add the 'node-exporter' label to be used by serviceMonitor to match standard common usage in rules and grafana dashboards + ## + jobLabel: node-exporter + extraArgs: + - --collector.filesystem.ignored-mount-points=^/(dev|proc|sys|var/lib/docker/.+)($|/) + - --collector.filesystem.ignored-fs-types=^(autofs|binfmt_misc|cgroup|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|mqueue|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|sysfs|tracefs)$ +`) + + func TestUnmarshalMetadataFromNonSOPSFile(t *testing.T) { data := []byte(`hello: 2`) _, err := (&Store{}).LoadEncryptedFile(data) @@ -340,3 +367,17 @@ func TestComment7(t *testing.T) { assert.Equal(t, string(COMMENT_7_OUT), string(bytes)) assert.Equal(t, COMMENT_7_OUT, bytes) } + +func TestIndent1(t *testing.T) { + // First iteration: load and store + branches, err := (&Store{}).LoadPlainFile(INDENT_1_IN) + assert.Nil(t, err) + bytes, err := (&Store{ + config: config.YAMLStoreConfig{ + Indent: 2, + }, + }).EmitPlainFile(branches) + assert.Nil(t, err) + assert.Equal(t, string(INDENT_1_OUT), string(bytes)) + assert.Equal(t, INDENT_1_OUT, bytes) +} \ No newline at end of file From d2ee8df5de04c25b5a3912bfe8b64fdf6e779627 Mon Sep 17 00:00:00 2001 From: Bastien Wermeille Date: Sat, 9 Sep 2023 17:42:08 +0200 Subject: [PATCH 03/10] Implement feedback Co-authored-by: Felix Fontein Signed-off-by: Bastien Wermeille --- cmd/sops/common/common.go | 5 ----- cmd/sops/main.go | 18 +++++++----------- stores/dotenv/store.go | 3 --- stores/ini/store.go | 3 --- stores/json/store.go | 6 ------ stores/yaml/store.go | 3 --- 6 files changed, 7 insertions(+), 31 deletions(-) diff --git a/cmd/sops/common/common.go b/cmd/sops/common/common.go index 03d6f340a..4960e6e47 100644 --- a/cmd/sops/common/common.go +++ b/cmd/sops/common/common.go @@ -30,15 +30,10 @@ type ExampleFileEmitter interface { EmitExample() []byte } -type Configurable interface { - Configure(*config.Config) -} - // Store handles marshaling and unmarshaling from SOPS files type Store interface { sops.Store ExampleFileEmitter - Configurable } type storeConstructor = func(*config.StoresConfig) Store diff --git a/cmd/sops/main.go b/cmd/sops/main.go index 3cdc7e933..96ae2477a 100644 --- a/cmd/sops/main.go +++ b/cmd/sops/main.go @@ -706,7 +706,7 @@ func main() { }, cli.IntFlag{ Name: "indent", - Usage: "the number of space to indent yaml encoded file for encryption", + Usage: "the number of space to indent YAML encoded file for encryption", }, cli.BoolFlag{ Name: "verbose", @@ -1069,7 +1069,7 @@ func keyservices(c *cli.Context) (svcs []keyservice.KeyServiceClient) { return } -func inputStore(context *cli.Context, path string) common.Store { +func loadStoresConfig(context *cli.Context, path string) (*config.StoresConfig, error) { var configPath string if context.String("config") != "" { configPath = context.String("config") @@ -1077,20 +1077,16 @@ func inputStore(context *cli.Context, path string) common.Store { // Ignore config not found errors returned from FindConfigFile since the config file is not mandatory configPath, _ = config.FindConfigFile(".") } - storesConf, _ := config.LoadStoresConfig(configPath) + return config.LoadStoresConfig(configPath) +} +func inputStore(context *cli.Context, path string) common.Store { + storesConf, _ := loadStoresConfig(context, path) return common.DefaultStoreForPathOrFormat(storesConf, path, context.String("input-type")) } func outputStore(context *cli.Context, path string) common.Store { - var configPath string - if context.String("config") != "" { - configPath = context.String("config") - } else { - // Ignore config not found errors returned from FindConfigFile since the config file is not mandatory - configPath, _ = config.FindConfigFile(".") - } - storesConf, _ := config.LoadStoresConfig(configPath) + storesConf, _ := loadStoresConfig(context, path) if context.Int("indent") != 0 { storesConf.YAML.Indent = context.Int("indent") } diff --git a/stores/dotenv/store.go b/stores/dotenv/store.go index e930d3960..db895fcd9 100644 --- a/stores/dotenv/store.go +++ b/stores/dotenv/store.go @@ -202,6 +202,3 @@ func isComplexValue(v interface{}) bool { } return false } - -func (store *Store) Configure(c *config.Config) { -} diff --git a/stores/ini/store.go b/stores/ini/store.go index b74705e11..d703d87cf 100644 --- a/stores/ini/store.go +++ b/stores/ini/store.go @@ -288,6 +288,3 @@ func (store *Store) EmitExample() []byte { } return bytes } - -func (store *Store) Configure(c *config.Config) { -} diff --git a/stores/json/store.go b/stores/json/store.go index bcd8bdba9..5b56236ed 100644 --- a/stores/json/store.go +++ b/stores/json/store.go @@ -348,9 +348,3 @@ func (store *Store) EmitExample() []byte { } return bytes } - -func (store *Store) Configure(c *config.Config) { -} - -func (store *BinaryStore) Configure(c *config.Config) { -} diff --git a/stores/yaml/store.go b/stores/yaml/store.go index 50850bb10..701f7edc5 100644 --- a/stores/yaml/store.go +++ b/stores/yaml/store.go @@ -407,6 +407,3 @@ func (store *Store) EmitExample() []byte { } return bytes } - -func (store *Store) Configure(c *config.Config) { -} From 27f40497e837144878188cffbddadbcb0046aeb1 Mon Sep 17 00:00:00 2001 From: Bastien Wermeille Date: Wed, 13 Sep 2023 09:40:08 +0200 Subject: [PATCH 04/10] Add documentation Signed-off-by: Bastien Wermeille --- README.rst | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.rst b/README.rst index d7a4e80fd..4d311503b 100644 --- a/README.rst +++ b/README.rst @@ -1162,8 +1162,22 @@ When operating on stdin, use the ``--input-type`` and ``--output-type`` flags as $ cat myfile.json | sops --input-type json --output-type json -d /dev/stdin +YAML indentation +~~~~~~~~~~~~~~~~ + +``sops`` indent ``YAML`` files by default using 4 spaces. However, you can change +this default behaviour be either using the additional ``--indent=2`` cli option or +by configuring ``.sops.yaml`` with : + +.. code:: yaml + stores: + yaml: + indent: 2 + + YAML anchors ~~~~~~~~~~~~ + SOPS only supports a subset of ``YAML``'s many types. Encrypting YAML files that contain strings, numbers and booleans will work fine, but files that contain anchors will not work, because the anchors redefine the structure of the file at load time. From 6ad2a82d223902b5a248d9cb34a2807680df24ae Mon Sep 17 00:00:00 2001 From: Bastien Wermeille Date: Wed, 13 Sep 2023 14:41:12 +0200 Subject: [PATCH 05/10] Refactor duplicated code Signed-off-by: Bastien Wermeille --- stores/yaml/store.go | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/stores/yaml/store.go b/stores/yaml/store.go index 701f7edc5..aae961474 100644 --- a/stores/yaml/store.go +++ b/stores/yaml/store.go @@ -326,16 +326,19 @@ func (store *Store) LoadPlainFile(in []byte) (sops.TreeBranches, error) { return branches, nil } +func (store *Store) getIndentation() int { + if store.config.Indent != 0 { + return store.config.Indent + } + return IndentDefault +} + // EmitEncryptedFile returns the encrypted bytes of the yaml file corresponding to a // sops.Tree runtime object func (store *Store) EmitEncryptedFile(in sops.Tree) ([]byte, error) { var b bytes.Buffer e := yaml.NewEncoder(io.Writer(&b)) - indent := IndentDefault - if store.config.Indent != 0 { - indent = store.config.Indent - } - e.SetIndent(indent) + e.SetIndent(store.getIndentation()) for _, branch := range in.Branches { // Document root var doc = yaml.Node{} @@ -367,11 +370,7 @@ func (store *Store) EmitEncryptedFile(in sops.Tree) ([]byte, error) { func (store *Store) EmitPlainFile(branches sops.TreeBranches) ([]byte, error) { var b bytes.Buffer e := yaml.NewEncoder(io.Writer(&b)) - indent := IndentDefault - if store.config.Indent != 0 { - indent = store.config.Indent - } - e.SetIndent(indent) + e.SetIndent(store.getIndentation()) for _, branch := range branches { // Document root var doc = yaml.Node{} From c6dc5267e544a29e68f9a9a0693135019b750d97 Mon Sep 17 00:00:00 2001 From: Bastien Wermeille Date: Thu, 14 Sep 2023 08:41:23 +0200 Subject: [PATCH 06/10] Indentation of json files Signed-off-by: Bastien Wermeille --- README.rst | 13 ++++++- cmd/sops/main.go | 4 +- config/config.go | 4 +- stores/json/store.go | 7 +++- stores/json/store_test.go | 81 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 105 insertions(+), 4 deletions(-) diff --git a/README.rst b/README.rst index 4d311503b..4178ee1c9 100644 --- a/README.rst +++ b/README.rst @@ -1162,6 +1162,18 @@ When operating on stdin, use the ``--input-type`` and ``--output-type`` flags as $ cat myfile.json | sops --input-type json --output-type json -d /dev/stdin +JSON indentation +~~~~~~~~~~~~~~~~ + +``sops`` indent ``SOPS`` files by default using one ``tab``. However, you can change +this default behaviour to use spaces be either using the additional ``--indent=2`` cli option or +by configuring ``.sops.yaml`` with : + +.. code:: yaml + stores: + json: + indent: 2 + YAML indentation ~~~~~~~~~~~~~~~~ @@ -1174,7 +1186,6 @@ by configuring ``.sops.yaml`` with : yaml: indent: 2 - YAML anchors ~~~~~~~~~~~~ diff --git a/cmd/sops/main.go b/cmd/sops/main.go index 96ae2477a..3ee3ec0e8 100644 --- a/cmd/sops/main.go +++ b/cmd/sops/main.go @@ -1088,7 +1088,9 @@ func inputStore(context *cli.Context, path string) common.Store { func outputStore(context *cli.Context, path string) common.Store { storesConf, _ := loadStoresConfig(context, path) if context.Int("indent") != 0 { - storesConf.YAML.Indent = context.Int("indent") + indent := context.Int("indent") + storesConf.YAML.Indent = indent + storesConf.JSON.Indent = indent } return common.DefaultStoreForPathOrFormat(storesConf, path, context.String("output-type")) diff --git a/config/config.go b/config/config.go index 113a549f1..fbb99d535 100644 --- a/config/config.go +++ b/config/config.go @@ -67,7 +67,9 @@ type DotenvStoreConfig struct{} type INIStoreConfig struct{} -type JSONStoreConfig struct{} +type JSONStoreConfig struct{ + Indent int `yaml:"indent"` +} type JSONBinaryStoreConfig struct{} diff --git a/stores/json/store.go b/stores/json/store.go index 5b56236ed..41009d358 100644 --- a/stores/json/store.go +++ b/stores/json/store.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "io" + "strings" "github.com/getsops/sops/v3" "github.com/getsops/sops/v3/config" @@ -248,7 +249,11 @@ func (store Store) treeBranchFromJSON(in []byte) (sops.TreeBranch, error) { func (store Store) reindentJSON(in []byte) ([]byte, error) { var out bytes.Buffer - err := json.Indent(&out, in, "", "\t") + indent := "\t" + if store.config.Indent != 0 { + indent = strings.Repeat(" ", store.config.Indent) + } + err := json.Indent(&out, in, "", indent) return out.Bytes(), err } diff --git a/stores/json/store_test.go b/stores/json/store_test.go index 311276d0d..991610cac 100644 --- a/stores/json/store_test.go +++ b/stores/json/store_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/getsops/sops/v3" + "github.com/getsops/sops/v3/config" "github.com/stretchr/testify/assert" ) @@ -409,3 +410,83 @@ func TestEmitValueString(t *testing.T) { assert.Nil(t, err) assert.Equal(t, []byte("\"hello\""), bytes) } + +func TestIndentTwoSpaces(t *testing.T) { + tree := sops.Tree{ + Branches: sops.TreeBranches{ + sops.TreeBranch{ + sops.TreeItem{ + Key: "foo", + Value: []interface{}{ + sops.TreeBranch{ + sops.TreeItem{ + Key: "foo", + Value: 3, + }, + sops.TreeItem{ + Key: "bar", + Value: false, + }, + }, + 2, + }, + }, + }, + }, + } + expected := `{ + "foo": [ + { + "foo": 3, + "bar": false + }, + 2 + ] +}` + store := Store{ + config: config.JSONStoreConfig{ + Indent: 2, + }, + } + out, err := store.EmitPlainFile(tree.Branches) + assert.Nil(t, err) + assert.Equal(t, expected, string(out)) +} + +func TestIndentDefault(t *testing.T) { + tree := sops.Tree{ + Branches: sops.TreeBranches{ + sops.TreeBranch{ + sops.TreeItem{ + Key: "foo", + Value: []interface{}{ + sops.TreeBranch{ + sops.TreeItem{ + Key: "foo", + Value: 3, + }, + sops.TreeItem{ + Key: "bar", + Value: false, + }, + }, + 2, + }, + }, + }, + }, + } + expected := `{ + "foo": [ + { + "foo": 3, + "bar": false + }, + 2 + ] +}` + store := Store{} + out, err := store.EmitPlainFile(tree.Branches) + assert.Nil(t, err) + assert.Equal(t, expected, string(out)) +} From 755c16d49c6417bfc59542821f2aead9338dd468 Mon Sep 17 00:00:00 2001 From: Bastien Wermeille Date: Thu, 14 Sep 2023 20:41:56 +0200 Subject: [PATCH 07/10] Allow no indent at all for json store Signed-off-by: Bastien Wermeille --- README.rst | 6 ++--- config/config.go | 7 +++++ decrypt/decrypt.go | 2 +- stores/json/store.go | 2 +- stores/json/store_test.go | 54 +++++++++++++++++++++++++++++++++++++-- 5 files changed, 64 insertions(+), 7 deletions(-) diff --git a/README.rst b/README.rst index 4178ee1c9..6b89181db 100644 --- a/README.rst +++ b/README.rst @@ -1166,8 +1166,8 @@ JSON indentation ~~~~~~~~~~~~~~~~ ``sops`` indent ``SOPS`` files by default using one ``tab``. However, you can change -this default behaviour to use spaces be either using the additional ``--indent=2`` cli option or -by configuring ``.sops.yaml`` with : +this default behaviour to use spaces by either using the additional ``--indent=2`` cli option or +by configuring ``.sops.yaml`` with the code below. (value ``0`` is no indentation) .. code:: yaml stores: @@ -1178,7 +1178,7 @@ YAML indentation ~~~~~~~~~~~~~~~~ ``sops`` indent ``YAML`` files by default using 4 spaces. However, you can change -this default behaviour be either using the additional ``--indent=2`` cli option or +this default behaviour by either using the additional ``--indent=2`` cli option or by configuring ``.sops.yaml`` with : .. code:: yaml diff --git a/config/config.go b/config/config.go index fbb99d535..8ab15c235 100644 --- a/config/config.go +++ b/config/config.go @@ -149,6 +149,12 @@ type creationRule struct { MACOnlyEncrypted bool `yaml:"mac_only_encrypted"` } +func NewStoresConfig() *StoresConfig{ + storesConfig := &StoresConfig{} + storesConfig.JSON.Indent = -1 + return storesConfig +} + // Load loads a sops config file into a temporary struct func (f *configFile) load(bytes []byte) error { err := yaml.Unmarshal(bytes, f) @@ -252,6 +258,7 @@ func loadConfigFile(confPath string) (*configFile, error) { return nil, fmt.Errorf("could not read config file: %s", err) } conf := &configFile{} + conf.Stores = *NewStoresConfig() err = conf.load(confBytes) if err != nil { return nil, fmt.Errorf("error loading config: %s", err) diff --git a/decrypt/decrypt.go b/decrypt/decrypt.go index dc213fd36..e26cbe479 100644 --- a/decrypt/decrypt.go +++ b/decrypt/decrypt.go @@ -33,7 +33,7 @@ func File(path, format string) (cleartext []byte, err error) { // decrypts the data and returns its cleartext in an []byte. func DataWithFormat(data []byte, format Format) (cleartext []byte, err error) { - store := common.StoreForFormat(format, &config.StoresConfig{}) + store := common.StoreForFormat(format, config.NewStoresConfig()) // Load SOPS file and access the data key tree, err := store.LoadEncryptedFile(data) diff --git a/stores/json/store.go b/stores/json/store.go index 41009d358..da1cc34ff 100644 --- a/stores/json/store.go +++ b/stores/json/store.go @@ -250,7 +250,7 @@ func (store Store) treeBranchFromJSON(in []byte) (sops.TreeBranch, error) { func (store Store) reindentJSON(in []byte) ([]byte, error) { var out bytes.Buffer indent := "\t" - if store.config.Indent != 0 { + if store.config.Indent != -1 { indent = strings.Repeat(" ", store.config.Indent) } err := json.Indent(&out, in, "", indent) diff --git a/stores/json/store_test.go b/stores/json/store_test.go index 991610cac..eca6d3e96 100644 --- a/stores/json/store_test.go +++ b/stores/json/store_test.go @@ -313,7 +313,11 @@ func TestEncodeJSONArrayOfObjects(t *testing.T) { 2 ] }` - store := Store{} + store := Store{ + config: config.JSONStoreConfig{ + Indent: -1, + }, + } out, err := store.EmitPlainFile(tree.Branches) assert.Nil(t, err) assert.Equal(t, expected, string(out)) @@ -485,7 +489,53 @@ func TestIndentDefault(t *testing.T) { 2 ] }` - store := Store{} + store := Store{ + config: config.JSONStoreConfig{ + Indent: -1, + }, + } + out, err := store.EmitPlainFile(tree.Branches) + assert.Nil(t, err) + assert.Equal(t, expected, string(out)) +} + +func TestNoIndent(t *testing.T) { + tree := sops.Tree{ + Branches: sops.TreeBranches{ + sops.TreeBranch{ + sops.TreeItem{ + Key: "foo", + Value: []interface{}{ + sops.TreeBranch{ + sops.TreeItem{ + Key: "foo", + Value: 3, + }, + sops.TreeItem{ + Key: "bar", + Value: false, + }, + }, + 2, + }, + }, + }, + }, + } + expected := `{ +"foo": [ +{ +"foo": 3, +"bar": false +}, +2 +] +}` + store := Store{ + config: config.JSONStoreConfig{ + Indent: 0, + }, + } out, err := store.EmitPlainFile(tree.Branches) assert.Nil(t, err) assert.Equal(t, expected, string(out)) From 9d20985c583d2da7d15fa90c9f56e3f06a5d0b91 Mon Sep 17 00:00:00 2001 From: Bastien Date: Fri, 3 Nov 2023 09:56:19 +0100 Subject: [PATCH 08/10] Improve documentation Signed-off-by: Bastien --- README.rst | 22 +++++++++++++--------- cmd/sops/main.go | 4 ++-- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/README.rst b/README.rst index 6b89181db..cce88cbc5 100644 --- a/README.rst +++ b/README.rst @@ -1130,15 +1130,15 @@ Below is an example of publishing to Vault (using token auth with a local dev in Important information on types ------------------------------ -YAML and JSON type extensions -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +YAML, JSON, ENV and INI type extensions +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ SOPS uses the file extension to decide which encryption method to use on the file content. ``YAML``, ``JSON``, ``ENV``, and ``INI`` files are treated as trees of data, and key/values are extracted from the files to only encrypt the leaf values. The tree structure is also used to check the integrity of the file. -Therefore, if a file is encrypted using a specific format, it need to be decrypted +Therefore, if a file is encrypted using a specific format, it needs to be decrypted in the same format. The easiest way to achieve this is to conserve the original file extension after encrypting a file. For example: @@ -1165,11 +1165,14 @@ When operating on stdin, use the ``--input-type`` and ``--output-type`` flags as JSON indentation ~~~~~~~~~~~~~~~~ -``sops`` indent ``SOPS`` files by default using one ``tab``. However, you can change -this default behaviour to use spaces by either using the additional ``--indent=2`` cli option or -by configuring ``.sops.yaml`` with the code below. (value ``0`` is no indentation) +SOPS indents ``JSON`` files by default using one ``tab``. However, you can change +this default behaviour to use ``spaces`` by either using the additional ``--indent=2`` CLI option or +by configuring ``.sops.yaml`` with the code below. + +The special value ``0`` disables indentation, and ``-1`` uses a single tab. .. code:: yaml + stores: json: indent: 2 @@ -1177,11 +1180,12 @@ by configuring ``.sops.yaml`` with the code below. (value ``0`` is no indentatio YAML indentation ~~~~~~~~~~~~~~~~ -``sops`` indent ``YAML`` files by default using 4 spaces. However, you can change -this default behaviour by either using the additional ``--indent=2`` cli option or -by configuring ``.sops.yaml`` with : +SOPS indents ``YAML`` files by default using 4 spaces. However, you can change +this default behaviour by either using the additional ``--indent=2`` CLI option or +by configuring ``.sops.yaml`` with: .. code:: yaml + stores: yaml: indent: 2 diff --git a/cmd/sops/main.go b/cmd/sops/main.go index 3ee3ec0e8..9dfb63e6d 100644 --- a/cmd/sops/main.go +++ b/cmd/sops/main.go @@ -706,7 +706,7 @@ func main() { }, cli.IntFlag{ Name: "indent", - Usage: "the number of space to indent YAML encoded file for encryption", + Usage: "the number of spaces to indent YAML or JSON encoded file for encryption", }, cli.BoolFlag{ Name: "verbose", @@ -1087,7 +1087,7 @@ func inputStore(context *cli.Context, path string) common.Store { func outputStore(context *cli.Context, path string) common.Store { storesConf, _ := loadStoresConfig(context, path) - if context.Int("indent") != 0 { + if context.IsSet("indent") { indent := context.Int("indent") storesConf.YAML.Indent = indent storesConf.JSON.Indent = indent From 8a63bb0d21d8f744b4d49ee4e3e44a14891379b0 Mon Sep 17 00:00:00 2001 From: Bastien Date: Mon, 13 Nov 2023 08:55:19 +0100 Subject: [PATCH 09/10] Add checks for indentation value Signed-off-by: Bastien --- stores/json/store.go | 4 +++- stores/yaml/store.go | 23 +++++++++++++++++------ 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/stores/json/store.go b/stores/json/store.go index da1cc34ff..3744563bc 100644 --- a/stores/json/store.go +++ b/stores/json/store.go @@ -250,8 +250,10 @@ func (store Store) treeBranchFromJSON(in []byte) (sops.TreeBranch, error) { func (store Store) reindentJSON(in []byte) ([]byte, error) { var out bytes.Buffer indent := "\t" - if store.config.Indent != -1 { + if store.config.Indent > -1 { indent = strings.Repeat(" ", store.config.Indent) + } else if store.config.Indent < -1 { + return nil, errors.New("JSON Indentation parameter smaller than -1 is not accepted") } err := json.Indent(&out, in, "", indent) return out.Bytes(), err diff --git a/stores/yaml/store.go b/stores/yaml/store.go index aae961474..4d036f366 100644 --- a/stores/yaml/store.go +++ b/stores/yaml/store.go @@ -2,6 +2,7 @@ package yaml //import "github.com/getsops/sops/v3/stores/yaml" import ( "bytes" + "errors" "fmt" "io" "strings" @@ -326,11 +327,13 @@ func (store *Store) LoadPlainFile(in []byte) (sops.TreeBranches, error) { return branches, nil } -func (store *Store) getIndentation() int { - if store.config.Indent != 0 { - return store.config.Indent +func (store *Store) getIndentation() (int, error) { + if store.config.Indent > 0 { + return store.config.Indent, nil + } else if store.config.Indent < 0 { + return 0, errors.New("YAML Negative indentation not accepted") } - return IndentDefault + return IndentDefault, nil } // EmitEncryptedFile returns the encrypted bytes of the yaml file corresponding to a @@ -338,7 +341,11 @@ func (store *Store) getIndentation() int { func (store *Store) EmitEncryptedFile(in sops.Tree) ([]byte, error) { var b bytes.Buffer e := yaml.NewEncoder(io.Writer(&b)) - e.SetIndent(store.getIndentation()) + indent, err := store.getIndentation() + if err != nil { + return nil, err + } + e.SetIndent(indent) for _, branch := range in.Branches { // Document root var doc = yaml.Node{} @@ -370,7 +377,11 @@ func (store *Store) EmitEncryptedFile(in sops.Tree) ([]byte, error) { func (store *Store) EmitPlainFile(branches sops.TreeBranches) ([]byte, error) { var b bytes.Buffer e := yaml.NewEncoder(io.Writer(&b)) - e.SetIndent(store.getIndentation()) + indent, err := store.getIndentation() + if err != nil { + return nil, err + } + e.SetIndent(indent) for _, branch := range branches { // Document root var doc = yaml.Node{} From 42018ef4a560c06de9b877340e1760a2075b2a2d Mon Sep 17 00:00:00 2001 From: Bastien Date: Tue, 14 Nov 2023 07:27:47 +0100 Subject: [PATCH 10/10] Add indentation settings for json_binary Signed-off-by: Bastien --- README.rst | 6 ++++-- cmd/sops/main.go | 1 + config/config.go | 11 +++++++---- stores/json/store.go | 4 +++- 4 files changed, 15 insertions(+), 7 deletions(-) diff --git a/README.rst b/README.rst index cce88cbc5..c6151315c 100644 --- a/README.rst +++ b/README.rst @@ -1162,8 +1162,8 @@ When operating on stdin, use the ``--input-type`` and ``--output-type`` flags as $ cat myfile.json | sops --input-type json --output-type json -d /dev/stdin -JSON indentation -~~~~~~~~~~~~~~~~ +JSON and JSON_binary indentation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ SOPS indents ``JSON`` files by default using one ``tab``. However, you can change this default behaviour to use ``spaces`` by either using the additional ``--indent=2`` CLI option or @@ -1176,6 +1176,8 @@ The special value ``0`` disables indentation, and ``-1`` uses a single tab. stores: json: indent: 2 + json_binary: + indent: 2 YAML indentation ~~~~~~~~~~~~~~~~ diff --git a/cmd/sops/main.go b/cmd/sops/main.go index 9dfb63e6d..a1d4807a1 100644 --- a/cmd/sops/main.go +++ b/cmd/sops/main.go @@ -1091,6 +1091,7 @@ func outputStore(context *cli.Context, path string) common.Store { indent := context.Int("indent") storesConf.YAML.Indent = indent storesConf.JSON.Indent = indent + storesConf.JSONBinary.Indent = indent } return common.DefaultStoreForPathOrFormat(storesConf, path, context.String("output-type")) diff --git a/config/config.go b/config/config.go index 8ab15c235..643268682 100644 --- a/config/config.go +++ b/config/config.go @@ -67,11 +67,13 @@ type DotenvStoreConfig struct{} type INIStoreConfig struct{} -type JSONStoreConfig struct{ +type JSONStoreConfig struct { Indent int `yaml:"indent"` } -type JSONBinaryStoreConfig struct{} +type JSONBinaryStoreConfig struct { + Indent int `yaml:"indent"` +} type YAMLStoreConfig struct { Indent int `yaml:"indent"` @@ -149,9 +151,10 @@ type creationRule struct { MACOnlyEncrypted bool `yaml:"mac_only_encrypted"` } -func NewStoresConfig() *StoresConfig{ - storesConfig := &StoresConfig{} +func NewStoresConfig() *StoresConfig { + storesConfig := &StoresConfig{} storesConfig.JSON.Indent = -1 + storesConfig.JSONBinary.Indent = -1 return storesConfig } diff --git a/stores/json/store.go b/stores/json/store.go index 3744563bc..cbecd8362 100644 --- a/stores/json/store.go +++ b/stores/json/store.go @@ -29,7 +29,9 @@ type BinaryStore struct { } func NewBinaryStore(c *config.JSONBinaryStoreConfig) *BinaryStore { - return &BinaryStore{config: *c} + return &BinaryStore{config: *c, store: *NewStore(&config.JSONStoreConfig{ + Indent: c.Indent, + })} } // LoadEncryptedFile loads an encrypted json file onto a sops.Tree object