diff --git a/README.rst b/README.rst index d7a4e80fd..c6151315c 100644 --- a/README.rst +++ b/README.rst @@ -1130,15 +1130,15 @@ Below is an example of publishing to Vault (using token auth with a local dev in Important information on types ------------------------------ -YAML and JSON type extensions -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +YAML, JSON, ENV and INI type extensions +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ SOPS uses the file extension to decide which encryption method to use on the file content. ``YAML``, ``JSON``, ``ENV``, and ``INI`` files are treated as trees of data, and key/values are extracted from the files to only encrypt the leaf values. The tree structure is also used to check the integrity of the file. -Therefore, if a file is encrypted using a specific format, it need to be decrypted +Therefore, if a file is encrypted using a specific format, it needs to be decrypted in the same format. The easiest way to achieve this is to conserve the original file extension after encrypting a file. For example: @@ -1162,8 +1162,39 @@ When operating on stdin, use the ``--input-type`` and ``--output-type`` flags as $ cat myfile.json | sops --input-type json --output-type json -d /dev/stdin +JSON and JSON_binary indentation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +SOPS indents ``JSON`` files by default using one ``tab``. However, you can change +this default behaviour to use ``spaces`` by either using the additional ``--indent=2`` CLI option or +by configuring ``.sops.yaml`` with the code below. + +The special value ``0`` disables indentation, and ``-1`` uses a single tab. + +.. code:: yaml + + stores: + json: + indent: 2 + json_binary: + indent: 2 + +YAML indentation +~~~~~~~~~~~~~~~~ + +SOPS indents ``YAML`` files by default using 4 spaces. However, you can change +this default behaviour by either using the additional ``--indent=2`` CLI option or +by configuring ``.sops.yaml`` with: + +.. code:: yaml + + stores: + yaml: + indent: 2 + YAML anchors ~~~~~~~~~~~~ + SOPS only supports a subset of ``YAML``'s many types. Encrypting YAML files that contain strings, numbers and booleans will work fine, but files that contain anchors will not work, because the anchors redefine the structure of the file at load time. diff --git a/cmd/sops/common/common.go b/cmd/sops/common/common.go index fa54ecc7d..4960e6e47 100644 --- a/cmd/sops/common/common.go +++ b/cmd/sops/common/common.go @@ -10,6 +10,7 @@ import ( "github.com/getsops/sops/v3" "github.com/getsops/sops/v3/cmd/sops/codes" . "github.com/getsops/sops/v3/cmd/sops/formats" + "github.com/getsops/sops/v3/config" "github.com/getsops/sops/v3/keys" "github.com/getsops/sops/v3/keyservice" "github.com/getsops/sops/v3/kms" @@ -35,26 +36,26 @@ type Store interface { ExampleFileEmitter } -type storeConstructor = func() Store +type storeConstructor = func(*config.StoresConfig) Store -func newBinaryStore() Store { - return &json.BinaryStore{} +func newBinaryStore(c *config.StoresConfig) Store { + return json.NewBinaryStore(&c.JSONBinary) } -func newDotenvStore() Store { - return &dotenv.Store{} +func newDotenvStore(c *config.StoresConfig) Store { + return dotenv.NewStore(&c.Dotenv) } -func newIniStore() Store { - return &ini.Store{} +func newIniStore(c *config.StoresConfig) Store { + return ini.NewStore(&c.INI) } -func newJsonStore() Store { - return &json.Store{} +func newJsonStore(c *config.StoresConfig) Store { + return json.NewStore(&c.JSON) } -func newYamlStore() Store { - return &yaml.Store{} +func newYamlStore(c *config.StoresConfig) Store { + return yaml.NewStore(&c.YAML) } var storeConstructors = map[Format]storeConstructor{ @@ -153,27 +154,27 @@ func NewExitError(i interface{}, exitCode int) *cli.ExitError { // StoreForFormat returns the correct format-specific implementation // of the Store interface given the format. -func StoreForFormat(format Format) Store { +func StoreForFormat(format Format, c *config.StoresConfig) Store { storeConst, found := storeConstructors[format] if !found { storeConst = storeConstructors[Binary] // default } - return storeConst() + return storeConst(c) } // DefaultStoreForPath returns the correct format-specific implementation // of the Store interface given the path to a file -func DefaultStoreForPath(path string) Store { +func DefaultStoreForPath(c *config.StoresConfig, path string) Store { format := FormatForPath(path) - return StoreForFormat(format) + return StoreForFormat(format, c) } // DefaultStoreForPathOrFormat returns the correct format-specific implementation // of the Store interface given the formatString if specified, or the path to a file. // This is to support the cli, where both are provided. -func DefaultStoreForPathOrFormat(path, format string) Store { +func DefaultStoreForPathOrFormat(c *config.StoresConfig, path string, format string) Store { formatFmt := FormatForPathOrString(path, format) - return StoreForFormat(formatFmt) + return StoreForFormat(formatFmt, c) } // KMS_ENC_CTX_BUG_FIXED_VERSION represents the SOPS version in which the diff --git a/cmd/sops/main.go b/cmd/sops/main.go index c78b51478..a1d4807a1 100644 --- a/cmd/sops/main.go +++ b/cmd/sops/main.go @@ -704,6 +704,10 @@ func main() { Name: "shamir-secret-sharing-threshold", Usage: "the number of master keys required to retrieve the data key with shamir", }, + cli.IntFlag{ + Name: "indent", + Usage: "the number of spaces to indent YAML or JSON encoded file for encryption", + }, cli.BoolFlag{ Name: "verbose", Usage: "Enable verbose logging output", @@ -1065,12 +1069,32 @@ func keyservices(c *cli.Context) (svcs []keyservice.KeyServiceClient) { return } +func loadStoresConfig(context *cli.Context, path string) (*config.StoresConfig, error) { + var configPath string + if context.String("config") != "" { + configPath = context.String("config") + } else { + // Ignore config not found errors returned from FindConfigFile since the config file is not mandatory + configPath, _ = config.FindConfigFile(".") + } + return config.LoadStoresConfig(configPath) +} + func inputStore(context *cli.Context, path string) common.Store { - return common.DefaultStoreForPathOrFormat(path, context.String("input-type")) + storesConf, _ := loadStoresConfig(context, path) + return common.DefaultStoreForPathOrFormat(storesConf, path, context.String("input-type")) } func outputStore(context *cli.Context, path string) common.Store { - return common.DefaultStoreForPathOrFormat(path, context.String("output-type")) + storesConf, _ := loadStoresConfig(context, path) + if context.IsSet("indent") { + indent := context.Int("indent") + storesConf.YAML.Indent = indent + storesConf.JSON.Indent = indent + storesConf.JSONBinary.Indent = indent + } + + return common.DefaultStoreForPathOrFormat(storesConf, path, context.String("output-type")) } func parseTreePath(arg string) ([]interface{}, error) { diff --git a/cmd/sops/subcommand/updatekeys/updatekeys.go b/cmd/sops/subcommand/updatekeys/updatekeys.go index 6bb105864..66ea01e22 100644 --- a/cmd/sops/subcommand/updatekeys/updatekeys.go +++ b/cmd/sops/subcommand/updatekeys/updatekeys.go @@ -40,7 +40,11 @@ func UpdateKeys(opts Opts) error { } func updateFile(opts Opts) error { - store := common.DefaultStoreForPathOrFormat(opts.InputPath, opts.InputType) + sc, err := config.LoadStoresConfig(opts.ConfigPath) + if err != nil { + return err + } + store := common.DefaultStoreForPath(sc, opts.InputPath) log.Printf("Syncing keys for file %s", opts.InputPath) tree, err := common.LoadEncryptedFile(store, opts.InputPath) if err != nil { diff --git a/config/config.go b/config/config.go index 67ddea1bb..643268682 100644 --- a/config/config.go +++ b/config/config.go @@ -63,9 +63,34 @@ func FindConfigFile(start string) (string, error) { return "", fmt.Errorf("Config file not found") } +type DotenvStoreConfig struct{} + +type INIStoreConfig struct{} + +type JSONStoreConfig struct { + Indent int `yaml:"indent"` +} + +type JSONBinaryStoreConfig struct { + Indent int `yaml:"indent"` +} + +type YAMLStoreConfig struct { + Indent int `yaml:"indent"` +} + +type StoresConfig struct { + Dotenv DotenvStoreConfig `yaml:"dotenv"` + INI INIStoreConfig `yaml:"ini"` + JSONBinary JSONBinaryStoreConfig `yaml:"json_binary"` + JSON JSONStoreConfig `yaml:"json"` + YAML YAMLStoreConfig `yaml:"yaml"` +} + type configFile struct { CreationRules []creationRule `yaml:"creation_rules"` DestinationRules []destinationRule `yaml:"destination_rules"` + Stores StoresConfig `yaml:"stores"` } type keyGroup struct { @@ -126,6 +151,13 @@ type creationRule struct { MACOnlyEncrypted bool `yaml:"mac_only_encrypted"` } +func NewStoresConfig() *StoresConfig { + storesConfig := &StoresConfig{} + storesConfig.JSON.Indent = -1 + storesConfig.JSONBinary.Indent = -1 + return storesConfig +} + // Load loads a sops config file into a temporary struct func (f *configFile) load(bytes []byte) error { err := yaml.Unmarshal(bytes, f) @@ -229,6 +261,7 @@ func loadConfigFile(confPath string) (*configFile, error) { return nil, fmt.Errorf("could not read config file: %s", err) } conf := &configFile{} + conf.Stores = *NewStoresConfig() err = conf.load(confBytes) if err != nil { return nil, fmt.Errorf("error loading config: %s", err) @@ -386,3 +419,11 @@ func LoadDestinationRuleForFile(confPath string, filePath string, kmsEncryptionC } return parseDestinationRuleForFile(conf, filePath, kmsEncryptionContext) } + +func LoadStoresConfig(confPath string) (*StoresConfig, error) { + conf, err := loadConfigFile(confPath) + if err != nil { + return nil, err + } + return &conf.Stores, nil +} diff --git a/decrypt/decrypt.go b/decrypt/decrypt.go index c3b2ba64b..e26cbe479 100644 --- a/decrypt/decrypt.go +++ b/decrypt/decrypt.go @@ -12,6 +12,7 @@ import ( "github.com/getsops/sops/v3/aes" "github.com/getsops/sops/v3/cmd/sops/common" . "github.com/getsops/sops/v3/cmd/sops/formats" // Re-export + "github.com/getsops/sops/v3/config" ) // File is a wrapper around Data that reads a local encrypted @@ -32,7 +33,7 @@ func File(path, format string) (cleartext []byte, err error) { // decrypts the data and returns its cleartext in an []byte. func DataWithFormat(data []byte, format Format) (cleartext []byte, err error) { - store := common.StoreForFormat(format) + store := common.StoreForFormat(format, config.NewStoresConfig()) // Load SOPS file and access the data key tree, err := store.LoadEncryptedFile(data) diff --git a/stores/dotenv/store.go b/stores/dotenv/store.go index fad0f3494..db895fcd9 100644 --- a/stores/dotenv/store.go +++ b/stores/dotenv/store.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/getsops/sops/v3" + "github.com/getsops/sops/v3/config" "github.com/getsops/sops/v3/stores" ) @@ -16,6 +17,11 @@ const SopsPrefix = "sops_" // Store handles storage of dotenv data type Store struct { + config config.DotenvStoreConfig +} + +func NewStore(c *config.DotenvStoreConfig) *Store { + return &Store{config: *c} } // LoadEncryptedFile loads an encrypted file's bytes onto a sops.Tree runtime object diff --git a/stores/ini/store.go b/stores/ini/store.go index 6485467f1..d703d87cf 100644 --- a/stores/ini/store.go +++ b/stores/ini/store.go @@ -9,12 +9,18 @@ import ( "strings" "github.com/getsops/sops/v3" + "github.com/getsops/sops/v3/config" "github.com/getsops/sops/v3/stores" "gopkg.in/ini.v1" ) // Store handles storage of ini data. type Store struct { + config *config.INIStoreConfig +} + +func NewStore(c *config.INIStoreConfig) *Store { + return &Store{config: c} } func (store Store) encodeTree(branches sops.TreeBranches) ([]byte, error) { diff --git a/stores/json/store.go b/stores/json/store.go index 16a4b5d05..cbecd8362 100644 --- a/stores/json/store.go +++ b/stores/json/store.go @@ -6,18 +6,32 @@ import ( "errors" "fmt" "io" + "strings" "github.com/getsops/sops/v3" + "github.com/getsops/sops/v3/config" "github.com/getsops/sops/v3/stores" ) // Store handles storage of JSON data. type Store struct { + config config.JSONStoreConfig +} + +func NewStore(c *config.JSONStoreConfig) *Store { + return &Store{config: *c} } // BinaryStore handles storage of binary data in a JSON envelope. type BinaryStore struct { - store Store + store Store + config config.JSONBinaryStoreConfig +} + +func NewBinaryStore(c *config.JSONBinaryStoreConfig) *BinaryStore { + return &BinaryStore{config: *c, store: *NewStore(&config.JSONStoreConfig{ + Indent: c.Indent, + })} } // LoadEncryptedFile loads an encrypted json file onto a sops.Tree object @@ -237,7 +251,13 @@ func (store Store) treeBranchFromJSON(in []byte) (sops.TreeBranch, error) { func (store Store) reindentJSON(in []byte) ([]byte, error) { var out bytes.Buffer - err := json.Indent(&out, in, "", "\t") + indent := "\t" + if store.config.Indent > -1 { + indent = strings.Repeat(" ", store.config.Indent) + } else if store.config.Indent < -1 { + return nil, errors.New("JSON Indentation parameter smaller than -1 is not accepted") + } + err := json.Indent(&out, in, "", indent) return out.Bytes(), err } diff --git a/stores/json/store_test.go b/stores/json/store_test.go index 311276d0d..eca6d3e96 100644 --- a/stores/json/store_test.go +++ b/stores/json/store_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/getsops/sops/v3" + "github.com/getsops/sops/v3/config" "github.com/stretchr/testify/assert" ) @@ -312,7 +313,11 @@ func TestEncodeJSONArrayOfObjects(t *testing.T) { 2 ] }` - store := Store{} + store := Store{ + config: config.JSONStoreConfig{ + Indent: -1, + }, + } out, err := store.EmitPlainFile(tree.Branches) assert.Nil(t, err) assert.Equal(t, expected, string(out)) @@ -409,3 +414,129 @@ func TestEmitValueString(t *testing.T) { assert.Nil(t, err) assert.Equal(t, []byte("\"hello\""), bytes) } + +func TestIndentTwoSpaces(t *testing.T) { + tree := sops.Tree{ + Branches: sops.TreeBranches{ + sops.TreeBranch{ + sops.TreeItem{ + Key: "foo", + Value: []interface{}{ + sops.TreeBranch{ + sops.TreeItem{ + Key: "foo", + Value: 3, + }, + sops.TreeItem{ + Key: "bar", + Value: false, + }, + }, + 2, + }, + }, + }, + }, + } + expected := `{ + "foo": [ + { + "foo": 3, + "bar": false + }, + 2 + ] +}` + store := Store{ + config: config.JSONStoreConfig{ + Indent: 2, + }, + } + out, err := store.EmitPlainFile(tree.Branches) + assert.Nil(t, err) + assert.Equal(t, expected, string(out)) +} + +func TestIndentDefault(t *testing.T) { + tree := sops.Tree{ + Branches: sops.TreeBranches{ + sops.TreeBranch{ + sops.TreeItem{ + Key: "foo", + Value: []interface{}{ + sops.TreeBranch{ + sops.TreeItem{ + Key: "foo", + Value: 3, + }, + sops.TreeItem{ + Key: "bar", + Value: false, + }, + }, + 2, + }, + }, + }, + }, + } + expected := `{ + "foo": [ + { + "foo": 3, + "bar": false + }, + 2 + ] +}` + store := Store{ + config: config.JSONStoreConfig{ + Indent: -1, + }, + } + out, err := store.EmitPlainFile(tree.Branches) + assert.Nil(t, err) + assert.Equal(t, expected, string(out)) +} + +func TestNoIndent(t *testing.T) { + tree := sops.Tree{ + Branches: sops.TreeBranches{ + sops.TreeBranch{ + sops.TreeItem{ + Key: "foo", + Value: []interface{}{ + sops.TreeBranch{ + sops.TreeItem{ + Key: "foo", + Value: 3, + }, + sops.TreeItem{ + Key: "bar", + Value: false, + }, + }, + 2, + }, + }, + }, + }, + } + expected := `{ +"foo": [ +{ +"foo": 3, +"bar": false +}, +2 +] +}` + store := Store{ + config: config.JSONStoreConfig{ + Indent: 0, + }, + } + out, err := store.EmitPlainFile(tree.Branches) + assert.Nil(t, err) + assert.Equal(t, expected, string(out)) +} diff --git a/stores/yaml/store.go b/stores/yaml/store.go index 29fe2652a..4d036f366 100644 --- a/stores/yaml/store.go +++ b/stores/yaml/store.go @@ -2,17 +2,26 @@ package yaml //import "github.com/getsops/sops/v3/stores/yaml" import ( "bytes" + "errors" "fmt" "io" "strings" "github.com/getsops/sops/v3" + "github.com/getsops/sops/v3/config" "github.com/getsops/sops/v3/stores" "gopkg.in/yaml.v3" ) +const IndentDefault = 4 + // Store handles storage of YAML data type Store struct { + config config.YAMLStoreConfig +} + +func NewStore(c *config.YAMLStoreConfig) *Store { + return &Store{config: *c} } func (store Store) appendCommentToList(comment string, list []interface{}) []interface{} { @@ -318,12 +327,25 @@ func (store *Store) LoadPlainFile(in []byte) (sops.TreeBranches, error) { return branches, nil } +func (store *Store) getIndentation() (int, error) { + if store.config.Indent > 0 { + return store.config.Indent, nil + } else if store.config.Indent < 0 { + return 0, errors.New("YAML Negative indentation not accepted") + } + return IndentDefault, nil +} + // EmitEncryptedFile returns the encrypted bytes of the yaml file corresponding to a // sops.Tree runtime object func (store *Store) EmitEncryptedFile(in sops.Tree) ([]byte, error) { var b bytes.Buffer e := yaml.NewEncoder(io.Writer(&b)) - e.SetIndent(4) + indent, err := store.getIndentation() + if err != nil { + return nil, err + } + e.SetIndent(indent) for _, branch := range in.Branches { // Document root var doc = yaml.Node{} @@ -355,7 +377,11 @@ func (store *Store) EmitEncryptedFile(in sops.Tree) ([]byte, error) { func (store *Store) EmitPlainFile(branches sops.TreeBranches) ([]byte, error) { var b bytes.Buffer e := yaml.NewEncoder(io.Writer(&b)) - e.SetIndent(4) + indent, err := store.getIndentation() + if err != nil { + return nil, err + } + e.SetIndent(indent) for _, branch := range branches { // Document root var doc = yaml.Node{} diff --git a/stores/yaml/store_test.go b/stores/yaml/store_test.go index 4851068a3..41b3004d0 100644 --- a/stores/yaml/store_test.go +++ b/stores/yaml/store_test.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/getsops/sops/v3" + "github.com/getsops/sops/v3/config" "github.com/stretchr/testify/assert" ) @@ -211,6 +212,32 @@ e: - f `) +var INDENT_1_IN = []byte(`## Configuration for prometheus-node-exporter subchart +## +prometheus-node-exporter: + podLabels: + ## Add the 'node-exporter' label to be used by serviceMonitor to match standard common usage in rules and grafana dashboards + ## + + jobLabel: node-exporter + extraArgs: + - --collector.filesystem.ignored-mount-points=^/(dev|proc|sys|var/lib/docker/.+)($|/) + - --collector.filesystem.ignored-fs-types=^(autofs|binfmt_misc|cgroup|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|mqueue|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|sysfs|tracefs)$ +`) + +var INDENT_1_OUT = []byte(`## Configuration for prometheus-node-exporter subchart +## +prometheus-node-exporter: + podLabels: + ## Add the 'node-exporter' label to be used by serviceMonitor to match standard common usage in rules and grafana dashboards + ## + jobLabel: node-exporter + extraArgs: + - --collector.filesystem.ignored-mount-points=^/(dev|proc|sys|var/lib/docker/.+)($|/) + - --collector.filesystem.ignored-fs-types=^(autofs|binfmt_misc|cgroup|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|mqueue|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|sysfs|tracefs)$ +`) + + func TestUnmarshalMetadataFromNonSOPSFile(t *testing.T) { data := []byte(`hello: 2`) _, err := (&Store{}).LoadEncryptedFile(data) @@ -340,3 +367,17 @@ func TestComment7(t *testing.T) { assert.Equal(t, string(COMMENT_7_OUT), string(bytes)) assert.Equal(t, COMMENT_7_OUT, bytes) } + +func TestIndent1(t *testing.T) { + // First iteration: load and store + branches, err := (&Store{}).LoadPlainFile(INDENT_1_IN) + assert.Nil(t, err) + bytes, err := (&Store{ + config: config.YAMLStoreConfig{ + Indent: 2, + }, + }).EmitPlainFile(branches) + assert.Nil(t, err) + assert.Equal(t, string(INDENT_1_OUT), string(bytes)) + assert.Equal(t, INDENT_1_OUT, bytes) +} \ No newline at end of file