Skip to content

Commit

Permalink
Merge pull request #1273 from Ph0tonic/main
Browse files Browse the repository at this point in the history
  • Loading branch information
hiddeco authored Nov 24, 2023
2 parents 2443097 + 42018ef commit 681a386
Show file tree
Hide file tree
Showing 12 changed files with 361 additions and 29 deletions.
37 changes: 34 additions & 3 deletions README.rst
Original file line number Diff line number Diff line change
Expand Up @@ -1130,15 +1130,15 @@ Below is an example of publishing to Vault (using token auth with a local dev in
Important information on types
------------------------------
YAML and JSON type extensions
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
YAML, JSON, ENV and INI type extensions
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
SOPS uses the file extension to decide which encryption method to use on the file
content. ``YAML``, ``JSON``, ``ENV``, and ``INI`` files are treated as trees of data, and key/values are
extracted from the files to only encrypt the leaf values. The tree structure is also
used to check the integrity of the file.
Therefore, if a file is encrypted using a specific format, it need to be decrypted
Therefore, if a file is encrypted using a specific format, it needs to be decrypted
in the same format. The easiest way to achieve this is to conserve the original file
extension after encrypting a file. For example:
Expand All @@ -1162,8 +1162,39 @@ When operating on stdin, use the ``--input-type`` and ``--output-type`` flags as
$ cat myfile.json | sops --input-type json --output-type json -d /dev/stdin
JSON and JSON_binary indentation
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
SOPS indents ``JSON`` files by default using one ``tab``. However, you can change
this default behaviour to use ``spaces`` by either using the additional ``--indent=2`` CLI option or
by configuring ``.sops.yaml`` with the code below.
The special value ``0`` disables indentation, and ``-1`` uses a single tab.
.. code:: yaml
stores:
json:
indent: 2
json_binary:
indent: 2
YAML indentation
~~~~~~~~~~~~~~~~
SOPS indents ``YAML`` files by default using 4 spaces. However, you can change
this default behaviour by either using the additional ``--indent=2`` CLI option or
by configuring ``.sops.yaml`` with:
.. code:: yaml
stores:
yaml:
indent: 2
YAML anchors
~~~~~~~~~~~~
SOPS only supports a subset of ``YAML``'s many types. Encrypting YAML files that
contain strings, numbers and booleans will work fine, but files that contain anchors
will not work, because the anchors redefine the structure of the file at load time.
Expand Down
35 changes: 18 additions & 17 deletions cmd/sops/common/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"github.com/getsops/sops/v3"
"github.com/getsops/sops/v3/cmd/sops/codes"
. "github.com/getsops/sops/v3/cmd/sops/formats"
"github.com/getsops/sops/v3/config"
"github.com/getsops/sops/v3/keys"
"github.com/getsops/sops/v3/keyservice"
"github.com/getsops/sops/v3/kms"
Expand All @@ -35,26 +36,26 @@ type Store interface {
ExampleFileEmitter
}

type storeConstructor = func() Store
type storeConstructor = func(*config.StoresConfig) Store

func newBinaryStore() Store {
return &json.BinaryStore{}
func newBinaryStore(c *config.StoresConfig) Store {
return json.NewBinaryStore(&c.JSONBinary)
}

func newDotenvStore() Store {
return &dotenv.Store{}
func newDotenvStore(c *config.StoresConfig) Store {
return dotenv.NewStore(&c.Dotenv)
}

func newIniStore() Store {
return &ini.Store{}
func newIniStore(c *config.StoresConfig) Store {
return ini.NewStore(&c.INI)
}

func newJsonStore() Store {
return &json.Store{}
func newJsonStore(c *config.StoresConfig) Store {
return json.NewStore(&c.JSON)
}

func newYamlStore() Store {
return &yaml.Store{}
func newYamlStore(c *config.StoresConfig) Store {
return yaml.NewStore(&c.YAML)
}

var storeConstructors = map[Format]storeConstructor{
Expand Down Expand Up @@ -153,27 +154,27 @@ func NewExitError(i interface{}, exitCode int) *cli.ExitError {

// StoreForFormat returns the correct format-specific implementation
// of the Store interface given the format.
func StoreForFormat(format Format) Store {
func StoreForFormat(format Format, c *config.StoresConfig) Store {
storeConst, found := storeConstructors[format]
if !found {
storeConst = storeConstructors[Binary] // default
}
return storeConst()
return storeConst(c)
}

// DefaultStoreForPath returns the correct format-specific implementation
// of the Store interface given the path to a file
func DefaultStoreForPath(path string) Store {
func DefaultStoreForPath(c *config.StoresConfig, path string) Store {
format := FormatForPath(path)
return StoreForFormat(format)
return StoreForFormat(format, c)
}

// DefaultStoreForPathOrFormat returns the correct format-specific implementation
// of the Store interface given the formatString if specified, or the path to a file.
// This is to support the cli, where both are provided.
func DefaultStoreForPathOrFormat(path, format string) Store {
func DefaultStoreForPathOrFormat(c *config.StoresConfig, path string, format string) Store {
formatFmt := FormatForPathOrString(path, format)
return StoreForFormat(formatFmt)
return StoreForFormat(formatFmt, c)
}

// KMS_ENC_CTX_BUG_FIXED_VERSION represents the SOPS version in which the
Expand Down
28 changes: 26 additions & 2 deletions cmd/sops/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -704,6 +704,10 @@ func main() {
Name: "shamir-secret-sharing-threshold",
Usage: "the number of master keys required to retrieve the data key with shamir",
},
cli.IntFlag{
Name: "indent",
Usage: "the number of spaces to indent YAML or JSON encoded file for encryption",
},
cli.BoolFlag{
Name: "verbose",
Usage: "Enable verbose logging output",
Expand Down Expand Up @@ -1065,12 +1069,32 @@ func keyservices(c *cli.Context) (svcs []keyservice.KeyServiceClient) {
return
}

func loadStoresConfig(context *cli.Context, path string) (*config.StoresConfig, error) {
var configPath string
if context.String("config") != "" {
configPath = context.String("config")
} else {
// Ignore config not found errors returned from FindConfigFile since the config file is not mandatory
configPath, _ = config.FindConfigFile(".")
}
return config.LoadStoresConfig(configPath)
}

func inputStore(context *cli.Context, path string) common.Store {
return common.DefaultStoreForPathOrFormat(path, context.String("input-type"))
storesConf, _ := loadStoresConfig(context, path)
return common.DefaultStoreForPathOrFormat(storesConf, path, context.String("input-type"))
}

func outputStore(context *cli.Context, path string) common.Store {
return common.DefaultStoreForPathOrFormat(path, context.String("output-type"))
storesConf, _ := loadStoresConfig(context, path)
if context.IsSet("indent") {
indent := context.Int("indent")
storesConf.YAML.Indent = indent
storesConf.JSON.Indent = indent
storesConf.JSONBinary.Indent = indent
}

return common.DefaultStoreForPathOrFormat(storesConf, path, context.String("output-type"))
}

func parseTreePath(arg string) ([]interface{}, error) {
Expand Down
6 changes: 5 additions & 1 deletion cmd/sops/subcommand/updatekeys/updatekeys.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,11 @@ func UpdateKeys(opts Opts) error {
}

func updateFile(opts Opts) error {
store := common.DefaultStoreForPathOrFormat(opts.InputPath, opts.InputType)
sc, err := config.LoadStoresConfig(opts.ConfigPath)
if err != nil {
return err
}
store := common.DefaultStoreForPath(sc, opts.InputPath)
log.Printf("Syncing keys for file %s", opts.InputPath)
tree, err := common.LoadEncryptedFile(store, opts.InputPath)
if err != nil {
Expand Down
41 changes: 41 additions & 0 deletions config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -63,9 +63,34 @@ func FindConfigFile(start string) (string, error) {
return "", fmt.Errorf("Config file not found")
}

type DotenvStoreConfig struct{}

type INIStoreConfig struct{}

type JSONStoreConfig struct {
Indent int `yaml:"indent"`
}

type JSONBinaryStoreConfig struct {
Indent int `yaml:"indent"`
}

type YAMLStoreConfig struct {
Indent int `yaml:"indent"`
}

type StoresConfig struct {
Dotenv DotenvStoreConfig `yaml:"dotenv"`
INI INIStoreConfig `yaml:"ini"`
JSONBinary JSONBinaryStoreConfig `yaml:"json_binary"`
JSON JSONStoreConfig `yaml:"json"`
YAML YAMLStoreConfig `yaml:"yaml"`
}

type configFile struct {
CreationRules []creationRule `yaml:"creation_rules"`
DestinationRules []destinationRule `yaml:"destination_rules"`
Stores StoresConfig `yaml:"stores"`
}

type keyGroup struct {
Expand Down Expand Up @@ -126,6 +151,13 @@ type creationRule struct {
MACOnlyEncrypted bool `yaml:"mac_only_encrypted"`
}

func NewStoresConfig() *StoresConfig {
storesConfig := &StoresConfig{}
storesConfig.JSON.Indent = -1
storesConfig.JSONBinary.Indent = -1
return storesConfig
}

// Load loads a sops config file into a temporary struct
func (f *configFile) load(bytes []byte) error {
err := yaml.Unmarshal(bytes, f)
Expand Down Expand Up @@ -229,6 +261,7 @@ func loadConfigFile(confPath string) (*configFile, error) {
return nil, fmt.Errorf("could not read config file: %s", err)
}
conf := &configFile{}
conf.Stores = *NewStoresConfig()
err = conf.load(confBytes)
if err != nil {
return nil, fmt.Errorf("error loading config: %s", err)
Expand Down Expand Up @@ -386,3 +419,11 @@ func LoadDestinationRuleForFile(confPath string, filePath string, kmsEncryptionC
}
return parseDestinationRuleForFile(conf, filePath, kmsEncryptionContext)
}

func LoadStoresConfig(confPath string) (*StoresConfig, error) {
conf, err := loadConfigFile(confPath)
if err != nil {
return nil, err
}
return &conf.Stores, nil
}
3 changes: 2 additions & 1 deletion decrypt/decrypt.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"github.com/getsops/sops/v3/aes"
"github.com/getsops/sops/v3/cmd/sops/common"
. "github.com/getsops/sops/v3/cmd/sops/formats" // Re-export
"github.com/getsops/sops/v3/config"
)

// File is a wrapper around Data that reads a local encrypted
Expand All @@ -32,7 +33,7 @@ func File(path, format string) (cleartext []byte, err error) {
// decrypts the data and returns its cleartext in an []byte.
func DataWithFormat(data []byte, format Format) (cleartext []byte, err error) {

store := common.StoreForFormat(format)
store := common.StoreForFormat(format, config.NewStoresConfig())

// Load SOPS file and access the data key
tree, err := store.LoadEncryptedFile(data)
Expand Down
6 changes: 6 additions & 0 deletions stores/dotenv/store.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"strings"

"github.com/getsops/sops/v3"
"github.com/getsops/sops/v3/config"
"github.com/getsops/sops/v3/stores"
)

Expand All @@ -16,6 +17,11 @@ const SopsPrefix = "sops_"

// Store handles storage of dotenv data
type Store struct {
config config.DotenvStoreConfig
}

func NewStore(c *config.DotenvStoreConfig) *Store {
return &Store{config: *c}
}

// LoadEncryptedFile loads an encrypted file's bytes onto a sops.Tree runtime object
Expand Down
6 changes: 6 additions & 0 deletions stores/ini/store.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,18 @@ import (
"strings"

"github.com/getsops/sops/v3"
"github.com/getsops/sops/v3/config"
"github.com/getsops/sops/v3/stores"
"gopkg.in/ini.v1"
)

// Store handles storage of ini data.
type Store struct {
config *config.INIStoreConfig
}

func NewStore(c *config.INIStoreConfig) *Store {
return &Store{config: c}
}

func (store Store) encodeTree(branches sops.TreeBranches) ([]byte, error) {
Expand Down
24 changes: 22 additions & 2 deletions stores/json/store.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,18 +6,32 @@ import (
"errors"
"fmt"
"io"
"strings"

"github.com/getsops/sops/v3"
"github.com/getsops/sops/v3/config"
"github.com/getsops/sops/v3/stores"
)

// Store handles storage of JSON data.
type Store struct {
config config.JSONStoreConfig
}

func NewStore(c *config.JSONStoreConfig) *Store {
return &Store{config: *c}
}

// BinaryStore handles storage of binary data in a JSON envelope.
type BinaryStore struct {
store Store
store Store
config config.JSONBinaryStoreConfig
}

func NewBinaryStore(c *config.JSONBinaryStoreConfig) *BinaryStore {
return &BinaryStore{config: *c, store: *NewStore(&config.JSONStoreConfig{
Indent: c.Indent,
})}
}

// LoadEncryptedFile loads an encrypted json file onto a sops.Tree object
Expand Down Expand Up @@ -237,7 +251,13 @@ func (store Store) treeBranchFromJSON(in []byte) (sops.TreeBranch, error) {

func (store Store) reindentJSON(in []byte) ([]byte, error) {
var out bytes.Buffer
err := json.Indent(&out, in, "", "\t")
indent := "\t"
if store.config.Indent > -1 {
indent = strings.Repeat(" ", store.config.Indent)
} else if store.config.Indent < -1 {
return nil, errors.New("JSON Indentation parameter smaller than -1 is not accepted")
}
err := json.Indent(&out, in, "", indent)
return out.Bytes(), err
}

Expand Down
Loading

0 comments on commit 681a386

Please sign in to comment.