Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Backend] New filesystem backend #812

Open
wants to merge 14 commits into
base: master
Choose a base branch
from
2 changes: 1 addition & 1 deletion .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ go:
- 1.x
- tip
env:
- VAULT_ADDR='http://127.0.0.1:8200' CONSUL_VERSION=0.9.3 ETCD_VERSION=3.3.1 DYNAMODB_VERSION=2017-02-16 VAULT_VERSION=0.10.1 ZOOKEEPER_VERSION=3.4.10 RANCHER_VERSION=0.6.0
- VAULT_ADDR='http://127.0.0.1:8200' CONSUL_VERSION=0.9.3 ETCD_VERSION=3.3.1 DYNAMODB_VERSION=2017-02-16 VAULT_VERSION=0.10.1 ZOOKEEPER_VERSION=3.4.14 RANCHER_VERSION=0.6.0
services:
- redis
before_install:
Expand Down
3 changes: 3 additions & 0 deletions backends/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"github.com/kelseyhightower/confd/backends/etcd"
"github.com/kelseyhightower/confd/backends/etcdv3"
"github.com/kelseyhightower/confd/backends/file"
"github.com/kelseyhightower/confd/backends/filesystem"
"github.com/kelseyhightower/confd/backends/rancher"
"github.com/kelseyhightower/confd/backends/redis"
"github.com/kelseyhightower/confd/backends/ssm"
Expand Down Expand Up @@ -64,6 +65,8 @@ func New(config Config) (StoreClient, error) {
return env.NewEnvClient()
case "file":
return file.NewFileClient(config.YAMLFile, config.Filter)
case "filesystem":
return filesystem.NewFileSystemClient(config.MaxFileSize)
case "vault":
vaultConfig := map[string]string{
"app-id": config.AppID,
Expand Down
45 changes: 23 additions & 22 deletions backends/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,26 +5,27 @@ import (
)

type Config struct {
AuthToken string `toml:"auth_token"`
AuthType string `toml:"auth_type"`
Backend string `toml:"backend"`
BasicAuth bool `toml:"basic_auth"`
ClientCaKeys string `toml:"client_cakeys"`
ClientCert string `toml:"client_cert"`
ClientKey string `toml:"client_key"`
ClientInsecure bool `toml:"client_insecure"`
BackendNodes util.Nodes `toml:"nodes"`
Password string `toml:"password"`
Scheme string `toml:"scheme"`
Table string `toml:"table"`
Separator string `toml:"separator"`
Username string `toml:"username"`
AppID string `toml:"app_id"`
UserID string `toml:"user_id"`
RoleID string `toml:"role_id"`
SecretID string `toml:"secret_id"`
YAMLFile util.Nodes `toml:"file"`
Filter string `toml:"filter"`
Path string `toml:"path"`
Role string
AuthToken string `toml:"auth_token"`
AuthType string `toml:"auth_type"`
Backend string `toml:"backend"`
BasicAuth bool `toml:"basic_auth"`
ClientCaKeys string `toml:"client_cakeys"`
ClientCert string `toml:"client_cert"`
ClientKey string `toml:"client_key"`
ClientInsecure bool `toml:"client_insecure"`
BackendNodes util.Nodes `toml:"nodes"`
Password string `toml:"password"`
Scheme string `toml:"scheme"`
Table string `toml:"table"`
Separator string `toml:"separator"`
Username string `toml:"username"`
AppID string `toml:"app_id"`
UserID string `toml:"user_id"`
RoleID string `toml:"role_id"`
SecretID string `toml:"secret_id"`
YAMLFile util.Nodes `toml:"file"`
Filter string `toml:"filter"`
Path string `toml:"path"`
Role string
MaxFileSize int64 `toml:"max_file_size"`
}
2 changes: 1 addition & 1 deletion backends/etcd/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import (

"github.com/coreos/etcd/client"
"golang.org/x/net/context"
"github.com/kelseyhightower/confd/log"
"github.com/kelseyhightower/confd/log"
)

// Client is a wrapper around the etcd client
Expand Down
77 changes: 77 additions & 0 deletions backends/filesystem/client.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
package filesystem

import (
"fmt"
"io/ioutil"
"os"
"strings"

util "github.com/kelseyhightower/confd/util"
)

// Client provides a shell for the filesystem client
type Client struct {
MaxFileSize int64
}

// NewFilesystemClient returns a new client
func NewFileSystemClient(max int64) (*Client, error) {
return &Client{MaxFileSize: max}, nil
}

// GetValues queries the filesystem for keys
func (c *Client) GetValues(keys []string) (map[string]string, error) {
vars := make(map[string]string)
for _, key := range keys {
fpath := toPath(key)
stat, err := os.Stat(fpath)
if err != nil {
return nil, err
}

if stat.IsDir() {
// Walk subdirs
flist, err := util.RecursiveFilesLookup(fpath, "*")
if err != nil {
return nil, err
}

klist := make([]string, 0)
for _, v := range flist {
klist = append(klist, toKey(v))
}

// Recursively get values for sub-keys (fles)
recurseVars, err := c.GetValues(klist)
if err != nil {
return nil, err
}

// Add to our vars map
for k, v := range recurseVars {
vars[k] = v
}
} else {
if stat.Size() > c.MaxFileSize {
return nil, fmt.Errorf("size of %s too large", key)
}

// Read contents of file
b, err := ioutil.ReadFile(fpath)
if err != nil {
return nil, err
}

// Add to vars
vars[key] = strings.TrimSpace(string(b))

}
}

return vars, nil
}

func (c *Client) WatchPrefix(prefix string, keys []string, waitIndex uint64, stopChan chan bool) (uint64, error) {
<-stopChan
return 0, nil
}
15 changes: 15 additions & 0 deletions backends/filesystem/helpers_posix.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
// +build !windows

package filesystem

import (
"path/filepath"
)

func toPath(key string) string {
return filepath.FromSlash(key)
}

func toKey(path string) string {
return filepath.ToSlash(path)
}
37 changes: 37 additions & 0 deletions backends/filesystem/helpers_windows.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
package filesystem

import (
"os"
"path/filepath"
"strings"
)

func toPath(key string) string {
sep := string(os.PathSeparator)

// Replace slashes with path seperator
key = filepath.FromSlash(key)

// Trim leading backslash, so \c\path\to\file becomes c\path\to\file
key = strings.TrimPrefix(key, sep)

// Replace first slash with colon-slash so c\path\to\file becomes c:\path\to\file
key = strings.Replace(key, sep, ":"+sep, 1)

return key
}

func toKey(path string) string {

// Replace backslash with forward slash
path = filepath.ToSlash(path)

// Get rid of colon
path = strings.Replace(path, ":", "", 1)

if !strings.HasPrefix(path, "/") {
path = "/" + path
}

return path
}
3 changes: 2 additions & 1 deletion config.go
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ func init() {
flag.StringVar(&config.ClientCaKeys, "client-ca-keys", "", "client ca keys")
flag.StringVar(&config.ClientCert, "client-cert", "", "the client cert")
flag.StringVar(&config.ClientKey, "client-key", "", "the client key")
flag.BoolVar(&config.ClientInsecure, "client-insecure", false, "Allow connections to SSL sites without certs (only used with -backend=etcd)")
flag.BoolVar(&config.ClientInsecure, "client-insecure", false, "Allow connections to SSL sites without certs (only used with -backend=etcd)")
flag.StringVar(&config.ConfDir, "confdir", "/etc/confd", "confd conf directory")
flag.StringVar(&config.ConfigFile, "config-file", "/etc/confd/confd.toml", "the confd config file")
flag.Var(&config.YAMLFile, "file", "the YAML file to watch for changes (only used with -backend=file)")
Expand Down Expand Up @@ -72,6 +72,7 @@ func init() {
flag.StringVar(&config.Separator, "separator", "", "the separator to replace '/' with when looking up keys in the backend, prefixed '/' will also be removed (only used with -backend=redis)")
flag.StringVar(&config.Username, "username", "", "the username to authenticate as (only used with vault and etcd backends)")
flag.StringVar(&config.Password, "password", "", "the password to authenticate with (only used with vault and etcd backends)")
flag.Int64Var(&config.MaxFileSize, "max-file-size", 16384, "the maximum file size to read (only used with -backend=filesystem)")
flag.BoolVar(&config.Watch, "watch", false, "enable watch support")
}

Expand Down
1 change: 1 addition & 0 deletions config_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ func TestInitConfigDefaultConfig(t *testing.T) {
ConfigFile: "/etc/confd/confd.toml",
Interval: 600,
}
want.MaxFileSize = 16384
if err := initConfig(); err != nil {
t.Errorf(err.Error())
}
Expand Down
2 changes: 2 additions & 0 deletions docs/command-line-flags.md
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@ Usage of confd:
keep staged files
-log-level string
level which confd should log messages
-max-file-size integer
maximum file size to read (only used with -backend=filesystem) (default 16384)
-node value
list of backend nodes
-noop
Expand Down
1 change: 1 addition & 0 deletions docs/configuration-guide.md
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ Optional:
* `file` (array of strings) - The YAML file to watch for changes (only used with -backend=file).
* `filter` (string) - Files filter (only used with -backend=file) (default "*").
* `path` (string) - Vault mount path of the auth method (only used with -backend=vault).
* `max-file-size` (int64) - Maximum file size to read (only used with -backend=filesystem) (default 16384).

Example:

Expand Down
14 changes: 14 additions & 0 deletions docs/quick-start-guide.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ confd supports the following backends:
* vault
* environment variables
* file
* filesystem
* redis
* zookeeper
* dynamodb
Expand Down Expand Up @@ -58,6 +59,13 @@ myapp:
user: rob
```

#### filesystem

```
echo "db.example.com" > /myapp/database/url
echo "rob" > /myapp/database/user
```

#### redis

```
Expand Down Expand Up @@ -182,6 +190,12 @@ confd -onetime -backend env
confd -onetime -backend file -file myapp.yaml
```

#### filesystem

```
confd -onetime -backend filesystem
```

#### redis

```
Expand Down
27 changes: 27 additions & 0 deletions integration/filesystem/test.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
#!/bin/bash

export HOSTNAME="localhost"

sudo mkdir -p /database /upstream /nested/{east,west} /prefix/database /prefix/nested/{east,west} /prefix/upstream

echo "foobar" | sudo tee /key
echo "127.0.0.1" | sudo tee /database/host
echo "p@sSw0rd" | sudo tee /database/password
echo "3306" | sudo tee /database/port
echo "confd" | sudo tee /database/username
echo "confd" | sudo tee /database/username
echo "10.0.1.10:8080" | sudo tee /upstream/app1
echo "10.0.1.11:8080" | sudo tee /upstream/app2
echo "10.0.1.10:8080" | sudo tee /nested/east/app1
echo "10.0.1.11:8080" | sudo tee /nested/west/app2
echo "127.0.0.1" | sudo tee /prefix/database/host
echo "p@sSw0rd" | sudo tee /prefix/database/password
echo "3306" | sudo tee /prefix/database/port
echo "confd" | sudo tee /prefix/database/username
echo "10.0.1.10:8080" | sudo tee /prefix/upstream/app1
echo "10.0.1.11:8080" | sudo tee /prefix/upstream/app2
echo "10.0.1.10:8080" | sudo tee /prefix/nested/east/app1
echo "10.0.1.11:8080" | sudo tee /prefix/nested/west/app2

# Run confd
confd --onetime --log-level debug --confdir ./integration/confdir --backend filesystem --watch