diff --git a/README.md b/README.md index 5d9a677bb..8bdd9d797 100644 --- a/README.md +++ b/README.md @@ -2,74 +2,102 @@ [![Build Status](https://travis-ci.org/kelseyhightower/confd.png?branch=master)](https://travis-ci.org/kelseyhightower/confd) -* IRC: `#confd` on Freenode -* Mailing list: [Google Groups](https://groups.google.com/forum/#!forum/confd-users) - `confd` is a lightweight configuration management tool focused on: * keeping local configuration files up-to-date by polling [etcd](https://github.com/coreos/etcd) and processing [template resources](https://github.com/kelseyhightower/confd#template-resources). * reloading applications to pick up new config file changes -## Getting Started +## Community -### Installing confd +* IRC: `#confd` on Freenode +* Mailing list: [Google Groups](https://groups.google.com/forum/#!forum/confd-users) +* Website: [www.confd.io](http://www.confd.io) -Download the latest binary from [Github](https://github.com/kelseyhightower/confd/releases/tag/v0.1.2). +## Quick Start -### Building +Before we begin be sure to [download and install confd](https://github.com/kelseyhightower/confd/wiki/Installation). -You can build confd from source: +### Add keys to etcd + +This guide assumes you have a working [etcd](https://github.com/coreos/etcd#getting-started) server up and running and the ability to add new keys. Using the `etcdctl` command line tool add the following keys and values to etcd: ``` -git clone https://github.com/kelseyhightower/confd.git -cd confd -go build +etcdctl set /myapp/database/url db.example.com +etcdctl set /myapp/database/user rob ``` -This will produce the `confd` binary in the current directory. - -## Usage +### Create the confdir -The following commands will process all the [template resources](https://github.com/kelseyhightower/confd#template-resources) found under `/etc/confd/conf.d`. +The confdir is where template resource configs and source templates are stored. The default confdir is `/etc/confd`. Create the confdir by executing the following command: -### Poll the etcd cluster in 30 second intervals +```Bash +sudo mkdir -p /etc/confd/{conf.d,templates} +``` -The "/production" string will be prefixed to keys when querying etcd at http://127.0.0.1:4001. +You don't have to use the default `confdir` location. For example you can create the confdir under your home directory. Then you tell confd to use the new `confdir` via the `-confdir` flag. -``` -confd -i 30 -p '/production' -n 'http://127.0.0.1:4001' +```Bash +mkdir -p ~/confd/{conf.d,templates} ``` -### Single run without polling +### Create a template resource config -Using default settings run one time and exit. +Template resources are defined in [TOML](https://github.com/mojombo/toml) config files under the `confdir` (i.e. ~/confd/conf.d/*.toml). -``` -confd -onetime +Create the following template resource config and save it as `~/confd/conf.d/myconfig.toml`. + +```toml +[template] +src = "myconfig.conf.tmpl" +dest = "/tmp/myconfig.conf" +keys = [ + "/myapp/database/url", + "/myapp/database/user", +] ``` -### Client authentication +### Create the source template -Same as above but authenticate with client certificates. +Source templates are plain old [Golang text templates](http://golang.org/pkg/text/template/#pkg-overview), and are stored under the `confdir` templates directory. Create the following source template and save it as `~/confd/templates/myconfig.conf.tmpl` ``` -confd -onetime -key /etc/confd/ssl/client.key -cert /etc/confd/ssl/client.crt +# This a comment +[myconfig] +database_url = {{ .myapp_database_url }} +database_user = {{ .myapp_database_user }} ``` -## Configuration +### Processing template resources -See the [Configuration Guide](https://github.com/kelseyhightower/confd/wiki/Configuration-Guide) +confd supports two modes of operation, daemon and onetime mode. In daemon mode, confd runs in the foreground and processing template resources every 5 mins by default. For this tutorial we are going to use onetime mode. -## Template Resources +Assuming you etcd server is running at http://127.0.0.1:4001 you can run the following command to process the `~/confd/conf.d/myconfig.toml` template resource: -See [Template Resources](https://github.com/kelseyhightower/confd/wiki/Template-Resources) +``` +confd -verbose -onetime -node 'http://127.0.0.1:4001' -confdir ~/confd +``` +Output: +``` +2013-11-03T18:00:47-08:00 confd[21294]: NOTICE Starting confd +2013-11-03T18:00:47-08:00 confd[21294]: NOTICE etcd nodes set to http://127.0.0.1:4001 +2013-11-03T18:00:47-08:00 confd[21294]: INFO Target config /tmp/myconfig.conf out of sync +2013-11-03T18:00:47-08:00 confd[21294]: INFO Target config /tmp/myconfig.conf has been updated +``` -## Templates +The `dest` config should now be in sync with the template resource configuration. -See [Templates](https://github.com/kelseyhightower/confd/wiki/Templates) +``` +cat /tmp/myconfig.conf +``` -## Extras +Output: +``` +# This a comment +[myconfig] +database_url = db.example.com +database_user = rob +``` -### Configuration Management +## Next steps -- [confd-cookbook](https://github.com/rjocoleman/confd-cookbook) +Checkout the [confd wiki](https://github.com/kelseyhightower/confd/wiki/_pages) for more docs and [usage examples](https://github.com/kelseyhightower/confd/wiki/Usage-Examples). diff --git a/confd.go b/confd.go index 13e879705..833afe386 100644 --- a/confd.go +++ b/confd.go @@ -6,50 +6,56 @@ package main import ( "flag" "os" + "strings" "time" + "github.com/kelseyhightower/confd/config" + "github.com/kelseyhightower/confd/etcd/etcdutil" "github.com/kelseyhightower/confd/log" + "github.com/kelseyhightower/confd/resource/template" ) var ( configFile = "" defaultConfigFile = "/etc/confd/confd.toml" onetime bool - quiet bool ) func init() { - flag.StringVar(&configFile, "C", "", "confd config file") + flag.StringVar(&configFile, "config-file", "", "the confd config file") flag.BoolVar(&onetime, "onetime", false, "run once and exit") - flag.BoolVar(&quiet, "q", false, "silence non-error messages") } func main() { - // Most flags are defined in the confd/config package which allow us to - // override configuration settings from the cli. Parse the flags now to - // make them active. + // Most flags are defined in the confd/config package which allows us to + // override configuration settings from the command line. Parse the flags now + // to make them active. flag.Parse() - // non-error messages are not printed by default, enable them now. - // If the "-q" flag was passed on the commandline non-error messages will - // not be printed. - log.SetQuiet(quiet) - log.Info("Starting confd") if configFile == "" { if IsFileExist(defaultConfigFile) { configFile = defaultConfigFile } } - if err := loadConfig(configFile); err != nil { + // Initialize the global configuration. + log.Debug("Loading confd configuration") + if err := config.LoadConfig(configFile); err != nil { log.Fatal(err.Error()) } + // Configure logging. While you can enable debug and verbose logging, however + // if quiet is set to true then debug and verbose messages will not be printed. + log.SetQuiet(config.Quiet()) + log.SetVerbose(config.Verbose()) + log.SetDebug(config.Debug()) + log.Notice("Starting confd") // Create the etcd client upfront and use it for the life of the process. // The etcdClient is an http.Client and designed to be reused. - etcdClient, err := newEtcdClient(EtcdNodes(), ClientCert(), ClientKey()) + log.Notice("etcd nodes set to " + strings.Join(config.EtcdNodes(), ", ")) + etcdClient, err := etcdutil.NewEtcdClient(config.EtcdNodes(), config.ClientCert(), config.ClientKey()) if err != nil { log.Fatal(err.Error()) } for { - runErrors := ProcessTemplateResources(etcdClient) + runErrors := template.ProcessTemplateResources(etcdClient) // If the -onetime flag is passed on the command line we immediately exit // after processing the template config files. if onetime { @@ -58,7 +64,14 @@ func main() { } os.Exit(0) } - // By default we poll etcd every 30 seconds - time.Sleep(time.Duration(Interval()) * time.Second) + time.Sleep(time.Duration(config.Interval()) * time.Second) } } + +// IsFileExist reports whether path exits. +func IsFileExist(fpath string) bool { + if _, err := os.Stat(fpath); os.IsNotExist(err) { + return false + } + return true +} diff --git a/config.go b/config.go deleted file mode 100644 index b7fdd773c..000000000 --- a/config.go +++ /dev/null @@ -1,161 +0,0 @@ -// Copyright (c) 2013 Kelsey Hightower. All rights reserved. -// Use of this source code is governed by the Apache License, Version 2.0 -// that can be found in the LICENSE file. -package main - -import ( - "flag" - "fmt" - "path/filepath" - - "github.com/BurntSushi/toml" - "github.com/kelseyhightower/confd/log" -) - -var ( - config Config - nodes Nodes - confdir string - interval int - prefix string - clientCert string - clientKey string -) - -func init() { - flag.Var(&nodes, "n", "list of etcd nodes") - flag.StringVar(&confdir, "c", "/etc/confd", "confd config directory") - flag.IntVar(&interval, "i", 600, "etcd polling interval") - flag.StringVar(&prefix, "p", "/", "etcd key path prefix") - flag.StringVar(&clientCert, "cert", "", "the client cert") - flag.StringVar(&clientKey, "key", "", "the client key") -} - -// Nodes is a custom flag Var representing a list of etcd nodes. We use a custom -// Var to allow us to define more than one etcd node from the command line, and -// collect the results in a single value. -type Nodes []string - -func (n *Nodes) String() string { - return fmt.Sprintf("%d", *n) -} - -// Set appends the node to the etcd node list. -func (n *Nodes) Set(node string) error { - *n = append(*n, node) - return nil -} - -// Config represents the confd configuration settings. -type Config struct { - Confd confd -} - -// confd represents the parsed configuration settings. -type confd struct { - ConfDir string - ClientCert string `toml:"client_cert"` - ClientKey string `toml:"client_key"` - Interval int - Prefix string - EtcdNodes []string `toml:"etcd_nodes"` -} - -// loadConfig initializes the confd configuration by first setting defaults, -// then overriding setting from the confd config file, and finally overriding -// settings from flags set on the command line. -// It returns an error if any. -func loadConfig(path string) error { - setDefaults() - if path == "" { - log.Warning("Skipping confd config file.") - } else { - log.Debug("Loading " + path) - if err := loadConfFile(path); err != nil { - return err - } - } - overrideConfig() - return nil -} - -// ConfigDir returns the path to the confd config dir. -func ConfigDir() string { - return filepath.Join(config.Confd.ConfDir, "conf.d") -} - -// ClientCert returns the path to the client cert. -func ClientCert() string { - return config.Confd.ClientCert -} - -// ClientKey returns the path to the client key. -func ClientKey() string { - return config.Confd.ClientKey -} - -// EtcdNodes returns a list of etcd node url strings. -// For example: ["http://203.0.113.30:4001"] -func EtcdNodes() []string { - return config.Confd.EtcdNodes -} - -// Interval returns the number of seconds to wait between configuration runs. -func Interval() int { - return config.Confd.Interval -} - -// Prefix returns the etcd key prefix to use when querying etcd. -func Prefix() string { - return config.Confd.Prefix -} - -// TemplateDir returns the path to the directory of config file templates. -func TemplateDir() string { - return filepath.Join(config.Confd.ConfDir, "templates") -} - -func setDefaults() { - config = Config{ - Confd: confd{ - ConfDir: "/etc/confd", - Interval: 600, - Prefix: "/", - EtcdNodes: []string{"http://127.0.0.1:4001"}, - }, - } -} - -// loadConfFile sets the etcd configuration settings from a file. -func loadConfFile(path string) error { - _, err := toml.DecodeFile(path, &config) - if err != nil { - return err - } - return nil -} - -// override sets configuration settings based on values passed in through -// command line flags; overwriting current values. -func override(f *flag.Flag) { - switch f.Name { - case "c": - config.Confd.ConfDir = confdir - case "i": - config.Confd.Interval = interval - case "n": - config.Confd.EtcdNodes = nodes - case "p": - config.Confd.Prefix = prefix - case "cert": - config.Confd.ClientCert = clientCert - case "key": - config.Confd.ClientKey = clientKey - } -} - -// overrideConfig iterates through each flag set on the command line and -// overrides corresponding configuration settings. -func overrideConfig() { - flag.Visit(override) -} diff --git a/config/config.go b/config/config.go new file mode 100644 index 000000000..31761b59c --- /dev/null +++ b/config/config.go @@ -0,0 +1,272 @@ +// Copyright (c) 2013 Kelsey Hightower. All rights reserved. +// Use of this source code is governed by the Apache License, Version 2.0 +// that can be found in the LICENSE file. +package config + +import ( + "errors" + "flag" + "net" + "net/url" + "path/filepath" + "strconv" + + "github.com/BurntSushi/toml" + "github.com/kelseyhightower/confd/log" +) + +var ( + clientCert string + clientKey string + config Config // holds the global confd config. + confdir string + debug bool + etcdNodes Nodes + etcdScheme string + interval int + noop bool + prefix string + quiet bool + srvDomain string + verbose bool +) + +// Config represents the confd configuration settings. +type Config struct { + Confd confd +} + +// confd represents the parsed configuration settings. +type confd struct { + Debug bool `toml:"debug"` + ClientCert string `toml:"client_cert"` + ClientKey string `toml:"client_key"` + ConfDir string `toml:"confdir"` + EtcdNodes []string `toml:"etcd_nodes"` + EtcdScheme string `toml:"etcd_scheme"` + Interval int `toml:"interval"` + Noop bool `toml:"noop"` + Prefix string `toml:"prefix"` + Quiet bool `toml:"quiet"` + SRVDomain string `toml:"srv_domain"` + Verbose bool `toml:"verbose"` +} + +func init() { + flag.BoolVar(&debug, "debug", false, "enable debug logging") + flag.StringVar(&clientCert, "client-cert", "", "the client cert") + flag.StringVar(&clientKey, "client-key", "", "the client key") + flag.StringVar(&confdir, "confdir", "/etc/confd", "confd conf directory") + flag.Var(&etcdNodes, "node", "list of etcd nodes") + flag.StringVar(&etcdScheme, "etcd-scheme", "http", "the etcd URI scheme. (http or https)") + flag.IntVar(&interval, "interval", 600, "etcd polling interval") + flag.BoolVar(&noop, "noop", false, "only show pending changes, don't sync configs.") + flag.StringVar(&prefix, "prefix", "/", "etcd key path prefix") + flag.BoolVar(&quiet, "quiet", false, "enable quiet logging. Only error messages are printed.") + flag.StringVar(&srvDomain, "srv-domain", "", "the domain to query for the etcd SRV record, i.e. example.com") + flag.BoolVar(&verbose, "verbose", false, "enable verbose logging") +} + +// LoadConfig initializes the confd configuration by first setting defaults, +// then overriding setting from the confd config file, and finally overriding +// settings from flags set on the command line. +// It returns an error if any. +func LoadConfig(path string) error { + setDefaults() + if path == "" { + log.Warning("Skipping confd config file.") + } else { + log.Debug("Loading " + path) + _, err := toml.DecodeFile(path, &config) + if err != nil { + return err + } + } + processFlags() + if !isValidateEtcdScheme(config.Confd.EtcdScheme) { + return errors.New("Invalid etcd scheme: " + config.Confd.EtcdScheme) + } + err := setEtcdHosts() + if err != nil { + return err + } + return nil +} + +// Debug reports whether debug mode is enabled. +func Debug() bool { + return config.Confd.Debug +} + +// ClientCert returns the client cert path. +func ClientCert() string { + return config.Confd.ClientCert +} + +// ClientKey returns the client key path. +func ClientKey() string { + return config.Confd.ClientKey +} + +// ConfDir returns the path to the confd config dir. +func ConfDir() string { + return config.Confd.ConfDir +} + +// ConfigDir returns the path to the confd config dir. +func ConfigDir() string { + return filepath.Join(config.Confd.ConfDir, "conf.d") +} + +// EtcdNodes returns a list of etcd node url strings. +// For example: ["http://203.0.113.30:4001"] +func EtcdNodes() []string { + return config.Confd.EtcdNodes +} + +// Interval returns the number of seconds to wait between configuration runs. +func Interval() int { + return config.Confd.Interval +} + +// Noop reports whether noop mode is enabled. +func Noop() bool { + return config.Confd.Noop +} + +// Prefix returns the etcd key prefix to use when querying etcd. +func Prefix() string { + return config.Confd.Prefix +} + +// Quiet reports whether quiet mode is enabled. +func Quiet() bool { + return config.Confd.Quiet +} + +// Verbose reports whether verbose mode is enabled. +func Verbose() bool { + return config.Confd.Verbose +} + +// SetConfDir sets the confd conf dir. +func SetConfDir(path string) { + config.Confd.ConfDir = path +} + +// SetNoop sets noop. +func SetNoop(enabled bool) { + config.Confd.Noop = enabled +} + +// SetPrefix sets the key prefix. +func SetPrefix(prefix string) { + config.Confd.Prefix = prefix +} + +// SRVDomain returns the domain name used in etcd SRV record lookups. +func SRVDomain() string { + return config.Confd.SRVDomain +} + +// TemplateDir returns the template directory path. +func TemplateDir() string { + return filepath.Join(config.Confd.ConfDir, "templates") +} + +func setDefaults() { + config = Config{ + Confd: confd{ + ConfDir: "/etc/confd", + Interval: 600, + Prefix: "/", + EtcdNodes: []string{"127.0.0.1:4001"}, + EtcdScheme: "http", + }, + } +} + +// setEtcdHosts. +func setEtcdHosts() error { + scheme := config.Confd.EtcdScheme + hosts := make([]string, 0) + // If a domain name is given then lookup the etcd SRV record, and override + // all other etcd node settings. + if config.Confd.SRVDomain != "" { + log.Info("SRV domain set to " + config.Confd.SRVDomain) + etcdHosts, err := getEtcdHostsFromSRV(config.Confd.SRVDomain) + if err != nil { + return errors.New("Cannot get etcd hosts from SRV records " + err.Error()) + } + for _, h := range etcdHosts { + uri := formatEtcdHostURL(scheme, h.Hostname, strconv.FormatUint(uint64(h.Port), 10)) + hosts = append(hosts, uri) + } + config.Confd.EtcdNodes = hosts + return nil + } + // No domain name was given, so just process the etcd node list. + // An etcdNode can be a URL, http://etcd.example.com:4001, or a host, etcd.example.com:4001. + for _, node := range config.Confd.EtcdNodes { + etcdURL, err := url.Parse(node) + if err != nil { + log.Error(err.Error()) + return err + } + if etcdURL.Scheme != "" && etcdURL.Host != "" { + if !isValidateEtcdScheme(etcdURL.Scheme) { + return errors.New("The etcd node list contains an invalid URL: " + node) + } + host, port, err := net.SplitHostPort(etcdURL.Host) + if err != nil { + return err + } + hosts = append(hosts, formatEtcdHostURL(etcdURL.Scheme, host, port)) + continue + } + // At this point node is not an etcd URL, i.e. http://etcd.example.com:4001, + // but a host:port string, i.e. etcd.example.com:4001 + host, port, err := net.SplitHostPort(node) + if err != nil { + return err + } + hosts = append(hosts, formatEtcdHostURL(scheme, host, port)) + } + config.Confd.EtcdNodes = hosts + return nil +} + +// processFlags iterates through each flag set on the command line and +// overrides corresponding configuration settings. +func processFlags() { + flag.Visit(setConfigFromFlag) +} + +func setConfigFromFlag(f *flag.Flag) { + switch f.Name { + case "debug": + config.Confd.Debug = debug + case "client-cert": + config.Confd.ClientCert = clientCert + case "client-key": + config.Confd.ClientKey = clientKey + case "confdir": + config.Confd.ConfDir = confdir + case "node": + config.Confd.EtcdNodes = etcdNodes + case "etcd-scheme": + config.Confd.EtcdScheme = etcdScheme + case "interval": + config.Confd.Interval = interval + case "noop": + config.Confd.Noop = noop + case "prefix": + config.Confd.Prefix = prefix + case "quiet": + config.Confd.Quiet = quiet + case "srv-domain": + config.Confd.SRVDomain = srvDomain + case "verbose": + config.Confd.Verbose = verbose + } +} diff --git a/config_test.go b/config/config_test.go similarity index 82% rename from config_test.go rename to config/config_test.go index c091c5f42..1708ff32a 100644 --- a/config_test.go +++ b/config/config_test.go @@ -1,10 +1,15 @@ -package main +// Copyright (c) 2013 Kelsey Hightower. All rights reserved. +// Use of this source code is governed by the Apache License, Version 2.0 +// that can be found in the LICENSE file. +package config import ( "testing" + "github.com/kelseyhightower/confd/log" ) func TestLoadConfig(t *testing.T) { + log.SetQuiet(true) var expected = struct { clientCert string clientKey string @@ -17,7 +22,7 @@ func TestLoadConfig(t *testing.T) { "", "", "/etc/confd/conf.d", []string{"http://127.0.0.1:4001"}, 600, "/", "/etc/confd/templates", } - loadConfig("") + LoadConfig("") cc := ClientCert() if cc != expected.clientCert { t.Errorf("Expected default clientCert = %s, got %s", expected.clientCert, cc) diff --git a/config/node_var.go b/config/node_var.go new file mode 100644 index 000000000..03bd93885 --- /dev/null +++ b/config/node_var.go @@ -0,0 +1,22 @@ +// Copyright (c) 2013 Kelsey Hightower. All rights reserved. +// Use of this source code is governed by the Apache License, Version 2.0 +// that can be found in the LICENSE file. +package config + +import ( + "fmt" +) + +// Nodes is a custom flag Var representing a list of etcd nodes. +type Nodes []string + +// String returns the string representation of a node var. +func (n *Nodes) String() string { + return fmt.Sprintf("%d", *n) +} + +// Set appends the node to the etcd node list. +func (n *Nodes) Set(node string) error { + *n = append(*n, node) + return nil +} diff --git a/config/util.go b/config/util.go new file mode 100644 index 000000000..019708acb --- /dev/null +++ b/config/util.go @@ -0,0 +1,63 @@ +// Copyright (c) 2013 Kelsey Hightower. All rights reserved. +// Use of this source code is governed by the Apache License, Version 2.0 +// that can be found in the LICENSE file. +package config + +import ( + "fmt" + "net" + "strings" +) + +// etcdHost +type etcdHost struct { + Hostname string + Port uint16 +} + +// etcdHostsFromSRV converts an etcd SRV record to a list of etcdHost. +func etcdHostsFromSRV(addrs []*net.SRV) []*etcdHost { + hosts := make([]*etcdHost, 0) + for _, srv := range addrs { + hostname := strings.TrimRight(srv.Target, ".") + hosts = append(hosts, &etcdHost{Hostname: hostname, Port: srv.Port}) + } + return hosts +} + +// formatEtcdHostURL. +func formatEtcdHostURL(scheme, host, port string) string { + return fmt.Sprintf("%s://%s:%s", scheme, host, port) +} + +// getEtcdHostsFromSRV returns a list of etcHost. +func getEtcdHostsFromSRV(domain string) ([]*etcdHost, error) { + addrs, err := lookupEtcdSRV(domain) + if err != nil { + return nil, err + } + etcdHosts := etcdHostsFromSRV(addrs) + return etcdHosts, nil +} + +// lookupEtcdSrv tries to resolve an SRV query for the etcd service for the +// specified domain. +// +// lookupEtcdSRV constructs the DNS name to look up following RFC 2782. +// That is, it looks up _etcd._tcp.domain. +func lookupEtcdSRV(domain string) ([]*net.SRV, error) { + // Ignore the CNAME as we don't need it. + _, addrs, err := net.LookupSRV("etcd", "tcp", domain) + if err != nil { + return addrs, err + } + return addrs, nil +} + +// isValidateEtcdScheme. +func isValidateEtcdScheme(scheme string) bool { + if scheme == "http" || scheme == "https" { + return true + } + return false +} diff --git a/etcdtest/client.go b/etcd/etcdtest/client.go similarity index 78% rename from etcdtest/client.go rename to etcd/etcdtest/client.go index 5b6e6204d..815c4c80b 100644 --- a/etcdtest/client.go +++ b/etcd/etcdtest/client.go @@ -1,3 +1,6 @@ +// Copyright (c) 2013 Kelsey Hightower. All rights reserved. +// Use of this source code is governed by the Apache License, Version 2.0 +// that can be found in the LICENSE file. package etcdtest import ( diff --git a/etcd/etcdtest/client_test.go b/etcd/etcdtest/client_test.go new file mode 100644 index 000000000..4379c8d1a --- /dev/null +++ b/etcd/etcdtest/client_test.go @@ -0,0 +1,4 @@ +// Copyright (c) 2013 Kelsey Hightower. All rights reserved. +// Use of this source code is governed by the Apache License, Version 2.0 +// that can be found in the LICENSE file. +package etcdtest diff --git a/etcd_client.go b/etcd/etcdutil/client.go similarity index 80% rename from etcd_client.go rename to etcd/etcdutil/client.go index 0eecd1c85..b3d29d26e 100644 --- a/etcd_client.go +++ b/etcd/etcdutil/client.go @@ -1,4 +1,7 @@ -package main +// Copyright (c) 2013 Kelsey Hightower. All rights reserved. +// Use of this source code is governed by the Apache License, Version 2.0 +// that can be found in the LICENSE file. +package etcdutil import ( "errors" @@ -9,9 +12,9 @@ import ( var replacer = strings.NewReplacer("/", "_") -// newEtcdClient returns an *etcd.Client with a connection to named machines. +// NewEtcdClient returns an *etcd.Client with a connection to named machines. // It returns an error if a connection to the cluster cannot be made. -func newEtcdClient(machines []string, cert, key string) (*etcd.Client, error) { +func NewEtcdClient(machines []string, cert, key string) (*etcd.Client, error) { c := etcd.NewClient(machines) if cert != "" && key != "" { _, err := c.SetCertAndKey(cert, key) @@ -30,13 +33,13 @@ type EtcdClient interface { Get(key string) ([]*etcd.Response, error) } -// getValues queries etcd for keys prefixed by prefix. +// GetValues queries etcd for keys prefixed by prefix. // Etcd paths (keys) are translated into names more suitable for use in // templates. For example if prefix where set to '/production' and one of the // keys where '/nginx/port'; the prefixed '/production/nginx/port' key would // be quired for. If the value for the prefixed key where 80, the returned map // would contain the entry vars["nginx_port"] = "80". -func getValues(c EtcdClient, prefix string, keys []string) (map[string]interface{}, error) { +func GetValues(c EtcdClient, prefix string, keys []string) (map[string]interface{}, error) { vars := make(map[string]interface{}) for _, key := range keys { err := etcdWalk(c, filepath.Join(prefix, key), prefix, vars) diff --git a/etcd_client_test.go b/etcd/etcdutil/client_test.go similarity index 86% rename from etcd_client_test.go rename to etcd/etcdutil/client_test.go index c64df53c7..42571066e 100644 --- a/etcd_client_test.go +++ b/etcd/etcdutil/client_test.go @@ -1,8 +1,11 @@ -package main +// Copyright (c) 2013 Kelsey Hightower. All rights reserved. +// Use of this source code is governed by the Apache License, Version 2.0 +// that can be found in the LICENSE file. +package etcdutil import ( "github.com/coreos/go-etcd/etcd" - "github.com/kelseyhightower/confd/etcdtest" + "github.com/kelseyhightower/confd/etcd/etcdtest" "testing" ) @@ -50,7 +53,7 @@ func TestGetValues(t *testing.T) { c.AddResponse("/foo/three", fooThreeResp) c.AddResponse("/nginx", nginxResp) keys := []string{"/nginx", "/foo"} - values, err := getValues(c, "", keys) + values, err := GetValues(c, "", keys) if err != nil { t.Error(err.Error()) } diff --git a/fileinfo_darwin.go b/fileinfo_darwin.go deleted file mode 100644 index 4242f1d9c..000000000 --- a/fileinfo_darwin.go +++ /dev/null @@ -1,9 +0,0 @@ -package main - -// A fileInfo describes a configuration file and is returned by fileStat. -type fileInfo struct { - Uid uint32 - Gid uint32 - Mode uint16 - Md5 string -} diff --git a/fileinfo_linux.go b/fileinfo_linux.go deleted file mode 100644 index 69355d04f..000000000 --- a/fileinfo_linux.go +++ /dev/null @@ -1,9 +0,0 @@ -package main - -// A fileInfo describes a configuration file and is returned by fileStat. -type fileInfo struct { - Uid uint32 - Gid uint32 - Mode uint32 - Md5 string -} diff --git a/log/log.go b/log/log.go index e284c5a55..1324f9fe5 100644 --- a/log/log.go +++ b/log/log.go @@ -21,8 +21,11 @@ import ( // string will appear in all log entires. var tag string -// Silence non-error messages. -var quiet = true +var ( + quiet = false // Silence non-error messages. + verbose = false + debug = false +) func init() { tag = os.Args[0] @@ -34,13 +37,25 @@ func SetTag(t string) { } // SetQuiet sets quite mode. -func SetQuiet(q bool) { - quiet = q +func SetQuiet(enable bool) { + quiet = enable +} + +// SetDebug sets debug mode. +func SetDebug(enable bool) { + debug = enable +} + +// SetVerbose sets verbose mode. +func SetVerbose(enable bool) { + verbose = enable } // Debug logs a message with severity DEBUG. func Debug(msg string) { - write("DEBUG", msg) + if debug { + write("DEBUG", msg) + } } // Error logs a message with severity ERROR. @@ -61,7 +76,9 @@ func Info(msg string) { // Notice logs a message with severity NOTICE. func Notice(msg string) { - write("NOTICE", msg) + if verbose || debug { + write("NOTICE", msg) + } } // Warning logs a message with severity WARNING. @@ -78,9 +95,10 @@ func write(level, msg string) { hostname, _ := os.Hostname() switch level { case "DEBUG", "INFO", "NOTICE", "WARNING": - if !quiet { - w = os.Stdout + if quiet { + return } + w = os.Stdout case "ERROR": w = os.Stderr } diff --git a/log/log_test.go b/log/log_test.go index 7330d5405..a4fe73d82 100644 --- a/log/log_test.go +++ b/log/log_test.go @@ -1 +1,4 @@ +// Copyright (c) 2013 Kelsey Hightower. All rights reserved. +// Use of this source code is governed by the Apache License, Version 2.0 +// that can be found in the LICENSE file. package log diff --git a/resource/template/fileinfo_darwin.go b/resource/template/fileinfo_darwin.go new file mode 100644 index 000000000..40baa96fc --- /dev/null +++ b/resource/template/fileinfo_darwin.go @@ -0,0 +1,12 @@ +// Copyright (c) 2013 Kelsey Hightower. All rights reserved. +// Use of this source code is governed by the Apache License, Version 2.0 +// that can be found in the LICENSE file. +package template + +// fileInfo describes a configuration file and is returned by fileStat. +type fileInfo struct { + Uid uint32 + Gid uint32 + Mode uint16 + Md5 string +} diff --git a/resource/template/fileinfo_linux.go b/resource/template/fileinfo_linux.go new file mode 100644 index 000000000..65f3942da --- /dev/null +++ b/resource/template/fileinfo_linux.go @@ -0,0 +1,12 @@ +// Copyright (c) 2013 Kelsey Hightower. All rights reserved. +// Use of this source code is governed by the Apache License, Version 2.0 +// that can be found in the LICENSE file. +package template + +// fileInfo describes a configuration file and is returned by fileStat. +type fileInfo struct { + Uid uint32 + Gid uint32 + Mode uint32 + Md5 string +} diff --git a/template_resource.go b/resource/template/resource.go similarity index 78% rename from template_resource.go rename to resource/template/resource.go index 614ee40a9..5ddb2c3af 100644 --- a/template_resource.go +++ b/resource/template/resource.go @@ -1,12 +1,13 @@ -package main +// Copyright (c) 2013 Kelsey Hightower. All rights reserved. +// Use of this source code is governed by the Apache License, Version 2.0 +// that can be found in the LICENSE file. +package template import ( "bytes" "crypto/md5" "errors" "fmt" - "github.com/BurntSushi/toml" - "github.com/kelseyhightower/confd/log" "io" "io/ioutil" "os" @@ -15,6 +16,11 @@ import ( "strconv" "syscall" "text/template" + + "github.com/BurntSushi/toml" + "github.com/kelseyhightower/confd/config" + "github.com/kelseyhightower/confd/etcd/etcdutil" + "github.com/kelseyhightower/confd/log" ) // TemplateResourceConfig holds the parsed template resource. @@ -35,17 +41,18 @@ type TemplateResource struct { StageFile *os.File Src string Vars map[string]interface{} - etcdClient EtcdClient + etcdClient etcdutil.EtcdClient } // NewTemplateResourceFromPath creates a TemplateResource using a decoded file path // and the supplied EtcdClient as input. // It returns a TemplateResource and an error if any. -func NewTemplateResourceFromPath(path string, c EtcdClient) (*TemplateResource, error) { +func NewTemplateResourceFromPath(path string, c etcdutil.EtcdClient) (*TemplateResource, error) { if c == nil { return nil, errors.New("A valid EtcdClient is required.") } var tc *TemplateResourceConfig + log.Debug("Loading template resource from " + path) _, err := toml.DecodeFile(path, &tc) if err != nil { return nil, fmt.Errorf("Cannot process template resource %s - %s", path, err.Error()) @@ -57,7 +64,9 @@ func NewTemplateResourceFromPath(path string, c EtcdClient) (*TemplateResource, // setVars sets the Vars for template resource. func (t *TemplateResource) setVars() error { var err error - t.Vars, err = getValues(t.etcdClient, Prefix(), t.Keys) + log.Debug("Retrieving keys from etcd") + log.Debug("Key prefix set to " + config.Prefix()) + t.Vars, err = etcdutil.GetValues(t.etcdClient, config.Prefix(), t.Keys) if err != nil { return err } @@ -69,8 +78,9 @@ func (t *TemplateResource) setVars() error { // StageFile for the template resource. // It returns an error if any. func (t *TemplateResource) createStageFile() error { - t.Src = filepath.Join(TemplateDir(), t.Src) - if !IsFileExist(t.Src) { + t.Src = filepath.Join(config.TemplateDir(), t.Src) + log.Debug("Using source template " + t.Src) + if !isFileExist(t.Src) { return errors.New("Missing template: " + t.Src) } temp, err := ioutil.TempFile("", "") @@ -78,6 +88,7 @@ func (t *TemplateResource) createStageFile() error { os.Remove(temp.Name()) return err } + log.Debug("Compiling source template " + t.Src) tmpl := template.Must(template.ParseFiles(t.Src)) if err = tmpl.Execute(temp, t.Vars); err != nil { return err @@ -98,17 +109,23 @@ func (t *TemplateResource) createStageFile() error { func (t *TemplateResource) sync() error { staged := t.StageFile.Name() defer os.Remove(staged) + log.Debug("Comparing candidate config to " + t.Dest) ok, err := sameConfig(staged, t.Dest) if err != nil { log.Error(err.Error()) } + if config.Noop() { + log.Warning("Noop mode enabled " + t.Dest + " will not be modified") + return nil + } if !ok { - log.Info("syncing " + t.Dest) + log.Info("Target config " + t.Dest + " out of sync") if t.CheckCmd != "" { if err := t.check(); err != nil { return errors.New("Config check failed: " + err.Error()) } } + log.Debug("Overwriting target config " + t.Dest) if err := os.Rename(staged, t.Dest); err != nil { return err } @@ -117,8 +134,9 @@ func (t *TemplateResource) sync() error { return err } } + log.Info("Target config " + t.Dest + " has been updated") } else { - log.Info(t.Dest + " in sync") + log.Info("Target config " + t.Dest + " in sync") } return nil } @@ -181,10 +199,9 @@ func (t *TemplateResource) process() error { } // setFileMode sets the FileMode. -// It returns an error if any. func (t *TemplateResource) setFileMode() error { if t.Mode == "" { - if !IsFileExist(t.Dest) { + if !isFileExist(t.Dest) { t.FileMode = 0644 } else { fi, err := os.Stat(t.Dest) @@ -205,20 +222,26 @@ func (t *TemplateResource) setFileMode() error { // ProcessTemplateResources is a convenience function that loads all the // template resources and processes them serially. Called from main. -// It return an error if any. -func ProcessTemplateResources(c EtcdClient) []error { +// It returns a list of errors if any. +func ProcessTemplateResources(c etcdutil.EtcdClient) []error { runErrors := make([]error, 0) var err error if c == nil { runErrors = append(runErrors, errors.New("An etcd client is required")) return runErrors } - paths, err := filepath.Glob(filepath.Join(ConfigDir(), "*toml")) + log.Debug("Loading template resources from confdir " + config.ConfDir()) + if !isFileExist(config.ConfDir()) { + log.Warning(fmt.Sprintf("Cannot load template resources confdir '%s' does not exist", config.ConfDir())) + return runErrors + } + paths, err := filepath.Glob(filepath.Join(config.ConfigDir(), "*toml")) if err != nil { runErrors = append(runErrors, err) return runErrors } for _, p := range paths { + log.Debug("Processing template resource " + p) t, err := NewTemplateResourceFromPath(p, c) if err != nil { runErrors = append(runErrors, err) @@ -230,13 +253,14 @@ func ProcessTemplateResources(c EtcdClient) []error { log.Error(err.Error()) continue } + log.Debug("Processing of template resource " + p + " complete") } return runErrors } // fileStat return a fileInfo describing the named file. func fileStat(name string) (fi fileInfo, err error) { - if IsFileExist(name) { + if isFileExist(name) { f, err := os.Open(name) defer f.Close() if err != nil { @@ -260,7 +284,7 @@ func fileStat(name string) (fi fileInfo, err error) { // Unix permissions. The owner, group, and mode must match. // It return false in other cases. func sameConfig(src, dest string) (bool, error) { - if !IsFileExist(dest) { + if !isFileExist(dest) { return false, nil } d, err := fileStat(dest) @@ -288,3 +312,11 @@ func sameConfig(src, dest string) (bool, error) { } return true, nil } + +// isFileExist reports whether path exits. +func isFileExist(fpath string) bool { + if _, err := os.Stat(fpath); os.IsNotExist(err) { + return false + } + return true +} diff --git a/template_resource_test.go b/resource/template/resource_test.go similarity index 59% rename from template_resource_test.go rename to resource/template/resource_test.go index 57ee17ae9..f64d497d1 100644 --- a/template_resource_test.go +++ b/resource/template/resource_test.go @@ -1,13 +1,19 @@ -package main +// Copyright (c) 2013 Kelsey Hightower. All rights reserved. +// Use of this source code is governed by the Apache License, Version 2.0 +// that can be found in the LICENSE file. +package template import ( - "github.com/coreos/go-etcd/etcd" - "github.com/kelseyhightower/confd/etcdtest" "io/ioutil" "os" "path/filepath" "testing" "text/template" + + "github.com/coreos/go-etcd/etcd" + "github.com/kelseyhightower/confd/config" + "github.com/kelseyhightower/confd/etcd/etcdtest" + "github.com/kelseyhightower/confd/log" ) // createTempDirs is a helper function which creates temporary directories @@ -30,6 +36,8 @@ func createTempDirs() (string, error) { return confDir, nil } +var fakeFile = "/this/shoud/not/exist" + var templateResourceConfigTmpl = ` [template] src = "{{ .src }}" @@ -50,6 +58,7 @@ keys = [ ` func TestProcessTemplateResources(t *testing.T) { + log.SetQuiet(true) // Setup temporary conf, config, and template directories. tempConfDir, err := createTempDirs() if err != nil { @@ -90,12 +99,12 @@ func TestProcessTemplateResources(t *testing.T) { } // Load the confd configuration settings. - if err := loadConfig(""); err != nil { + if err := config.LoadConfig(""); err != nil { t.Errorf(err.Error()) } - config.Confd.Prefix = "" + config.SetPrefix("") // Use the temporary tempConfDir from above. - config.Confd.ConfDir = tempConfDir + config.SetConfDir(tempConfDir) // Create the stub etcd client. c := etcdtest.NewClient() @@ -122,7 +131,84 @@ func TestProcessTemplateResources(t *testing.T) { } } +func TestProcessTemplateResourcesNoop(t *testing.T) { + log.SetQuiet(true) + // Setup temporary conf, config, and template directories. + tempConfDir, err := createTempDirs() + if err != nil { + t.Errorf("Failed to create temp dirs: %s", err.Error()) + } + defer os.RemoveAll(tempConfDir) + + // Create the src template. + srcTemplateFile := filepath.Join(tempConfDir, "templates", "foo.tmpl") + err = ioutil.WriteFile(srcTemplateFile, []byte("foo = {{ .foo }}"), 0644) + if err != nil { + t.Error(err.Error()) + } + + // Create the dest. + destFile, err := ioutil.TempFile("", "") + if err != nil { + t.Errorf("Failed to create destFile: %s", err.Error()) + } + defer os.Remove(destFile.Name()) + + // Create the template resource configuration file. + templateResourcePath := filepath.Join(tempConfDir, "conf.d", "foo.toml") + templateResourceFile, err := os.Create(templateResourcePath) + if err != nil { + t.Errorf(err.Error()) + } + tmpl, err := template.New("templateResourceConfig").Parse(templateResourceConfigTmpl) + if err != nil { + t.Errorf("Unable to parse template resource template: %s", err.Error()) + } + data := make(map[string]string) + data["src"] = "foo.tmpl" + data["dest"] = destFile.Name() + err = tmpl.Execute(templateResourceFile, data) + if err != nil { + t.Errorf(err.Error()) + } + + // Load the confd configuration settings. + if err := config.LoadConfig(""); err != nil { + t.Errorf(err.Error()) + } + config.SetPrefix("") + // Use the temporary tempConfDir from above. + config.SetConfDir(tempConfDir) + // Enable noop mode. + config.SetNoop(true) + + // Create the stub etcd client. + c := etcdtest.NewClient() + fooResp := []*etcd.Response{ + {Key: "/foo", Dir: false, Value: "bar"}, + } + c.AddResponse("/foo", fooResp) + + // Process the test template resource. + runErrors := ProcessTemplateResources(c) + if len(runErrors) > 0 { + for _, e := range runErrors { + t.Errorf(e.Error()) + } + } + // Verify the results. + expected := "" + results, err := ioutil.ReadFile(destFile.Name()) + if err != nil { + t.Error(err.Error()) + } + if string(results) != expected { + t.Errorf("Expected contents of dest == '%s', got %s", expected, string(results)) + } +} + func TestBrokenTemplateResourceFile(t *testing.T) { + log.SetQuiet(true) tempFile, err := ioutil.TempFile("", "") defer os.Remove(tempFile.Name()) if err != nil { @@ -142,6 +228,7 @@ func TestBrokenTemplateResourceFile(t *testing.T) { } func TestSameConfigTrue(t *testing.T) { + log.SetQuiet(true) src, err := ioutil.TempFile("", "src") defer os.Remove(src.Name()) if err != nil { @@ -170,6 +257,7 @@ func TestSameConfigTrue(t *testing.T) { } func TestSameConfigFalse(t *testing.T) { + log.SetQuiet(true) src, err := ioutil.TempFile("", "src") defer os.Remove(src.Name()) if err != nil { @@ -196,3 +284,20 @@ func TestSameConfigFalse(t *testing.T) { t.Errorf("Expected sameConfig(src, dest) to be %v, got %v", false, status) } } + +func TestIsFileExist(t *testing.T) { + log.SetQuiet(true) + result := isFileExist(fakeFile) + if result != false { + t.Errorf("Expected IsFileExist(%s) to be false, got %v", fakeFile, result) + } + f, err := ioutil.TempFile("", "") + if err != nil { + t.Fatal(err.Error()) + } + defer os.Remove(f.Name()) + result = isFileExist(f.Name()) + if result != true { + t.Errorf("Expected IsFileExist(%s) to be true, got %v", f.Name(), result) + } +} diff --git a/util.go b/util.go deleted file mode 100644 index 57c7e82b7..000000000 --- a/util.go +++ /dev/null @@ -1,13 +0,0 @@ -package main - -import ( - "os" -) - -// IsFileExist reports whether path exits. -func IsFileExist(fpath string) bool { - if _, err := os.Stat(fpath); os.IsNotExist(err) { - return false - } - return true -} diff --git a/util_test.go b/util_test.go deleted file mode 100644 index 2fd50be36..000000000 --- a/util_test.go +++ /dev/null @@ -1,27 +0,0 @@ -package main - -import ( - "testing" - "io/ioutil" - "os" -) - -var ( - fakeFile = "/this/shoud/not/exist" -) - -func TestIsFileExist(t *testing.T) { - result := IsFileExist(fakeFile) - if result != false { - t.Errorf("Expected IsFileExist(%s) to be false, got %v", fakeFile, result) - } - f, err := ioutil.TempFile("", "") - if err != nil { - t.Fatal(err.Error()) - } - defer os.Remove(f.Name()) - result = IsFileExist(f.Name()) - if result != true { - t.Errorf("Expected IsFileExist(%s) to be true, got %v", f.Name(), result) - } -} diff --git a/version.go b/version.go index 7506c19d2..438c9ff38 100644 --- a/version.go +++ b/version.go @@ -1,3 +1,3 @@ package main -const version = "0.1.2" +const Version = "0.2.0"