From 24549b8550d220d263a7dc1598c2311dbcca0baf Mon Sep 17 00:00:00 2001 From: Kelsey Hightower Date: Sat, 2 Nov 2013 07:03:13 -0700 Subject: [PATCH 01/30] Add noop mode confd now supports a noop mode via the `-noop` commandline flag and `noop` configuration setting. Noop mode will process all template resources, but stop short of updating target config files. --- README.md | 7 ++++ config.go | 10 ++++++ template_resource.go | 4 +++ template_resource_test.go | 75 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 96 insertions(+) diff --git a/README.md b/README.md index c2823b818..39f64d84e 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,12 @@ The "/production" string will be prefixed to keys when querying etcd at http://1 confd -i 30 -p '/production' -n 'http://127.0.0.1:4001' ``` +### Same as above in noop mode + +``` +confd -noop -i 30 -p '/production' -n 'http://127.0.0.1:4001' +``` + ### Single run without polling Using default settings run one time and exit. @@ -71,6 +77,7 @@ Optional: etcd. The default is "/". * `client_cert` (string) The cert file of the client. * `client_key` (string) The key file of the client. +* `noop` (bool) - Enable noop mode. Process all template resource, but don't update target config. Example: diff --git a/config.go b/config.go index b7fdd773c..1ea4afb64 100644 --- a/config.go +++ b/config.go @@ -20,6 +20,7 @@ var ( prefix string clientCert string clientKey string + noop bool ) func init() { @@ -29,6 +30,7 @@ func init() { flag.StringVar(&prefix, "p", "/", "etcd key path prefix") flag.StringVar(&clientCert, "cert", "", "the client cert") flag.StringVar(&clientKey, "key", "", "the client key") + flag.BoolVar(&noop, "noop", false, "only show pending changes, don't sync configs.") } // Nodes is a custom flag Var representing a list of etcd nodes. We use a custom @@ -59,6 +61,7 @@ type confd struct { Interval int Prefix string EtcdNodes []string `toml:"etcd_nodes"` + Noop bool `toml:"noop"` } // loadConfig initializes the confd configuration by first setting defaults, @@ -105,6 +108,11 @@ func Interval() int { return config.Confd.Interval } +// Noop +func Noop() bool { + return config.Confd.Noop +} + // Prefix returns the etcd key prefix to use when querying etcd. func Prefix() string { return config.Confd.Prefix @@ -151,6 +159,8 @@ func override(f *flag.Flag) { config.Confd.ClientCert = clientCert case "key": config.Confd.ClientKey = clientKey + case "noop": + config.Confd.Noop = noop } } diff --git a/template_resource.go b/template_resource.go index 614ee40a9..7c917ac3c 100644 --- a/template_resource.go +++ b/template_resource.go @@ -102,6 +102,10 @@ func (t *TemplateResource) sync() error { if err != nil { log.Error(err.Error()) } + if Noop() { + log.Warning("In noop mode, not updating " + t.Dest) + return nil + } if !ok { log.Info("syncing " + t.Dest) if t.CheckCmd != "" { diff --git a/template_resource_test.go b/template_resource_test.go index 57ee17ae9..40fd1d18c 100644 --- a/template_resource_test.go +++ b/template_resource_test.go @@ -122,6 +122,81 @@ func TestProcessTemplateResources(t *testing.T) { } } +func TestProcessTemplateResourcesNoop(t *testing.T) { + // Setup temporary conf, config, and template directories. + tempConfDir, err := createTempDirs() + if err != nil { + t.Errorf("Failed to create temp dirs: %s", err.Error()) + } + defer os.RemoveAll(tempConfDir) + + // Create the src template. + srcTemplateFile := filepath.Join(tempConfDir, "templates", "foo.tmpl") + err = ioutil.WriteFile(srcTemplateFile, []byte("foo = {{ .foo }}"), 0644) + if err != nil { + t.Error(err.Error()) + } + + // Create the dest. + destFile, err := ioutil.TempFile("", "") + if err != nil { + t.Errorf("Failed to create destFile: %s", err.Error()) + } + defer os.Remove(destFile.Name()) + + // Create the template resource configuration file. + templateResourcePath := filepath.Join(tempConfDir, "conf.d", "foo.toml") + templateResourceFile, err := os.Create(templateResourcePath) + if err != nil { + t.Errorf(err.Error()) + } + tmpl, err := template.New("templateResourceConfig").Parse(templateResourceConfigTmpl) + if err != nil { + t.Errorf("Unable to parse template resource template: %s", err.Error()) + } + data := make(map[string]string) + data["src"] = "foo.tmpl" + data["dest"] = destFile.Name() + err = tmpl.Execute(templateResourceFile, data) + if err != nil { + t.Errorf(err.Error()) + } + + // Load the confd configuration settings. + if err := loadConfig(""); err != nil { + t.Errorf(err.Error()) + } + config.Confd.Prefix = "" + // Use the temporary tempConfDir from above. + config.Confd.ConfDir = tempConfDir + // Enable noop mode. + config.Confd.Noop = true + + // Create the stub etcd client. + c := etcdtest.NewClient() + fooResp := []*etcd.Response{ + {Key: "/foo", Dir: false, Value: "bar"}, + } + c.AddResponse("/foo", fooResp) + + // Process the test template resource. + runErrors := ProcessTemplateResources(c) + if len(runErrors) > 0 { + for _, e := range runErrors { + t.Errorf(e.Error()) + } + } + // Verify the results. + expected := "" + results, err := ioutil.ReadFile(destFile.Name()) + if err != nil { + t.Error(err.Error()) + } + if string(results) != expected { + t.Errorf("Expected contents of dest == '%s', got %s", expected, string(results)) + } +} + func TestBrokenTemplateResourceFile(t *testing.T) { tempFile, err := ioutil.TempFile("", "") defer os.Remove(tempFile.Name()) From 55d08784c88bb21835f9aa5766615c56b0dfa055 Mon Sep 17 00:00:00 2001 From: Kelsey Hightower Date: Sat, 2 Nov 2013 07:06:29 -0700 Subject: [PATCH 02/30] Bump version to 0.2.0 --- README.md | 2 +- version.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 39f64d84e..aad5e3bc0 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # confd -README version 0.1.2 +README version 0.2.0 [![Build Status](https://travis-ci.org/kelseyhightower/confd.png?branch=master)](https://travis-ci.org/kelseyhightower/confd) diff --git a/version.go b/version.go index 7506c19d2..df39deed8 100644 --- a/version.go +++ b/version.go @@ -1,3 +1,3 @@ package main -const version = "0.1.2" +const version = "0.2.0" From 9141770baf1c497ed4804d07c7844a3c8f31012d Mon Sep 17 00:00:00 2001 From: Kelsey Hightower Date: Sat, 2 Nov 2013 07:33:12 -0700 Subject: [PATCH 03/30] Add mailing list link --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index aad5e3bc0..9024cb65b 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # confd +* Mailing list: [Google Groups](https://groups.google.com/forum/#!forum/confd-users) + README version 0.2.0 [![Build Status](https://travis-ci.org/kelseyhightower/confd.png?branch=master)](https://travis-ci.org/kelseyhightower/confd) From 6681c3e248e744fe08b5677be0dd67e6d800a41f Mon Sep 17 00:00:00 2001 From: Kelsey Hightower Date: Sat, 2 Nov 2013 07:53:16 -0700 Subject: [PATCH 04/30] Add IRC info --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 9024cb65b..cf9b0f365 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ # confd +* IRC: `#confd` on Freenode * Mailing list: [Google Groups](https://groups.google.com/forum/#!forum/confd-users) README version 0.2.0 From 747033d0aecaea66beb5f721f76ecf19379b8a98 Mon Sep 17 00:00:00 2001 From: Kelsey Hightower Date: Sat, 2 Nov 2013 14:39:42 -0700 Subject: [PATCH 05/30] Add support for SRV records --- confd.go | 2 ++ config.go | 42 +++++++++++++++++++++++++++++++++++------- util.go | 42 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 79 insertions(+), 7 deletions(-) diff --git a/confd.go b/confd.go index 13e879705..2f786e14d 100644 --- a/confd.go +++ b/confd.go @@ -9,6 +9,7 @@ import ( "time" "github.com/kelseyhightower/confd/log" + "strings" ) var ( @@ -44,6 +45,7 @@ func main() { } // Create the etcd client upfront and use it for the life of the process. // The etcdClient is an http.Client and designed to be reused. + log.Debug("Connecting to " + strings.Join(EtcdNodes(), ", ")) etcdClient, err := newEtcdClient(EtcdNodes(), ClientCert(), ClientKey()) if err != nil { log.Fatal(err.Error()) diff --git a/config.go b/config.go index 1ea4afb64..55fc608c9 100644 --- a/config.go +++ b/config.go @@ -4,6 +4,7 @@ package main import ( + "errors" "flag" "fmt" "path/filepath" @@ -20,6 +21,7 @@ var ( prefix string clientCert string clientKey string + srvDomain string noop bool ) @@ -30,6 +32,7 @@ func init() { flag.StringVar(&prefix, "p", "/", "etcd key path prefix") flag.StringVar(&clientCert, "cert", "", "the client cert") flag.StringVar(&clientKey, "key", "", "the client key") + flag.StringVar(&srvDomain, "srv-domain", "", "the domain to query for the etcd SRV record, i.e. example.com") flag.BoolVar(&noop, "noop", false, "only show pending changes, don't sync configs.") } @@ -55,13 +58,15 @@ type Config struct { // confd represents the parsed configuration settings. type confd struct { - ConfDir string - ClientCert string `toml:"client_cert"` - ClientKey string `toml:"client_key"` - Interval int - Prefix string - EtcdNodes []string `toml:"etcd_nodes"` - Noop bool `toml:"noop"` + ConfDir string + ClientCert string `toml:"client_cert"` + ClientKey string `toml:"client_key"` + ConnectSecure bool `toml:"connect_secure"` + Interval int + Prefix string + EtcdNodes []string `toml:"etcd_nodes"` + Noop bool `toml:"noop"` + SRVDomain string `toml:"srv_domain"` } // loadConfig initializes the confd configuration by first setting defaults, @@ -79,6 +84,22 @@ func loadConfig(path string) error { } } overrideConfig() + if config.Confd.SRVDomain != "" { + hostUris := make([]string, 0) + scheme := "http" + if config.Confd.ConnectSecure { + scheme = "https" + } + etcdHosts, err := GetEtcdHostsFromSRV(config.Confd.SRVDomain) + if err != nil { + return errors.New("Cannot get etcd hosts from SRV records " + err.Error()) + } + for _, h := range etcdHosts { + uri := fmt.Sprintf("%s://%s:%d", scheme, h.Hostname, h.Port) + hostUris = append(hostUris, uri) + } + config.Confd.EtcdNodes = hostUris + } return nil } @@ -123,6 +144,11 @@ func TemplateDir() string { return filepath.Join(config.Confd.ConfDir, "templates") } +// SRVDomain return the domain name used to query etcd SRV records. +func SRVDomain() string { + return config.Confd.SRVDomain +} + func setDefaults() { config = Config{ Confd: confd{ @@ -161,6 +187,8 @@ func override(f *flag.Flag) { config.Confd.ClientKey = clientKey case "noop": config.Confd.Noop = noop + case "srv-domain": + config.Confd.SRVDomain = srvDomain } } diff --git a/util.go b/util.go index 57c7e82b7..adf98e22b 100644 --- a/util.go +++ b/util.go @@ -1,9 +1,17 @@ package main import ( + "net" "os" + "strings" ) +// etcdHost +type etcdHost struct { + Hostname string + Port uint16 +} + // IsFileExist reports whether path exits. func IsFileExist(fpath string) bool { if _, err := os.Stat(fpath); os.IsNotExist(err) { @@ -11,3 +19,37 @@ func IsFileExist(fpath string) bool { } return true } + +// GetEtcdHostsFromSRV returns a list of etcHost. +func GetEtcdHostsFromSRV(domain string) ([]*etcdHost, error) { + addrs, err := lookupEtcdSRV(domain) + if err != nil { + return nil, err + } + etcdHosts := etcdHostsFromSRV(addrs) + return etcdHosts, nil +} + +// lookupEtcdSrv tries to resolve an SRV query for the etcd service for the +// specified domain. +// +// lookupEtcdSRV constructs the DNS name to look up following RFC 2782. +// That is, it looks up _etcd._tcp.domain. +func lookupEtcdSRV(domain string) ([]*net.SRV, error) { + // Ignore the CNAME as we don't need it. + _, addrs, err := net.LookupSRV("etcd", "tcp", domain) + if err != nil { + return addrs, err + } + return addrs, nil +} + +// etcdHostsFromSRV converts an etcd SRV record to a list of etcdHost. +func etcdHostsFromSRV(addrs []*net.SRV) []*etcdHost { + hosts := make([]*etcdHost, 0) + for _, srv := range addrs { + hostname := strings.TrimRight(srv.Target, ".") + hosts = append(hosts, &etcdHost{Hostname: hostname, Port: srv.Port}) + } + return hosts +} From 5d7596a160ac27f3f1046ad8b5534e3be0c33c33 Mon Sep 17 00:00:00 2001 From: Kelsey Hightower Date: Sat, 2 Nov 2013 17:16:35 -0700 Subject: [PATCH 06/30] The etcd scheme is configurable --- config.go | 83 +++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 71 insertions(+), 12 deletions(-) diff --git a/config.go b/config.go index 55fc608c9..10b570816 100644 --- a/config.go +++ b/config.go @@ -7,10 +7,13 @@ import ( "errors" "flag" "fmt" + "net" + "net/url" "path/filepath" "github.com/BurntSushi/toml" "github.com/kelseyhightower/confd/log" + "strconv" ) var ( @@ -23,6 +26,7 @@ var ( clientKey string srvDomain string noop bool + etcdScheme string ) func init() { @@ -32,6 +36,7 @@ func init() { flag.StringVar(&prefix, "p", "/", "etcd key path prefix") flag.StringVar(&clientCert, "cert", "", "the client cert") flag.StringVar(&clientKey, "key", "", "the client key") + flag.StringVar(&etcdScheme, "etcd-scheme", "http", "the etcd URI scheme. (http or https)") flag.StringVar(&srvDomain, "srv-domain", "", "the domain to query for the etcd SRV record, i.e. example.com") flag.BoolVar(&noop, "noop", false, "only show pending changes, don't sync configs.") } @@ -65,6 +70,7 @@ type confd struct { Interval int Prefix string EtcdNodes []string `toml:"etcd_nodes"` + EtcdScheme string `toml:"etcd_scheme"` Noop bool `toml:"noop"` SRVDomain string `toml:"srv_domain"` } @@ -84,25 +90,75 @@ func loadConfig(path string) error { } } overrideConfig() + if !isValidateEtcdScheme(config.Confd.EtcdScheme) { + return errors.New("Invalid etcd scheme: " + config.Confd.EtcdScheme) + } + err := setEtcdHosts() + if err != nil { + return err + } + return nil +} + +func setEtcdHosts() error { + scheme := config.Confd.EtcdScheme + hosts := make([]string, 0) + // If a domain name is given then lookup the etcd SRV record, and override + // all other etcd node settings. if config.Confd.SRVDomain != "" { - hostUris := make([]string, 0) - scheme := "http" - if config.Confd.ConnectSecure { - scheme = "https" - } etcdHosts, err := GetEtcdHostsFromSRV(config.Confd.SRVDomain) if err != nil { return errors.New("Cannot get etcd hosts from SRV records " + err.Error()) } for _, h := range etcdHosts { - uri := fmt.Sprintf("%s://%s:%d", scheme, h.Hostname, h.Port) - hostUris = append(hostUris, uri) + uri := formatEtcdHostURI(scheme, h.Hostname, strconv.FormatUint(uint64(h.Port), 10)) + hosts = append(hosts, uri) } - config.Confd.EtcdNodes = hostUris + config.Confd.EtcdNodes = hosts + return nil } + // No domain name was given, so just process the etcd node list. + // An etcdNode can be a URL, http://etcd.example.com:4001, or a host, etcd.example.com:4001. + for _, node := range config.Confd.EtcdNodes { + etcdURL, err := url.Parse(node) + if err != nil { + log.Error(err.Error()) + return err + } + if etcdURL.Scheme != "" && etcdURL.Host != "" { + if !isValidateEtcdScheme(etcdURL.Scheme) { + return errors.New("The etcd node list contains an invalid URL: " + node) + } + host, port, err := net.SplitHostPort(etcdURL.Host) + if err != nil { + return err + } + hosts = append(hosts, formatEtcdHostURI(etcdURL.Scheme, host, port)) + continue + } + // At this point node is not an etcd URL, i.e. http://etcd.example.com:4001, + // but a host:port string, i.e. etcd.example.com:4001 + host, port, err := net.SplitHostPort(node) + if err != nil { + return err + } + hosts = append(hosts, formatEtcdHostURI(scheme, host, port)) + } + config.Confd.EtcdNodes = hosts return nil } +func formatEtcdHostURI(scheme, host, port string) string { + return fmt.Sprintf("%s://%s:%s", scheme, host, port) +} + +func isValidateEtcdScheme(scheme string) bool { + if scheme == "http" || scheme == "https" { + return true + } + return false +} + // ConfigDir returns the path to the confd config dir. func ConfigDir() string { return filepath.Join(config.Confd.ConfDir, "conf.d") @@ -152,10 +208,11 @@ func SRVDomain() string { func setDefaults() { config = Config{ Confd: confd{ - ConfDir: "/etc/confd", - Interval: 600, - Prefix: "/", - EtcdNodes: []string{"http://127.0.0.1:4001"}, + ConfDir: "/etc/confd", + Interval: 600, + Prefix: "/", + EtcdNodes: []string{"127.0.0.1:4001"}, + EtcdScheme: "http", }, } } @@ -189,6 +246,8 @@ func override(f *flag.Flag) { config.Confd.Noop = noop case "srv-domain": config.Confd.SRVDomain = srvDomain + case "etcd-scheme": + config.Confd.EtcdScheme = etcdScheme } } From 89b231c1b8554cef0aa13a088ff518ce6bb9fd2d Mon Sep 17 00:00:00 2001 From: Kelsey Hightower Date: Sat, 2 Nov 2013 17:25:24 -0700 Subject: [PATCH 07/30] Update README with new config options --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index cf9b0f365..614c5c8fd 100644 --- a/README.md +++ b/README.md @@ -81,6 +81,8 @@ Optional: * `client_cert` (string) The cert file of the client. * `client_key` (string) The key file of the client. * `noop` (bool) - Enable noop mode. Process all template resource, but don't update target config. +* `srv_domain` (string) - The domain to query for etcd SRV records. +* `etcd_scheme` (string) - The etcd scheme to use. Must be 'http' or 'https' Example: From ab7185afcd378279ac96288ad2b91e6240a646b7 Mon Sep 17 00:00:00 2001 From: Kelsey Hightower Date: Sat, 2 Nov 2013 17:26:41 -0700 Subject: [PATCH 08/30] Remove ConnectSecure parameter from the confd type --- config.go | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/config.go b/config.go index 10b570816..548c9ada6 100644 --- a/config.go +++ b/config.go @@ -63,16 +63,15 @@ type Config struct { // confd represents the parsed configuration settings. type confd struct { - ConfDir string - ClientCert string `toml:"client_cert"` - ClientKey string `toml:"client_key"` - ConnectSecure bool `toml:"connect_secure"` - Interval int - Prefix string - EtcdNodes []string `toml:"etcd_nodes"` - EtcdScheme string `toml:"etcd_scheme"` - Noop bool `toml:"noop"` - SRVDomain string `toml:"srv_domain"` + ConfDir string + ClientCert string `toml:"client_cert"` + ClientKey string `toml:"client_key"` + Interval int + Prefix string + EtcdNodes []string `toml:"etcd_nodes"` + EtcdScheme string `toml:"etcd_scheme"` + Noop bool `toml:"noop"` + SRVDomain string `toml:"srv_domain"` } // loadConfig initializes the confd configuration by first setting defaults, From b48a4a22a6360fdfedeba68cc1426d03192053d8 Mon Sep 17 00:00:00 2001 From: Kelsey Hightower Date: Sat, 2 Nov 2013 17:29:32 -0700 Subject: [PATCH 09/30] Update README, order config docs --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 614c5c8fd..a0e90db7e 100644 --- a/README.md +++ b/README.md @@ -71,18 +71,18 @@ and loaded from `/etc/confd/confd.toml` by default. Optional: +* `client_cert` (string) The cert file of the client. +* `client_key` (string) The key file of the client. * `confdir` (string) - The path to confd configs. The default is /etc/confd. * `etcd_nodes` (array of strings) - An array of etcd cluster nodes. The default is ["http://127.0.0.1:4001"]. +* `etcd_scheme` (string) - The etcd scheme to use. Must be 'http' or 'https' * `interval` (int) - The number of seconds to wait between calls to etcd. The default is 600. +* `noop` (bool) - Enable noop mode. Process all template resource, but don't update target config. * `prefix` (string) - The prefix string to prefix to keys when making calls to etcd. The default is "/". -* `client_cert` (string) The cert file of the client. -* `client_key` (string) The key file of the client. -* `noop` (bool) - Enable noop mode. Process all template resource, but don't update target config. * `srv_domain` (string) - The domain to query for etcd SRV records. -* `etcd_scheme` (string) - The etcd scheme to use. Must be 'http' or 'https' Example: From 03b3cf247dd79cfa54c7588001cfba3b39fa9762 Mon Sep 17 00:00:00 2001 From: Kelsey Hightower Date: Sat, 2 Nov 2013 17:38:07 -0700 Subject: [PATCH 10/30] Add an example usage of SRV records --- README.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/README.md b/README.md index a0e90db7e..9f22dae81 100644 --- a/README.md +++ b/README.md @@ -64,6 +64,22 @@ Same as above but authenticate with client certificates. confd -onetime -key /etc/confd/ssl/client.key -cert /etc/confd/ssl/client.crt ``` +### Lookup etcd node using SRV records and set the scheme to https + +``` +dig SRV _etcd._tcp.confd.io +... +;; ANSWER SECTION: +_etcd._tcp.confd.io. 300 IN SRV 1 50 4001 etcd0.confd.io. +_etcd._tcp.confd.io. 300 IN SRV 2 50 4001 etcd1.confd.io. +``` + +``` +confd -srv-domain example.com -etcd-scheme https +``` + +confd would connect to the nodes at `["https://etcd0.confd.io:4001", "https://etcd1.confd.io:4001"]` + ## Configuration The confd configuration file is written in [TOML](https://github.com/mojombo/toml) From cd70aa613bca7c86b5e6fbd5797aedefdfb7243e Mon Sep 17 00:00:00 2001 From: Kelsey Hightower Date: Sat, 2 Nov 2013 17:39:44 -0700 Subject: [PATCH 11/30] Update example usage of SRV records --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9f22dae81..33db9b99b 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ Same as above but authenticate with client certificates. confd -onetime -key /etc/confd/ssl/client.key -cert /etc/confd/ssl/client.crt ``` -### Lookup etcd node using SRV records and set the scheme to https +### Lookup etcd nodes using SRV records ``` dig SRV _etcd._tcp.confd.io From ac4d59a6d841e3317e41837178a9011603da877d Mon Sep 17 00:00:00 2001 From: Kelsey Hightower Date: Sat, 2 Nov 2013 17:43:11 -0700 Subject: [PATCH 12/30] Move community info down one section --- README.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 33db9b99b..bd06f1f98 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,20 @@ # confd -* IRC: `#confd` on Freenode -* Mailing list: [Google Groups](https://groups.google.com/forum/#!forum/confd-users) +[![Build Status](https://travis-ci.org/kelseyhightower/confd.png?branch=master)](https://travis-ci.org/kelseyhightower/confd) README version 0.2.0 -[![Build Status](https://travis-ci.org/kelseyhightower/confd.png?branch=master)](https://travis-ci.org/kelseyhightower/confd) - `confd` is a lightweight configuration management tool focused on: * keeping local configuration files up-to-date by polling [etcd](https://github.com/coreos/etcd) and processing [template resources](https://github.com/kelseyhightower/confd#template-resources). * reloading applications to pick up new config file changes +## Community + +* IRC: `#confd` on Freenode +* Mailing list: [Google Groups](https://groups.google.com/forum/#!forum/confd-users) +* Website: [www.confd.io](http://www.confd.io) + ## Getting Started ### Installing confd From 8a414b028e1b65ae69ac1b5c84b20f1b1790457a Mon Sep 17 00:00:00 2001 From: Kelsey Hightower Date: Sat, 2 Nov 2013 19:14:29 -0700 Subject: [PATCH 13/30] Move configs to separate package. --- confd.go | 11 +++-- config.go => config/config.go | 64 +++++++++++++++++++++++-- config_test.go => config/config_test.go | 2 +- template_resource.go | 14 +++--- template_resource_test.go | 20 ++++---- util.go | 40 ---------------- 6 files changed, 86 insertions(+), 65 deletions(-) rename config.go => config/config.go (82%) rename config_test.go => config/config_test.go (98%) diff --git a/confd.go b/confd.go index 2f786e14d..f0dac51c0 100644 --- a/confd.go +++ b/confd.go @@ -6,10 +6,11 @@ package main import ( "flag" "os" + "strings" "time" + "github.com/kelseyhightower/confd/config" "github.com/kelseyhightower/confd/log" - "strings" ) var ( @@ -40,13 +41,13 @@ func main() { configFile = defaultConfigFile } } - if err := loadConfig(configFile); err != nil { + if err := config.LoadConfig(configFile); err != nil { log.Fatal(err.Error()) } // Create the etcd client upfront and use it for the life of the process. // The etcdClient is an http.Client and designed to be reused. - log.Debug("Connecting to " + strings.Join(EtcdNodes(), ", ")) - etcdClient, err := newEtcdClient(EtcdNodes(), ClientCert(), ClientKey()) + log.Debug("Connecting to " + strings.Join(config.EtcdNodes(), ", ")) + etcdClient, err := newEtcdClient(config.EtcdNodes(), config.ClientCert(), config.ClientKey()) if err != nil { log.Fatal(err.Error()) } @@ -61,6 +62,6 @@ func main() { os.Exit(0) } // By default we poll etcd every 30 seconds - time.Sleep(time.Duration(Interval()) * time.Second) + time.Sleep(time.Duration(config.Interval()) * time.Second) } } diff --git a/config.go b/config/config.go similarity index 82% rename from config.go rename to config/config.go index 548c9ada6..2ad846548 100644 --- a/config.go +++ b/config/config.go @@ -1,7 +1,7 @@ // Copyright (c) 2013 Kelsey Hightower. All rights reserved. // Use of this source code is governed by the Apache License, Version 2.0 // that can be found in the LICENSE file. -package main +package config import ( "errors" @@ -10,12 +10,19 @@ import ( "net" "net/url" "path/filepath" + "strconv" + "strings" "github.com/BurntSushi/toml" "github.com/kelseyhightower/confd/log" - "strconv" ) +// etcdHost +type etcdHost struct { + Hostname string + Port uint16 +} + var ( config Config nodes Nodes @@ -74,11 +81,11 @@ type confd struct { SRVDomain string `toml:"srv_domain"` } -// loadConfig initializes the confd configuration by first setting defaults, +// LoadConfig initializes the confd configuration by first setting defaults, // then overriding setting from the confd config file, and finally overriding // settings from flags set on the command line. // It returns an error if any. -func loadConfig(path string) error { +func LoadConfig(path string) error { setDefaults() if path == "" { log.Warning("Skipping confd config file.") @@ -163,6 +170,11 @@ func ConfigDir() string { return filepath.Join(config.Confd.ConfDir, "conf.d") } +// SetConfDir. +func SetConfDir(path string) { + config.Confd.ConfDir = path +} + // ClientCert returns the path to the client cert. func ClientCert() string { return config.Confd.ClientCert @@ -189,11 +201,21 @@ func Noop() bool { return config.Confd.Noop } +// SetNoop. +func SetNoop(enabled bool) { + config.Confd.Noop = enabled +} + // Prefix returns the etcd key prefix to use when querying etcd. func Prefix() string { return config.Confd.Prefix } +// SetPrefix +func SetPrefix(prefix string) { + config.Confd.Prefix = prefix +} + // TemplateDir returns the path to the directory of config file templates. func TemplateDir() string { return filepath.Join(config.Confd.ConfDir, "templates") @@ -255,3 +277,37 @@ func override(f *flag.Flag) { func overrideConfig() { flag.Visit(override) } + +// GetEtcdHostsFromSRV returns a list of etcHost. +func GetEtcdHostsFromSRV(domain string) ([]*etcdHost, error) { + addrs, err := lookupEtcdSRV(domain) + if err != nil { + return nil, err + } + etcdHosts := etcdHostsFromSRV(addrs) + return etcdHosts, nil +} + +// lookupEtcdSrv tries to resolve an SRV query for the etcd service for the +// specified domain. +// +// lookupEtcdSRV constructs the DNS name to look up following RFC 2782. +// That is, it looks up _etcd._tcp.domain. +func lookupEtcdSRV(domain string) ([]*net.SRV, error) { + // Ignore the CNAME as we don't need it. + _, addrs, err := net.LookupSRV("etcd", "tcp", domain) + if err != nil { + return addrs, err + } + return addrs, nil +} + +// etcdHostsFromSRV converts an etcd SRV record to a list of etcdHost. +func etcdHostsFromSRV(addrs []*net.SRV) []*etcdHost { + hosts := make([]*etcdHost, 0) + for _, srv := range addrs { + hostname := strings.TrimRight(srv.Target, ".") + hosts = append(hosts, &etcdHost{Hostname: hostname, Port: srv.Port}) + } + return hosts +} diff --git a/config_test.go b/config/config_test.go similarity index 98% rename from config_test.go rename to config/config_test.go index c091c5f42..0ad5a9f18 100644 --- a/config_test.go +++ b/config/config_test.go @@ -1,4 +1,4 @@ -package main +package config import ( "testing" diff --git a/template_resource.go b/template_resource.go index 7c917ac3c..75c3c10c0 100644 --- a/template_resource.go +++ b/template_resource.go @@ -5,8 +5,6 @@ import ( "crypto/md5" "errors" "fmt" - "github.com/BurntSushi/toml" - "github.com/kelseyhightower/confd/log" "io" "io/ioutil" "os" @@ -15,6 +13,10 @@ import ( "strconv" "syscall" "text/template" + + "github.com/BurntSushi/toml" + "github.com/kelseyhightower/confd/config" + "github.com/kelseyhightower/confd/log" ) // TemplateResourceConfig holds the parsed template resource. @@ -57,7 +59,7 @@ func NewTemplateResourceFromPath(path string, c EtcdClient) (*TemplateResource, // setVars sets the Vars for template resource. func (t *TemplateResource) setVars() error { var err error - t.Vars, err = getValues(t.etcdClient, Prefix(), t.Keys) + t.Vars, err = getValues(t.etcdClient, config.Prefix(), t.Keys) if err != nil { return err } @@ -69,7 +71,7 @@ func (t *TemplateResource) setVars() error { // StageFile for the template resource. // It returns an error if any. func (t *TemplateResource) createStageFile() error { - t.Src = filepath.Join(TemplateDir(), t.Src) + t.Src = filepath.Join(config.TemplateDir(), t.Src) if !IsFileExist(t.Src) { return errors.New("Missing template: " + t.Src) } @@ -102,7 +104,7 @@ func (t *TemplateResource) sync() error { if err != nil { log.Error(err.Error()) } - if Noop() { + if config.Noop() { log.Warning("In noop mode, not updating " + t.Dest) return nil } @@ -217,7 +219,7 @@ func ProcessTemplateResources(c EtcdClient) []error { runErrors = append(runErrors, errors.New("An etcd client is required")) return runErrors } - paths, err := filepath.Glob(filepath.Join(ConfigDir(), "*toml")) + paths, err := filepath.Glob(filepath.Join(config.ConfigDir(), "*toml")) if err != nil { runErrors = append(runErrors, err) return runErrors diff --git a/template_resource_test.go b/template_resource_test.go index 40fd1d18c..317ad8345 100644 --- a/template_resource_test.go +++ b/template_resource_test.go @@ -1,13 +1,15 @@ package main import ( - "github.com/coreos/go-etcd/etcd" - "github.com/kelseyhightower/confd/etcdtest" "io/ioutil" "os" "path/filepath" "testing" "text/template" + + "github.com/coreos/go-etcd/etcd" + "github.com/kelseyhightower/confd/config" + "github.com/kelseyhightower/confd/etcdtest" ) // createTempDirs is a helper function which creates temporary directories @@ -90,12 +92,12 @@ func TestProcessTemplateResources(t *testing.T) { } // Load the confd configuration settings. - if err := loadConfig(""); err != nil { + if err := config.LoadConfig(""); err != nil { t.Errorf(err.Error()) } - config.Confd.Prefix = "" + config.SetPrefix("") // Use the temporary tempConfDir from above. - config.Confd.ConfDir = tempConfDir + config.SetConfDir(tempConfDir) // Create the stub etcd client. c := etcdtest.NewClient() @@ -163,14 +165,14 @@ func TestProcessTemplateResourcesNoop(t *testing.T) { } // Load the confd configuration settings. - if err := loadConfig(""); err != nil { + if err := config.LoadConfig(""); err != nil { t.Errorf(err.Error()) } - config.Confd.Prefix = "" + config.SetPrefix("") // Use the temporary tempConfDir from above. - config.Confd.ConfDir = tempConfDir + config.SetConfDir(tempConfDir) // Enable noop mode. - config.Confd.Noop = true + config.SetNoop(true) // Create the stub etcd client. c := etcdtest.NewClient() diff --git a/util.go b/util.go index adf98e22b..ed0b0d985 100644 --- a/util.go +++ b/util.go @@ -1,16 +1,9 @@ package main import ( - "net" "os" - "strings" ) -// etcdHost -type etcdHost struct { - Hostname string - Port uint16 -} // IsFileExist reports whether path exits. func IsFileExist(fpath string) bool { @@ -20,36 +13,3 @@ func IsFileExist(fpath string) bool { return true } -// GetEtcdHostsFromSRV returns a list of etcHost. -func GetEtcdHostsFromSRV(domain string) ([]*etcdHost, error) { - addrs, err := lookupEtcdSRV(domain) - if err != nil { - return nil, err - } - etcdHosts := etcdHostsFromSRV(addrs) - return etcdHosts, nil -} - -// lookupEtcdSrv tries to resolve an SRV query for the etcd service for the -// specified domain. -// -// lookupEtcdSRV constructs the DNS name to look up following RFC 2782. -// That is, it looks up _etcd._tcp.domain. -func lookupEtcdSRV(domain string) ([]*net.SRV, error) { - // Ignore the CNAME as we don't need it. - _, addrs, err := net.LookupSRV("etcd", "tcp", domain) - if err != nil { - return addrs, err - } - return addrs, nil -} - -// etcdHostsFromSRV converts an etcd SRV record to a list of etcdHost. -func etcdHostsFromSRV(addrs []*net.SRV) []*etcdHost { - hosts := make([]*etcdHost, 0) - for _, srv := range addrs { - hostname := strings.TrimRight(srv.Target, ".") - hosts = append(hosts, &etcdHost{Hostname: hostname, Port: srv.Port}) - } - return hosts -} From e813187111b2326c9920fe1b24e18d0d00b5365f Mon Sep 17 00:00:00 2001 From: Kelsey Hightower Date: Sat, 2 Nov 2013 19:18:52 -0700 Subject: [PATCH 14/30] Fix broken config package test --- config/config_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/config_test.go b/config/config_test.go index 0ad5a9f18..3ad5aa6ec 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -17,7 +17,7 @@ func TestLoadConfig(t *testing.T) { "", "", "/etc/confd/conf.d", []string{"http://127.0.0.1:4001"}, 600, "/", "/etc/confd/templates", } - loadConfig("") + LoadConfig("") cc := ClientCert() if cc != expected.clientCert { t.Errorf("Expected default clientCert = %s, got %s", expected.clientCert, cc) From f5e7c4aae8b68afd96959a41507caf6750c82385 Mon Sep 17 00:00:00 2001 From: Kelsey Hightower Date: Sat, 2 Nov 2013 19:48:42 -0700 Subject: [PATCH 15/30] Move etcdclient and template_resource to their own packages --- confd.go | 14 ++++++++++++-- {etcdtest => etcd/etcdtest}/client.go | 0 etcd/etcdtest/client_test.go | 1 + etcd_client.go => etcd/etcdutil/client.go | 10 +++++----- .../etcdutil/client_test.go | 6 +++--- .../template/fileinfo_darwin.go | 2 +- .../template/fileinfo_linux.go | 2 +- .../template/template_resource.go | 11 ++++++----- .../template/template_resource_test.go | 4 ++-- util.go => resource/template/util.go | 2 +- util_test.go => resource/template/util_test.go | 2 +- version_test.go | 1 + 12 files changed, 34 insertions(+), 21 deletions(-) rename {etcdtest => etcd/etcdtest}/client.go (100%) create mode 100644 etcd/etcdtest/client_test.go rename etcd_client.go => etcd/etcdutil/client.go (87%) rename etcd_client_test.go => etcd/etcdutil/client_test.go (94%) rename fileinfo_darwin.go => resource/template/fileinfo_darwin.go (89%) rename fileinfo_linux.go => resource/template/fileinfo_linux.go (89%) rename template_resource.go => resource/template/template_resource.go (95%) rename template_resource_test.go => resource/template/template_resource_test.go (99%) rename util.go => resource/template/util.go (91%) rename util_test.go => resource/template/util_test.go (96%) create mode 100644 version_test.go diff --git a/confd.go b/confd.go index f0dac51c0..a70651790 100644 --- a/confd.go +++ b/confd.go @@ -10,7 +10,9 @@ import ( "time" "github.com/kelseyhightower/confd/config" + "github.com/kelseyhightower/confd/etcd/etcdutil" "github.com/kelseyhightower/confd/log" + "github.com/kelseyhightower/confd/resource/template" ) var ( @@ -47,12 +49,12 @@ func main() { // Create the etcd client upfront and use it for the life of the process. // The etcdClient is an http.Client and designed to be reused. log.Debug("Connecting to " + strings.Join(config.EtcdNodes(), ", ")) - etcdClient, err := newEtcdClient(config.EtcdNodes(), config.ClientCert(), config.ClientKey()) + etcdClient, err := etcdutil.NewEtcdClient(config.EtcdNodes(), config.ClientCert(), config.ClientKey()) if err != nil { log.Fatal(err.Error()) } for { - runErrors := ProcessTemplateResources(etcdClient) + runErrors := template.ProcessTemplateResources(etcdClient) // If the -onetime flag is passed on the command line we immediately exit // after processing the template config files. if onetime { @@ -65,3 +67,11 @@ func main() { time.Sleep(time.Duration(config.Interval()) * time.Second) } } + +// IsFileExist reports whether path exits. +func IsFileExist(fpath string) bool { + if _, err := os.Stat(fpath); os.IsNotExist(err) { + return false + } + return true +} diff --git a/etcdtest/client.go b/etcd/etcdtest/client.go similarity index 100% rename from etcdtest/client.go rename to etcd/etcdtest/client.go diff --git a/etcd/etcdtest/client_test.go b/etcd/etcdtest/client_test.go new file mode 100644 index 000000000..bd7b7d9ff --- /dev/null +++ b/etcd/etcdtest/client_test.go @@ -0,0 +1 @@ +package etcdtest diff --git a/etcd_client.go b/etcd/etcdutil/client.go similarity index 87% rename from etcd_client.go rename to etcd/etcdutil/client.go index 0eecd1c85..a984cc7f2 100644 --- a/etcd_client.go +++ b/etcd/etcdutil/client.go @@ -1,4 +1,4 @@ -package main +package etcdutil import ( "errors" @@ -9,9 +9,9 @@ import ( var replacer = strings.NewReplacer("/", "_") -// newEtcdClient returns an *etcd.Client with a connection to named machines. +// NewEtcdClient returns an *etcd.Client with a connection to named machines. // It returns an error if a connection to the cluster cannot be made. -func newEtcdClient(machines []string, cert, key string) (*etcd.Client, error) { +func NewEtcdClient(machines []string, cert, key string) (*etcd.Client, error) { c := etcd.NewClient(machines) if cert != "" && key != "" { _, err := c.SetCertAndKey(cert, key) @@ -30,13 +30,13 @@ type EtcdClient interface { Get(key string) ([]*etcd.Response, error) } -// getValues queries etcd for keys prefixed by prefix. +// GetValues queries etcd for keys prefixed by prefix. // Etcd paths (keys) are translated into names more suitable for use in // templates. For example if prefix where set to '/production' and one of the // keys where '/nginx/port'; the prefixed '/production/nginx/port' key would // be quired for. If the value for the prefixed key where 80, the returned map // would contain the entry vars["nginx_port"] = "80". -func getValues(c EtcdClient, prefix string, keys []string) (map[string]interface{}, error) { +func GetValues(c EtcdClient, prefix string, keys []string) (map[string]interface{}, error) { vars := make(map[string]interface{}) for _, key := range keys { err := etcdWalk(c, filepath.Join(prefix, key), prefix, vars) diff --git a/etcd_client_test.go b/etcd/etcdutil/client_test.go similarity index 94% rename from etcd_client_test.go rename to etcd/etcdutil/client_test.go index c64df53c7..511e3bb12 100644 --- a/etcd_client_test.go +++ b/etcd/etcdutil/client_test.go @@ -1,8 +1,8 @@ -package main +package etcdutil import ( "github.com/coreos/go-etcd/etcd" - "github.com/kelseyhightower/confd/etcdtest" + "github.com/kelseyhightower/confd/etcd/etcdtest" "testing" ) @@ -50,7 +50,7 @@ func TestGetValues(t *testing.T) { c.AddResponse("/foo/three", fooThreeResp) c.AddResponse("/nginx", nginxResp) keys := []string{"/nginx", "/foo"} - values, err := getValues(c, "", keys) + values, err := GetValues(c, "", keys) if err != nil { t.Error(err.Error()) } diff --git a/fileinfo_darwin.go b/resource/template/fileinfo_darwin.go similarity index 89% rename from fileinfo_darwin.go rename to resource/template/fileinfo_darwin.go index 4242f1d9c..1a86cbe0e 100644 --- a/fileinfo_darwin.go +++ b/resource/template/fileinfo_darwin.go @@ -1,4 +1,4 @@ -package main +package template // A fileInfo describes a configuration file and is returned by fileStat. type fileInfo struct { diff --git a/fileinfo_linux.go b/resource/template/fileinfo_linux.go similarity index 89% rename from fileinfo_linux.go rename to resource/template/fileinfo_linux.go index 69355d04f..409073fad 100644 --- a/fileinfo_linux.go +++ b/resource/template/fileinfo_linux.go @@ -1,4 +1,4 @@ -package main +package template // A fileInfo describes a configuration file and is returned by fileStat. type fileInfo struct { diff --git a/template_resource.go b/resource/template/template_resource.go similarity index 95% rename from template_resource.go rename to resource/template/template_resource.go index 75c3c10c0..be07931e2 100644 --- a/template_resource.go +++ b/resource/template/template_resource.go @@ -1,4 +1,4 @@ -package main +package template import ( "bytes" @@ -16,6 +16,7 @@ import ( "github.com/BurntSushi/toml" "github.com/kelseyhightower/confd/config" + "github.com/kelseyhightower/confd/etcd/etcdutil" "github.com/kelseyhightower/confd/log" ) @@ -37,13 +38,13 @@ type TemplateResource struct { StageFile *os.File Src string Vars map[string]interface{} - etcdClient EtcdClient + etcdClient etcdutil.EtcdClient } // NewTemplateResourceFromPath creates a TemplateResource using a decoded file path // and the supplied EtcdClient as input. // It returns a TemplateResource and an error if any. -func NewTemplateResourceFromPath(path string, c EtcdClient) (*TemplateResource, error) { +func NewTemplateResourceFromPath(path string, c etcdutil.EtcdClient) (*TemplateResource, error) { if c == nil { return nil, errors.New("A valid EtcdClient is required.") } @@ -59,7 +60,7 @@ func NewTemplateResourceFromPath(path string, c EtcdClient) (*TemplateResource, // setVars sets the Vars for template resource. func (t *TemplateResource) setVars() error { var err error - t.Vars, err = getValues(t.etcdClient, config.Prefix(), t.Keys) + t.Vars, err = etcdutil.GetValues(t.etcdClient, config.Prefix(), t.Keys) if err != nil { return err } @@ -212,7 +213,7 @@ func (t *TemplateResource) setFileMode() error { // ProcessTemplateResources is a convenience function that loads all the // template resources and processes them serially. Called from main. // It return an error if any. -func ProcessTemplateResources(c EtcdClient) []error { +func ProcessTemplateResources(c etcdutil.EtcdClient) []error { runErrors := make([]error, 0) var err error if c == nil { diff --git a/template_resource_test.go b/resource/template/template_resource_test.go similarity index 99% rename from template_resource_test.go rename to resource/template/template_resource_test.go index 317ad8345..bec8a2dd0 100644 --- a/template_resource_test.go +++ b/resource/template/template_resource_test.go @@ -1,4 +1,4 @@ -package main +package template import ( "io/ioutil" @@ -9,7 +9,7 @@ import ( "github.com/coreos/go-etcd/etcd" "github.com/kelseyhightower/confd/config" - "github.com/kelseyhightower/confd/etcdtest" + "github.com/kelseyhightower/confd/etcd/etcdtest" ) // createTempDirs is a helper function which creates temporary directories diff --git a/util.go b/resource/template/util.go similarity index 91% rename from util.go rename to resource/template/util.go index ed0b0d985..46cb08856 100644 --- a/util.go +++ b/resource/template/util.go @@ -1,4 +1,4 @@ -package main +package template import ( "os" diff --git a/util_test.go b/resource/template/util_test.go similarity index 96% rename from util_test.go rename to resource/template/util_test.go index 2fd50be36..a76e18a0f 100644 --- a/util_test.go +++ b/resource/template/util_test.go @@ -1,4 +1,4 @@ -package main +package template import ( "testing" diff --git a/version_test.go b/version_test.go new file mode 100644 index 000000000..06ab7d0f9 --- /dev/null +++ b/version_test.go @@ -0,0 +1 @@ +package main From cacbafb145cf4d518395401295c19e5e43360429 Mon Sep 17 00:00:00 2001 From: Kelsey Hightower Date: Sat, 2 Nov 2013 19:50:50 -0700 Subject: [PATCH 16/30] Rename template_resource* source files --- resource/template/{template_resource.go => resource.go} | 0 resource/template/{template_resource_test.go => resource_test.go} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename resource/template/{template_resource.go => resource.go} (100%) rename resource/template/{template_resource_test.go => resource_test.go} (100%) diff --git a/resource/template/template_resource.go b/resource/template/resource.go similarity index 100% rename from resource/template/template_resource.go rename to resource/template/resource.go diff --git a/resource/template/template_resource_test.go b/resource/template/resource_test.go similarity index 100% rename from resource/template/template_resource_test.go rename to resource/template/resource_test.go From e97190e446119e34364f63efc6a4116f0c7ff661 Mon Sep 17 00:00:00 2001 From: Kelsey Hightower Date: Sat, 2 Nov 2013 19:56:17 -0700 Subject: [PATCH 17/30] Remove useless test --- version_test.go | 1 - 1 file changed, 1 deletion(-) delete mode 100644 version_test.go diff --git a/version_test.go b/version_test.go deleted file mode 100644 index 06ab7d0f9..000000000 --- a/version_test.go +++ /dev/null @@ -1 +0,0 @@ -package main From 065cc9207e2d6840980e9c99ee9034e5d0e5d553 Mon Sep 17 00:00:00 2001 From: Kelsey Hightower Date: Sat, 2 Nov 2013 20:03:49 -0700 Subject: [PATCH 18/30] Add missing copyright notices to source files --- config/config_test.go | 3 +++ etcd/etcdtest/client.go | 3 +++ etcd/etcdtest/client_test.go | 3 +++ etcd/etcdutil/client.go | 3 +++ etcd/etcdutil/client_test.go | 3 +++ log/log_test.go | 3 +++ resource/template/fileinfo_darwin.go | 3 +++ resource/template/fileinfo_linux.go | 3 +++ resource/template/resource.go | 3 +++ resource/template/resource_test.go | 3 +++ resource/template/util.go | 5 +++-- resource/template/util_test.go | 3 +++ 12 files changed, 36 insertions(+), 2 deletions(-) diff --git a/config/config_test.go b/config/config_test.go index 3ad5aa6ec..242b9c480 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -1,3 +1,6 @@ +// Copyright (c) 2013 Kelsey Hightower. All rights reserved. +// Use of this source code is governed by the Apache License, Version 2.0 +// that can be found in the LICENSE file. package config import ( diff --git a/etcd/etcdtest/client.go b/etcd/etcdtest/client.go index 5b6e6204d..815c4c80b 100644 --- a/etcd/etcdtest/client.go +++ b/etcd/etcdtest/client.go @@ -1,3 +1,6 @@ +// Copyright (c) 2013 Kelsey Hightower. All rights reserved. +// Use of this source code is governed by the Apache License, Version 2.0 +// that can be found in the LICENSE file. package etcdtest import ( diff --git a/etcd/etcdtest/client_test.go b/etcd/etcdtest/client_test.go index bd7b7d9ff..4379c8d1a 100644 --- a/etcd/etcdtest/client_test.go +++ b/etcd/etcdtest/client_test.go @@ -1 +1,4 @@ +// Copyright (c) 2013 Kelsey Hightower. All rights reserved. +// Use of this source code is governed by the Apache License, Version 2.0 +// that can be found in the LICENSE file. package etcdtest diff --git a/etcd/etcdutil/client.go b/etcd/etcdutil/client.go index a984cc7f2..b3d29d26e 100644 --- a/etcd/etcdutil/client.go +++ b/etcd/etcdutil/client.go @@ -1,3 +1,6 @@ +// Copyright (c) 2013 Kelsey Hightower. All rights reserved. +// Use of this source code is governed by the Apache License, Version 2.0 +// that can be found in the LICENSE file. package etcdutil import ( diff --git a/etcd/etcdutil/client_test.go b/etcd/etcdutil/client_test.go index 511e3bb12..42571066e 100644 --- a/etcd/etcdutil/client_test.go +++ b/etcd/etcdutil/client_test.go @@ -1,3 +1,6 @@ +// Copyright (c) 2013 Kelsey Hightower. All rights reserved. +// Use of this source code is governed by the Apache License, Version 2.0 +// that can be found in the LICENSE file. package etcdutil import ( diff --git a/log/log_test.go b/log/log_test.go index 7330d5405..a4fe73d82 100644 --- a/log/log_test.go +++ b/log/log_test.go @@ -1 +1,4 @@ +// Copyright (c) 2013 Kelsey Hightower. All rights reserved. +// Use of this source code is governed by the Apache License, Version 2.0 +// that can be found in the LICENSE file. package log diff --git a/resource/template/fileinfo_darwin.go b/resource/template/fileinfo_darwin.go index 1a86cbe0e..4ced7bc2c 100644 --- a/resource/template/fileinfo_darwin.go +++ b/resource/template/fileinfo_darwin.go @@ -1,3 +1,6 @@ +// Copyright (c) 2013 Kelsey Hightower. All rights reserved. +// Use of this source code is governed by the Apache License, Version 2.0 +// that can be found in the LICENSE file. package template // A fileInfo describes a configuration file and is returned by fileStat. diff --git a/resource/template/fileinfo_linux.go b/resource/template/fileinfo_linux.go index 409073fad..ce4d38738 100644 --- a/resource/template/fileinfo_linux.go +++ b/resource/template/fileinfo_linux.go @@ -1,3 +1,6 @@ +// Copyright (c) 2013 Kelsey Hightower. All rights reserved. +// Use of this source code is governed by the Apache License, Version 2.0 +// that can be found in the LICENSE file. package template // A fileInfo describes a configuration file and is returned by fileStat. diff --git a/resource/template/resource.go b/resource/template/resource.go index be07931e2..21637d551 100644 --- a/resource/template/resource.go +++ b/resource/template/resource.go @@ -1,3 +1,6 @@ +// Copyright (c) 2013 Kelsey Hightower. All rights reserved. +// Use of this source code is governed by the Apache License, Version 2.0 +// that can be found in the LICENSE file. package template import ( diff --git a/resource/template/resource_test.go b/resource/template/resource_test.go index bec8a2dd0..c16d7e247 100644 --- a/resource/template/resource_test.go +++ b/resource/template/resource_test.go @@ -1,3 +1,6 @@ +// Copyright (c) 2013 Kelsey Hightower. All rights reserved. +// Use of this source code is governed by the Apache License, Version 2.0 +// that can be found in the LICENSE file. package template import ( diff --git a/resource/template/util.go b/resource/template/util.go index 46cb08856..9a2ea933c 100644 --- a/resource/template/util.go +++ b/resource/template/util.go @@ -1,10 +1,12 @@ +// Copyright (c) 2013 Kelsey Hightower. All rights reserved. +// Use of this source code is governed by the Apache License, Version 2.0 +// that can be found in the LICENSE file. package template import ( "os" ) - // IsFileExist reports whether path exits. func IsFileExist(fpath string) bool { if _, err := os.Stat(fpath); os.IsNotExist(err) { @@ -12,4 +14,3 @@ func IsFileExist(fpath string) bool { } return true } - diff --git a/resource/template/util_test.go b/resource/template/util_test.go index a76e18a0f..a2a0253b3 100644 --- a/resource/template/util_test.go +++ b/resource/template/util_test.go @@ -1,3 +1,6 @@ +// Copyright (c) 2013 Kelsey Hightower. All rights reserved. +// Use of this source code is governed by the Apache License, Version 2.0 +// that can be found in the LICENSE file. package template import ( From c2cee0adf2d13e9fd20354248c244a6cb27096c3 Mon Sep 17 00:00:00 2001 From: Kelsey Hightower Date: Sat, 2 Nov 2013 20:22:26 -0700 Subject: [PATCH 19/30] Code cleanup and update comments --- confd.go | 12 ++++++------ config/config.go | 26 ++++++-------------------- config/nodevar.go | 24 ++++++++++++++++++++++++ version.go | 2 +- 4 files changed, 37 insertions(+), 27 deletions(-) create mode 100644 config/nodevar.go diff --git a/confd.go b/confd.go index a70651790..d4ba4efdf 100644 --- a/confd.go +++ b/confd.go @@ -29,12 +29,12 @@ func init() { } func main() { - // Most flags are defined in the confd/config package which allow us to - // override configuration settings from the cli. Parse the flags now to - // make them active. + // Most flags are defined in the confd/config package which allows us to + // override configuration settings from the command line. Parse the flags now + // to make them active. flag.Parse() - // non-error messages are not printed by default, enable them now. - // If the "-q" flag was passed on the commandline non-error messages will + // Non-error messages are not printed by default, enable them now. + // If the "-q" flag was passed on the command line non-error messages will // not be printed. log.SetQuiet(quiet) log.Info("Starting confd") @@ -43,6 +43,7 @@ func main() { configFile = defaultConfigFile } } + // Initialize the global configuration. if err := config.LoadConfig(configFile); err != nil { log.Fatal(err.Error()) } @@ -63,7 +64,6 @@ func main() { } os.Exit(0) } - // By default we poll etcd every 30 seconds time.Sleep(time.Duration(config.Interval()) * time.Second) } } diff --git a/config/config.go b/config/config.go index 2ad846548..2668c60c5 100644 --- a/config/config.go +++ b/config/config.go @@ -17,12 +17,6 @@ import ( "github.com/kelseyhightower/confd/log" ) -// etcdHost -type etcdHost struct { - Hostname string - Port uint16 -} - var ( config Config nodes Nodes @@ -36,6 +30,12 @@ var ( etcdScheme string ) +// etcdHost +type etcdHost struct { + Hostname string + Port uint16 +} + func init() { flag.Var(&nodes, "n", "list of etcd nodes") flag.StringVar(&confdir, "c", "/etc/confd", "confd config directory") @@ -48,20 +48,6 @@ func init() { flag.BoolVar(&noop, "noop", false, "only show pending changes, don't sync configs.") } -// Nodes is a custom flag Var representing a list of etcd nodes. We use a custom -// Var to allow us to define more than one etcd node from the command line, and -// collect the results in a single value. -type Nodes []string - -func (n *Nodes) String() string { - return fmt.Sprintf("%d", *n) -} - -// Set appends the node to the etcd node list. -func (n *Nodes) Set(node string) error { - *n = append(*n, node) - return nil -} // Config represents the confd configuration settings. type Config struct { diff --git a/config/nodevar.go b/config/nodevar.go new file mode 100644 index 000000000..1e467c1b6 --- /dev/null +++ b/config/nodevar.go @@ -0,0 +1,24 @@ +// Copyright (c) 2013 Kelsey Hightower. All rights reserved. +// Use of this source code is governed by the Apache License, Version 2.0 +// that can be found in the LICENSE file. +package config + +import ( + "fmt" +) + +// Nodes is a custom flag Var representing a list of etcd nodes. We use a custom +// Var to allow us to define more than one etcd node from the command line, and +// collect the results in a single value. +type Nodes []string + +// String. +func (n *Nodes) String() string { + return fmt.Sprintf("%d", *n) +} + +// Set appends the node to the etcd node list. +func (n *Nodes) Set(node string) error { + *n = append(*n, node) + return nil +} diff --git a/version.go b/version.go index df39deed8..438c9ff38 100644 --- a/version.go +++ b/version.go @@ -1,3 +1,3 @@ package main -const version = "0.2.0" +const Version = "0.2.0" From 6f8def3a269d0a7021f030f176f86e67e510465d Mon Sep 17 00:00:00 2001 From: Kelsey Hightower Date: Sat, 2 Nov 2013 22:00:00 -0700 Subject: [PATCH 20/30] Don't use single letters for command line flags --- README.md | 6 +- config/config.go | 291 +++++++++++------------------ config/{nodevar.go => node_var.go} | 6 +- config/util.go | 63 +++++++ 4 files changed, 182 insertions(+), 184 deletions(-) rename config/{nodevar.go => node_var.go} (76%) create mode 100644 config/util.go diff --git a/README.md b/README.md index bd06f1f98..72c54b6e9 100644 --- a/README.md +++ b/README.md @@ -42,13 +42,13 @@ The following commands will process all the [template resources](https://github. The "/production" string will be prefixed to keys when querying etcd at http://127.0.0.1:4001. ``` -confd -i 30 -p '/production' -n 'http://127.0.0.1:4001' +confd -interval 30 -prefix '/production' -node 'http://127.0.0.1:4001' ``` ### Same as above in noop mode ``` -confd -noop -i 30 -p '/production' -n 'http://127.0.0.1:4001' +confd -interval 30 -prefix '/production' -node 'http://127.0.0.1:4001' -noop ``` ### Single run without polling @@ -64,7 +64,7 @@ confd -onetime Same as above but authenticate with client certificates. ``` -confd -onetime -key /etc/confd/ssl/client.key -cert /etc/confd/ssl/client.crt +confd -onetime -client-key /etc/confd/ssl/client.key -client-cert /etc/confd/ssl/client.crt ``` ### Lookup etcd nodes using SRV records diff --git a/config/config.go b/config/config.go index 2668c60c5..08a1dbc89 100644 --- a/config/config.go +++ b/config/config.go @@ -6,49 +6,28 @@ package config import ( "errors" "flag" - "fmt" "net" "net/url" "path/filepath" "strconv" - "strings" "github.com/BurntSushi/toml" "github.com/kelseyhightower/confd/log" ) var ( - config Config - nodes Nodes + clientCert string + clientKey string + config Config // holds the global confd config. confdir string + etcdNodes Nodes + etcdScheme string interval int + noop bool prefix string - clientCert string - clientKey string srvDomain string - noop bool - etcdScheme string ) -// etcdHost -type etcdHost struct { - Hostname string - Port uint16 -} - -func init() { - flag.Var(&nodes, "n", "list of etcd nodes") - flag.StringVar(&confdir, "c", "/etc/confd", "confd config directory") - flag.IntVar(&interval, "i", 600, "etcd polling interval") - flag.StringVar(&prefix, "p", "/", "etcd key path prefix") - flag.StringVar(&clientCert, "cert", "", "the client cert") - flag.StringVar(&clientKey, "key", "", "the client key") - flag.StringVar(&etcdScheme, "etcd-scheme", "http", "the etcd URI scheme. (http or https)") - flag.StringVar(&srvDomain, "srv-domain", "", "the domain to query for the etcd SRV record, i.e. example.com") - flag.BoolVar(&noop, "noop", false, "only show pending changes, don't sync configs.") -} - - // Config represents the confd configuration settings. type Config struct { Confd confd @@ -56,15 +35,27 @@ type Config struct { // confd represents the parsed configuration settings. type confd struct { - ConfDir string ClientCert string `toml:"client_cert"` ClientKey string `toml:"client_key"` - Interval int - Prefix string + ConfDir string EtcdNodes []string `toml:"etcd_nodes"` EtcdScheme string `toml:"etcd_scheme"` - Noop bool `toml:"noop"` - SRVDomain string `toml:"srv_domain"` + Interval int + Noop bool `toml:"noop"` + Prefix string + SRVDomain string `toml:"srv_domain"` +} + +func init() { + flag.StringVar(&clientCert, "client-cert", "", "the client cert") + flag.StringVar(&clientKey, "client-key", "", "the client key") + flag.StringVar(&confdir, "confdir", "/etc/confd", "confd conf directory") + flag.Var(&etcdNodes, "node", "list of etcd nodes") + flag.StringVar(&etcdScheme, "etcd-scheme", "http", "the etcd URI scheme. (http or https)") + flag.IntVar(&interval, "interval", 600, "etcd polling interval") + flag.BoolVar(&noop, "noop", false, "only show pending changes, don't sync configs.") + flag.StringVar(&prefix, "prefix", "/", "etcd key path prefix") + flag.StringVar(&srvDomain, "srv-domain", "", "the domain to query for the etcd SRV record, i.e. example.com") } // LoadConfig initializes the confd configuration by first setting defaults, @@ -77,11 +68,12 @@ func LoadConfig(path string) error { log.Warning("Skipping confd config file.") } else { log.Debug("Loading " + path) - if err := loadConfFile(path); err != nil { + _, err := toml.DecodeFile(path, &config) + if err != nil { return err } } - overrideConfig() + processFlags() if !isValidateEtcdScheme(config.Confd.EtcdScheme) { return errors.New("Invalid etcd scheme: " + config.Confd.EtcdScheme) } @@ -92,63 +84,14 @@ func LoadConfig(path string) error { return nil } -func setEtcdHosts() error { - scheme := config.Confd.EtcdScheme - hosts := make([]string, 0) - // If a domain name is given then lookup the etcd SRV record, and override - // all other etcd node settings. - if config.Confd.SRVDomain != "" { - etcdHosts, err := GetEtcdHostsFromSRV(config.Confd.SRVDomain) - if err != nil { - return errors.New("Cannot get etcd hosts from SRV records " + err.Error()) - } - for _, h := range etcdHosts { - uri := formatEtcdHostURI(scheme, h.Hostname, strconv.FormatUint(uint64(h.Port), 10)) - hosts = append(hosts, uri) - } - config.Confd.EtcdNodes = hosts - return nil - } - // No domain name was given, so just process the etcd node list. - // An etcdNode can be a URL, http://etcd.example.com:4001, or a host, etcd.example.com:4001. - for _, node := range config.Confd.EtcdNodes { - etcdURL, err := url.Parse(node) - if err != nil { - log.Error(err.Error()) - return err - } - if etcdURL.Scheme != "" && etcdURL.Host != "" { - if !isValidateEtcdScheme(etcdURL.Scheme) { - return errors.New("The etcd node list contains an invalid URL: " + node) - } - host, port, err := net.SplitHostPort(etcdURL.Host) - if err != nil { - return err - } - hosts = append(hosts, formatEtcdHostURI(etcdURL.Scheme, host, port)) - continue - } - // At this point node is not an etcd URL, i.e. http://etcd.example.com:4001, - // but a host:port string, i.e. etcd.example.com:4001 - host, port, err := net.SplitHostPort(node) - if err != nil { - return err - } - hosts = append(hosts, formatEtcdHostURI(scheme, host, port)) - } - config.Confd.EtcdNodes = hosts - return nil -} - -func formatEtcdHostURI(scheme, host, port string) string { - return fmt.Sprintf("%s://%s:%s", scheme, host, port) +// ClientCert returns the client cert path. +func ClientCert() string { + return config.Confd.ClientCert } -func isValidateEtcdScheme(scheme string) bool { - if scheme == "http" || scheme == "https" { - return true - } - return false +// ClientKey returns the client key path. +func ClientKey() string { + return config.Confd.ClientKey } // ConfigDir returns the path to the confd config dir. @@ -156,21 +99,6 @@ func ConfigDir() string { return filepath.Join(config.Confd.ConfDir, "conf.d") } -// SetConfDir. -func SetConfDir(path string) { - config.Confd.ConfDir = path -} - -// ClientCert returns the path to the client cert. -func ClientCert() string { - return config.Confd.ClientCert -} - -// ClientKey returns the path to the client key. -func ClientKey() string { - return config.Confd.ClientKey -} - // EtcdNodes returns a list of etcd node url strings. // For example: ["http://203.0.113.30:4001"] func EtcdNodes() []string { @@ -182,36 +110,41 @@ func Interval() int { return config.Confd.Interval } -// Noop +// Noop returns the state of noop mode. func Noop() bool { return config.Confd.Noop } -// SetNoop. -func SetNoop(enabled bool) { - config.Confd.Noop = enabled -} - // Prefix returns the etcd key prefix to use when querying etcd. func Prefix() string { return config.Confd.Prefix } -// SetPrefix -func SetPrefix(prefix string) { - config.Confd.Prefix = prefix +// SetConfDir sets the confd conf dir. +func SetConfDir(path string) { + config.Confd.ConfDir = path } -// TemplateDir returns the path to the directory of config file templates. -func TemplateDir() string { - return filepath.Join(config.Confd.ConfDir, "templates") +// SetNoop sets noop. +func SetNoop(enabled bool) { + config.Confd.Noop = enabled +} + +// SetPrefix sets the key prefix. +func SetPrefix(prefix string) { + config.Confd.Prefix = prefix } -// SRVDomain return the domain name used to query etcd SRV records. +// SRVDomain returns the domain name used in etcd SRV record lookups. func SRVDomain() string { return config.Confd.SRVDomain } +// TemplateDir returns the template directory path. +func TemplateDir() string { + return filepath.Join(config.Confd.ConfDir, "templates") +} + func setDefaults() { config = Config{ Confd: confd{ @@ -224,76 +157,80 @@ func setDefaults() { } } -// loadConfFile sets the etcd configuration settings from a file. -func loadConfFile(path string) error { - _, err := toml.DecodeFile(path, &config) - if err != nil { - return err +// setEtcdHosts. +func setEtcdHosts() error { + scheme := config.Confd.EtcdScheme + hosts := make([]string, 0) + // If a domain name is given then lookup the etcd SRV record, and override + // all other etcd node settings. + if config.Confd.SRVDomain != "" { + etcdHosts, err := getEtcdHostsFromSRV(config.Confd.SRVDomain) + if err != nil { + return errors.New("Cannot get etcd hosts from SRV records " + err.Error()) + } + for _, h := range etcdHosts { + uri := formatEtcdHostURL(scheme, h.Hostname, strconv.FormatUint(uint64(h.Port), 10)) + hosts = append(hosts, uri) + } + config.Confd.EtcdNodes = hosts + return nil + } + // No domain name was given, so just process the etcd node list. + // An etcdNode can be a URL, http://etcd.example.com:4001, or a host, etcd.example.com:4001. + for _, node := range config.Confd.EtcdNodes { + etcdURL, err := url.Parse(node) + if err != nil { + log.Error(err.Error()) + return err + } + if etcdURL.Scheme != "" && etcdURL.Host != "" { + if !isValidateEtcdScheme(etcdURL.Scheme) { + return errors.New("The etcd node list contains an invalid URL: " + node) + } + host, port, err := net.SplitHostPort(etcdURL.Host) + if err != nil { + return err + } + hosts = append(hosts, formatEtcdHostURL(etcdURL.Scheme, host, port)) + continue + } + // At this point node is not an etcd URL, i.e. http://etcd.example.com:4001, + // but a host:port string, i.e. etcd.example.com:4001 + host, port, err := net.SplitHostPort(node) + if err != nil { + return err + } + hosts = append(hosts, formatEtcdHostURL(scheme, host, port)) } + config.Confd.EtcdNodes = hosts return nil } -// override sets configuration settings based on values passed in through -// command line flags; overwriting current values. -func override(f *flag.Flag) { +// processFlags iterates through each flag set on the command line and +// overrides corresponding configuration settings. +func processFlags() { + flag.Visit(setConfigFromFlag) +} + +func setConfigFromFlag(f *flag.Flag) { switch f.Name { - case "c": - config.Confd.ConfDir = confdir - case "i": - config.Confd.Interval = interval - case "n": - config.Confd.EtcdNodes = nodes - case "p": - config.Confd.Prefix = prefix - case "cert": + case "client-cert": config.Confd.ClientCert = clientCert - case "key": + case "client-key": config.Confd.ClientKey = clientKey + case "confdir": + config.Confd.ConfDir = confdir + case "node": + config.Confd.EtcdNodes = etcdNodes + case "etcd-scheme": + config.Confd.EtcdScheme = etcdScheme + case "interval": + config.Confd.Interval = interval case "noop": config.Confd.Noop = noop + case "prefix": + config.Confd.Prefix = prefix case "srv-domain": config.Confd.SRVDomain = srvDomain - case "etcd-scheme": - config.Confd.EtcdScheme = etcdScheme - } -} - -// overrideConfig iterates through each flag set on the command line and -// overrides corresponding configuration settings. -func overrideConfig() { - flag.Visit(override) -} - -// GetEtcdHostsFromSRV returns a list of etcHost. -func GetEtcdHostsFromSRV(domain string) ([]*etcdHost, error) { - addrs, err := lookupEtcdSRV(domain) - if err != nil { - return nil, err - } - etcdHosts := etcdHostsFromSRV(addrs) - return etcdHosts, nil -} - -// lookupEtcdSrv tries to resolve an SRV query for the etcd service for the -// specified domain. -// -// lookupEtcdSRV constructs the DNS name to look up following RFC 2782. -// That is, it looks up _etcd._tcp.domain. -func lookupEtcdSRV(domain string) ([]*net.SRV, error) { - // Ignore the CNAME as we don't need it. - _, addrs, err := net.LookupSRV("etcd", "tcp", domain) - if err != nil { - return addrs, err - } - return addrs, nil -} - -// etcdHostsFromSRV converts an etcd SRV record to a list of etcdHost. -func etcdHostsFromSRV(addrs []*net.SRV) []*etcdHost { - hosts := make([]*etcdHost, 0) - for _, srv := range addrs { - hostname := strings.TrimRight(srv.Target, ".") - hosts = append(hosts, &etcdHost{Hostname: hostname, Port: srv.Port}) } - return hosts } diff --git a/config/nodevar.go b/config/node_var.go similarity index 76% rename from config/nodevar.go rename to config/node_var.go index 1e467c1b6..03bd93885 100644 --- a/config/nodevar.go +++ b/config/node_var.go @@ -7,12 +7,10 @@ import ( "fmt" ) -// Nodes is a custom flag Var representing a list of etcd nodes. We use a custom -// Var to allow us to define more than one etcd node from the command line, and -// collect the results in a single value. +// Nodes is a custom flag Var representing a list of etcd nodes. type Nodes []string -// String. +// String returns the string representation of a node var. func (n *Nodes) String() string { return fmt.Sprintf("%d", *n) } diff --git a/config/util.go b/config/util.go new file mode 100644 index 000000000..019708acb --- /dev/null +++ b/config/util.go @@ -0,0 +1,63 @@ +// Copyright (c) 2013 Kelsey Hightower. All rights reserved. +// Use of this source code is governed by the Apache License, Version 2.0 +// that can be found in the LICENSE file. +package config + +import ( + "fmt" + "net" + "strings" +) + +// etcdHost +type etcdHost struct { + Hostname string + Port uint16 +} + +// etcdHostsFromSRV converts an etcd SRV record to a list of etcdHost. +func etcdHostsFromSRV(addrs []*net.SRV) []*etcdHost { + hosts := make([]*etcdHost, 0) + for _, srv := range addrs { + hostname := strings.TrimRight(srv.Target, ".") + hosts = append(hosts, &etcdHost{Hostname: hostname, Port: srv.Port}) + } + return hosts +} + +// formatEtcdHostURL. +func formatEtcdHostURL(scheme, host, port string) string { + return fmt.Sprintf("%s://%s:%s", scheme, host, port) +} + +// getEtcdHostsFromSRV returns a list of etcHost. +func getEtcdHostsFromSRV(domain string) ([]*etcdHost, error) { + addrs, err := lookupEtcdSRV(domain) + if err != nil { + return nil, err + } + etcdHosts := etcdHostsFromSRV(addrs) + return etcdHosts, nil +} + +// lookupEtcdSrv tries to resolve an SRV query for the etcd service for the +// specified domain. +// +// lookupEtcdSRV constructs the DNS name to look up following RFC 2782. +// That is, it looks up _etcd._tcp.domain. +func lookupEtcdSRV(domain string) ([]*net.SRV, error) { + // Ignore the CNAME as we don't need it. + _, addrs, err := net.LookupSRV("etcd", "tcp", domain) + if err != nil { + return addrs, err + } + return addrs, nil +} + +// isValidateEtcdScheme. +func isValidateEtcdScheme(scheme string) bool { + if scheme == "http" || scheme == "https" { + return true + } + return false +} From 573a9a8f9e6953e542fba8e568a23e556b451479 Mon Sep 17 00:00:00 2001 From: Kelsey Hightower Date: Sat, 2 Nov 2013 22:09:30 -0700 Subject: [PATCH 21/30] Don't use single letter flags --- confd.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/confd.go b/confd.go index d4ba4efdf..cc52ed6d2 100644 --- a/confd.go +++ b/confd.go @@ -23,9 +23,9 @@ var ( ) func init() { - flag.StringVar(&configFile, "C", "", "confd config file") + flag.StringVar(&configFile, "config-file", "", "the confd config file") flag.BoolVar(&onetime, "onetime", false, "run once and exit") - flag.BoolVar(&quiet, "q", false, "silence non-error messages") + flag.BoolVar(&quiet, "quiet", false, "silence non-error messages") } func main() { From aae1d88902e95aed602dc7da02a5bec3093a6d36 Mon Sep 17 00:00:00 2001 From: Kelsey Hightower Date: Sat, 2 Nov 2013 23:38:48 -0700 Subject: [PATCH 22/30] Code clean up and minor refactor --- resource/template/fileinfo_darwin.go | 2 +- resource/template/fileinfo_linux.go | 2 +- resource/template/resource.go | 19 ++++++++++++------ resource/template/resource_test.go | 18 +++++++++++++++++ resource/template/util.go | 16 --------------- resource/template/util_test.go | 30 ---------------------------- 6 files changed, 33 insertions(+), 54 deletions(-) delete mode 100644 resource/template/util.go delete mode 100644 resource/template/util_test.go diff --git a/resource/template/fileinfo_darwin.go b/resource/template/fileinfo_darwin.go index 4ced7bc2c..40baa96fc 100644 --- a/resource/template/fileinfo_darwin.go +++ b/resource/template/fileinfo_darwin.go @@ -3,7 +3,7 @@ // that can be found in the LICENSE file. package template -// A fileInfo describes a configuration file and is returned by fileStat. +// fileInfo describes a configuration file and is returned by fileStat. type fileInfo struct { Uid uint32 Gid uint32 diff --git a/resource/template/fileinfo_linux.go b/resource/template/fileinfo_linux.go index ce4d38738..65f3942da 100644 --- a/resource/template/fileinfo_linux.go +++ b/resource/template/fileinfo_linux.go @@ -3,7 +3,7 @@ // that can be found in the LICENSE file. package template -// A fileInfo describes a configuration file and is returned by fileStat. +// fileInfo describes a configuration file and is returned by fileStat. type fileInfo struct { Uid uint32 Gid uint32 diff --git a/resource/template/resource.go b/resource/template/resource.go index 21637d551..b16a00c77 100644 --- a/resource/template/resource.go +++ b/resource/template/resource.go @@ -76,7 +76,7 @@ func (t *TemplateResource) setVars() error { // It returns an error if any. func (t *TemplateResource) createStageFile() error { t.Src = filepath.Join(config.TemplateDir(), t.Src) - if !IsFileExist(t.Src) { + if !isFileExist(t.Src) { return errors.New("Missing template: " + t.Src) } temp, err := ioutil.TempFile("", "") @@ -191,10 +191,9 @@ func (t *TemplateResource) process() error { } // setFileMode sets the FileMode. -// It returns an error if any. func (t *TemplateResource) setFileMode() error { if t.Mode == "" { - if !IsFileExist(t.Dest) { + if !isFileExist(t.Dest) { t.FileMode = 0644 } else { fi, err := os.Stat(t.Dest) @@ -215,7 +214,7 @@ func (t *TemplateResource) setFileMode() error { // ProcessTemplateResources is a convenience function that loads all the // template resources and processes them serially. Called from main. -// It return an error if any. +// It returns a list of errors if any. func ProcessTemplateResources(c etcdutil.EtcdClient) []error { runErrors := make([]error, 0) var err error @@ -246,7 +245,7 @@ func ProcessTemplateResources(c etcdutil.EtcdClient) []error { // fileStat return a fileInfo describing the named file. func fileStat(name string) (fi fileInfo, err error) { - if IsFileExist(name) { + if isFileExist(name) { f, err := os.Open(name) defer f.Close() if err != nil { @@ -270,7 +269,7 @@ func fileStat(name string) (fi fileInfo, err error) { // Unix permissions. The owner, group, and mode must match. // It return false in other cases. func sameConfig(src, dest string) (bool, error) { - if !IsFileExist(dest) { + if !isFileExist(dest) { return false, nil } d, err := fileStat(dest) @@ -298,3 +297,11 @@ func sameConfig(src, dest string) (bool, error) { } return true, nil } + +// isFileExist reports whether path exits. +func isFileExist(fpath string) bool { + if _, err := os.Stat(fpath); os.IsNotExist(err) { + return false + } + return true +} diff --git a/resource/template/resource_test.go b/resource/template/resource_test.go index c16d7e247..7f5595ac3 100644 --- a/resource/template/resource_test.go +++ b/resource/template/resource_test.go @@ -35,6 +35,8 @@ func createTempDirs() (string, error) { return confDir, nil } +var fakeFile = "/this/shoud/not/exist" + var templateResourceConfigTmpl = ` [template] src = "{{ .src }}" @@ -276,3 +278,19 @@ func TestSameConfigFalse(t *testing.T) { t.Errorf("Expected sameConfig(src, dest) to be %v, got %v", false, status) } } + +func TestIsFileExist(t *testing.T) { + result := isFileExist(fakeFile) + if result != false { + t.Errorf("Expected IsFileExist(%s) to be false, got %v", fakeFile, result) + } + f, err := ioutil.TempFile("", "") + if err != nil { + t.Fatal(err.Error()) + } + defer os.Remove(f.Name()) + result = isFileExist(f.Name()) + if result != true { + t.Errorf("Expected IsFileExist(%s) to be true, got %v", f.Name(), result) + } +} diff --git a/resource/template/util.go b/resource/template/util.go deleted file mode 100644 index 9a2ea933c..000000000 --- a/resource/template/util.go +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright (c) 2013 Kelsey Hightower. All rights reserved. -// Use of this source code is governed by the Apache License, Version 2.0 -// that can be found in the LICENSE file. -package template - -import ( - "os" -) - -// IsFileExist reports whether path exits. -func IsFileExist(fpath string) bool { - if _, err := os.Stat(fpath); os.IsNotExist(err) { - return false - } - return true -} diff --git a/resource/template/util_test.go b/resource/template/util_test.go deleted file mode 100644 index a2a0253b3..000000000 --- a/resource/template/util_test.go +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright (c) 2013 Kelsey Hightower. All rights reserved. -// Use of this source code is governed by the Apache License, Version 2.0 -// that can be found in the LICENSE file. -package template - -import ( - "testing" - "io/ioutil" - "os" -) - -var ( - fakeFile = "/this/shoud/not/exist" -) - -func TestIsFileExist(t *testing.T) { - result := IsFileExist(fakeFile) - if result != false { - t.Errorf("Expected IsFileExist(%s) to be false, got %v", fakeFile, result) - } - f, err := ioutil.TempFile("", "") - if err != nil { - t.Fatal(err.Error()) - } - defer os.Remove(f.Name()) - result = IsFileExist(f.Name()) - if result != true { - t.Errorf("Expected IsFileExist(%s) to be true, got %v", f.Name(), result) - } -} From 4c2a0effbd05ffe63176c0ad1ad1a307863f469b Mon Sep 17 00:00:00 2001 From: Kelsey Hightower Date: Sun, 3 Nov 2013 07:46:46 -0800 Subject: [PATCH 23/30] Better logging: new -verbose and -debug flags --- confd.go | 6 ++++++ config/config_test.go | 2 ++ log/log.go | 34 +++++++++++++++++++++++------- resource/template/resource_test.go | 7 ++++++ 4 files changed, 41 insertions(+), 8 deletions(-) diff --git a/confd.go b/confd.go index cc52ed6d2..5945b4335 100644 --- a/confd.go +++ b/confd.go @@ -17,15 +17,19 @@ import ( var ( configFile = "" + debug bool defaultConfigFile = "/etc/confd/confd.toml" onetime bool quiet bool + verbose bool ) func init() { flag.StringVar(&configFile, "config-file", "", "the confd config file") + flag.BoolVar(&debug, "debug", false, "enable debug logging") flag.BoolVar(&onetime, "onetime", false, "run once and exit") flag.BoolVar(&quiet, "quiet", false, "silence non-error messages") + flag.BoolVar(&verbose, "verbose", false, "enable verbose logging") } func main() { @@ -37,6 +41,8 @@ func main() { // If the "-q" flag was passed on the command line non-error messages will // not be printed. log.SetQuiet(quiet) + log.SetVerbose(verbose) + log.SetDebug(debug) log.Info("Starting confd") if configFile == "" { if IsFileExist(defaultConfigFile) { diff --git a/config/config_test.go b/config/config_test.go index 242b9c480..1708ff32a 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -5,9 +5,11 @@ package config import ( "testing" + "github.com/kelseyhightower/confd/log" ) func TestLoadConfig(t *testing.T) { + log.SetQuiet(true) var expected = struct { clientCert string clientKey string diff --git a/log/log.go b/log/log.go index e284c5a55..1324f9fe5 100644 --- a/log/log.go +++ b/log/log.go @@ -21,8 +21,11 @@ import ( // string will appear in all log entires. var tag string -// Silence non-error messages. -var quiet = true +var ( + quiet = false // Silence non-error messages. + verbose = false + debug = false +) func init() { tag = os.Args[0] @@ -34,13 +37,25 @@ func SetTag(t string) { } // SetQuiet sets quite mode. -func SetQuiet(q bool) { - quiet = q +func SetQuiet(enable bool) { + quiet = enable +} + +// SetDebug sets debug mode. +func SetDebug(enable bool) { + debug = enable +} + +// SetVerbose sets verbose mode. +func SetVerbose(enable bool) { + verbose = enable } // Debug logs a message with severity DEBUG. func Debug(msg string) { - write("DEBUG", msg) + if debug { + write("DEBUG", msg) + } } // Error logs a message with severity ERROR. @@ -61,7 +76,9 @@ func Info(msg string) { // Notice logs a message with severity NOTICE. func Notice(msg string) { - write("NOTICE", msg) + if verbose || debug { + write("NOTICE", msg) + } } // Warning logs a message with severity WARNING. @@ -78,9 +95,10 @@ func write(level, msg string) { hostname, _ := os.Hostname() switch level { case "DEBUG", "INFO", "NOTICE", "WARNING": - if !quiet { - w = os.Stdout + if quiet { + return } + w = os.Stdout case "ERROR": w = os.Stderr } diff --git a/resource/template/resource_test.go b/resource/template/resource_test.go index 7f5595ac3..f64d497d1 100644 --- a/resource/template/resource_test.go +++ b/resource/template/resource_test.go @@ -13,6 +13,7 @@ import ( "github.com/coreos/go-etcd/etcd" "github.com/kelseyhightower/confd/config" "github.com/kelseyhightower/confd/etcd/etcdtest" + "github.com/kelseyhightower/confd/log" ) // createTempDirs is a helper function which creates temporary directories @@ -57,6 +58,7 @@ keys = [ ` func TestProcessTemplateResources(t *testing.T) { + log.SetQuiet(true) // Setup temporary conf, config, and template directories. tempConfDir, err := createTempDirs() if err != nil { @@ -130,6 +132,7 @@ func TestProcessTemplateResources(t *testing.T) { } func TestProcessTemplateResourcesNoop(t *testing.T) { + log.SetQuiet(true) // Setup temporary conf, config, and template directories. tempConfDir, err := createTempDirs() if err != nil { @@ -205,6 +208,7 @@ func TestProcessTemplateResourcesNoop(t *testing.T) { } func TestBrokenTemplateResourceFile(t *testing.T) { + log.SetQuiet(true) tempFile, err := ioutil.TempFile("", "") defer os.Remove(tempFile.Name()) if err != nil { @@ -224,6 +228,7 @@ func TestBrokenTemplateResourceFile(t *testing.T) { } func TestSameConfigTrue(t *testing.T) { + log.SetQuiet(true) src, err := ioutil.TempFile("", "src") defer os.Remove(src.Name()) if err != nil { @@ -252,6 +257,7 @@ func TestSameConfigTrue(t *testing.T) { } func TestSameConfigFalse(t *testing.T) { + log.SetQuiet(true) src, err := ioutil.TempFile("", "src") defer os.Remove(src.Name()) if err != nil { @@ -280,6 +286,7 @@ func TestSameConfigFalse(t *testing.T) { } func TestIsFileExist(t *testing.T) { + log.SetQuiet(true) result := isFileExist(fakeFile) if result != false { t.Errorf("Expected IsFileExist(%s) to be false, got %v", fakeFile, result) From 0c863960e203a892ecc5b1224ab2caa243622232 Mon Sep 17 00:00:00 2001 From: Kelsey Hightower Date: Sun, 3 Nov 2013 08:02:46 -0800 Subject: [PATCH 24/30] Update comment in confd.go regarding logging. --- confd.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/confd.go b/confd.go index 5945b4335..5c9dac736 100644 --- a/confd.go +++ b/confd.go @@ -37,9 +37,8 @@ func main() { // override configuration settings from the command line. Parse the flags now // to make them active. flag.Parse() - // Non-error messages are not printed by default, enable them now. - // If the "-q" flag was passed on the command line non-error messages will - // not be printed. + // Configure logging. While you can enable debug and verbose logging, however + // if quiet is set to true then debug and verbose messages will not be printed. log.SetQuiet(quiet) log.SetVerbose(verbose) log.SetDebug(debug) From d6887727112eeaf6e5a8c4e1a07c973eae798198 Mon Sep 17 00:00:00 2001 From: Kelsey Hightower Date: Sun, 3 Nov 2013 08:31:08 -0800 Subject: [PATCH 25/30] Logging can be configured from the confd config file --- README.md | 3 +++ confd.go | 18 ++++++------------ config/config.go | 46 ++++++++++++++++++++++++++++++++++++++-------- 3 files changed, 47 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index 72c54b6e9..8e2fe15bb 100644 --- a/README.md +++ b/README.md @@ -90,6 +90,7 @@ and loaded from `/etc/confd/confd.toml` by default. Optional: +* `debug` (bool) - Enable debug logging. * `client_cert` (string) The cert file of the client. * `client_key` (string) The key file of the client. * `confdir` (string) - The path to confd configs. The default is /etc/confd. @@ -101,7 +102,9 @@ Optional: * `noop` (bool) - Enable noop mode. Process all template resource, but don't update target config. * `prefix` (string) - The prefix string to prefix to keys when making calls to etcd. The default is "/". +* `quiet` (bool) - Enable quiet logging. Only error messages are printed. * `srv_domain` (string) - The domain to query for etcd SRV records. +* `verbose` (bool) - Enable verbose logging. Example: diff --git a/confd.go b/confd.go index 5c9dac736..4821be7b7 100644 --- a/confd.go +++ b/confd.go @@ -17,19 +17,13 @@ import ( var ( configFile = "" - debug bool defaultConfigFile = "/etc/confd/confd.toml" onetime bool - quiet bool - verbose bool ) func init() { flag.StringVar(&configFile, "config-file", "", "the confd config file") - flag.BoolVar(&debug, "debug", false, "enable debug logging") flag.BoolVar(&onetime, "onetime", false, "run once and exit") - flag.BoolVar(&quiet, "quiet", false, "silence non-error messages") - flag.BoolVar(&verbose, "verbose", false, "enable verbose logging") } func main() { @@ -37,12 +31,6 @@ func main() { // override configuration settings from the command line. Parse the flags now // to make them active. flag.Parse() - // Configure logging. While you can enable debug and verbose logging, however - // if quiet is set to true then debug and verbose messages will not be printed. - log.SetQuiet(quiet) - log.SetVerbose(verbose) - log.SetDebug(debug) - log.Info("Starting confd") if configFile == "" { if IsFileExist(defaultConfigFile) { configFile = defaultConfigFile @@ -52,6 +40,12 @@ func main() { if err := config.LoadConfig(configFile); err != nil { log.Fatal(err.Error()) } + // Configure logging. While you can enable debug and verbose logging, however + // if quiet is set to true then debug and verbose messages will not be printed. + log.SetQuiet(config.Quiet()) + log.SetVerbose(config.Verbose()) + log.SetDebug(config.Debug()) + log.Info("Starting confd") // Create the etcd client upfront and use it for the life of the process. // The etcdClient is an http.Client and designed to be reused. log.Debug("Connecting to " + strings.Join(config.EtcdNodes(), ", ")) diff --git a/config/config.go b/config/config.go index 08a1dbc89..467c89eed 100644 --- a/config/config.go +++ b/config/config.go @@ -20,12 +20,15 @@ var ( clientKey string config Config // holds the global confd config. confdir string + debug bool etcdNodes Nodes etcdScheme string interval int noop bool prefix string + quiet bool srvDomain string + verbose bool ) // Config represents the confd configuration settings. @@ -35,18 +38,22 @@ type Config struct { // confd represents the parsed configuration settings. type confd struct { - ClientCert string `toml:"client_cert"` - ClientKey string `toml:"client_key"` - ConfDir string + Debug bool `toml:"debug"` + ClientCert string `toml:"client_cert"` + ClientKey string `toml:"client_key"` + ConfDir string `toml:"confdir"` EtcdNodes []string `toml:"etcd_nodes"` EtcdScheme string `toml:"etcd_scheme"` - Interval int - Noop bool `toml:"noop"` - Prefix string - SRVDomain string `toml:"srv_domain"` + Interval int `toml:"interval"` + Noop bool `toml:"noop"` + Prefix string `toml:"prefix"` + Quiet bool `toml:"quiet"` + SRVDomain string `toml:"srv_domain"` + Verbose bool `toml:"verbose"` } func init() { + flag.BoolVar(&debug, "debug", false, "enable debug logging") flag.StringVar(&clientCert, "client-cert", "", "the client cert") flag.StringVar(&clientKey, "client-key", "", "the client key") flag.StringVar(&confdir, "confdir", "/etc/confd", "confd conf directory") @@ -55,7 +62,9 @@ func init() { flag.IntVar(&interval, "interval", 600, "etcd polling interval") flag.BoolVar(&noop, "noop", false, "only show pending changes, don't sync configs.") flag.StringVar(&prefix, "prefix", "/", "etcd key path prefix") + flag.BoolVar(&quiet, "quiet", false, "enable quiet logging. Only error messages are printed.") flag.StringVar(&srvDomain, "srv-domain", "", "the domain to query for the etcd SRV record, i.e. example.com") + flag.BoolVar(&verbose, "verbose", false, "enable verbose logging") } // LoadConfig initializes the confd configuration by first setting defaults, @@ -84,6 +93,11 @@ func LoadConfig(path string) error { return nil } +// Debug reports whether debug mode is enabled. +func Debug() bool { + return config.Confd.Debug +} + // ClientCert returns the client cert path. func ClientCert() string { return config.Confd.ClientCert @@ -110,7 +124,7 @@ func Interval() int { return config.Confd.Interval } -// Noop returns the state of noop mode. +// Noop reports whether noop mode is enabled. func Noop() bool { return config.Confd.Noop } @@ -120,6 +134,16 @@ func Prefix() string { return config.Confd.Prefix } +// Quiet reports whether quiet mode is enabled. +func Quiet() bool { + return config.Confd.Quiet +} + +// Verbose reports whether verbose mode is enabled. +func Verbose() bool { + return config.Confd.Verbose +} + // SetConfDir sets the confd conf dir. func SetConfDir(path string) { config.Confd.ConfDir = path @@ -214,6 +238,8 @@ func processFlags() { func setConfigFromFlag(f *flag.Flag) { switch f.Name { + case "debug": + config.Confd.Debug = debug case "client-cert": config.Confd.ClientCert = clientCert case "client-key": @@ -230,7 +256,11 @@ func setConfigFromFlag(f *flag.Flag) { config.Confd.Noop = noop case "prefix": config.Confd.Prefix = prefix + case "quiet": + config.Confd.Quiet = quiet case "srv-domain": config.Confd.SRVDomain = srvDomain + case "verbose": + config.Confd.Verbose = verbose } } From b8946ed2bcf111e537e153e97aebcddd09baef5f Mon Sep 17 00:00:00 2001 From: Kelsey Hightower Date: Sun, 3 Nov 2013 15:33:49 -0800 Subject: [PATCH 26/30] Add more debug statements and warn if confdir does not exist --- config/config.go | 5 +++++ resource/template/resource.go | 18 ++++++++++++++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/config/config.go b/config/config.go index 467c89eed..885f6efb8 100644 --- a/config/config.go +++ b/config/config.go @@ -108,6 +108,11 @@ func ClientKey() string { return config.Confd.ClientKey } +// ConfDir returns the path to the confd config dir. +func ConfDir() string { + return config.Confd.ConfDir +} + // ConfigDir returns the path to the confd config dir. func ConfigDir() string { return filepath.Join(config.Confd.ConfDir, "conf.d") diff --git a/resource/template/resource.go b/resource/template/resource.go index b16a00c77..da0dc77a7 100644 --- a/resource/template/resource.go +++ b/resource/template/resource.go @@ -52,6 +52,7 @@ func NewTemplateResourceFromPath(path string, c etcdutil.EtcdClient) (*TemplateR return nil, errors.New("A valid EtcdClient is required.") } var tc *TemplateResourceConfig + log.Debug("Loading template resource from " + path) _, err := toml.DecodeFile(path, &tc) if err != nil { return nil, fmt.Errorf("Cannot process template resource %s - %s", path, err.Error()) @@ -63,6 +64,8 @@ func NewTemplateResourceFromPath(path string, c etcdutil.EtcdClient) (*TemplateR // setVars sets the Vars for template resource. func (t *TemplateResource) setVars() error { var err error + log.Debug("Retrieving keys from etcd") + log.Debug("Key prefix set to " + config.Prefix()) t.Vars, err = etcdutil.GetValues(t.etcdClient, config.Prefix(), t.Keys) if err != nil { return err @@ -76,6 +79,7 @@ func (t *TemplateResource) setVars() error { // It returns an error if any. func (t *TemplateResource) createStageFile() error { t.Src = filepath.Join(config.TemplateDir(), t.Src) + log.Debug("Using source template " + t.Src) if !isFileExist(t.Src) { return errors.New("Missing template: " + t.Src) } @@ -84,6 +88,7 @@ func (t *TemplateResource) createStageFile() error { os.Remove(temp.Name()) return err } + log.Debug("Compiling source template " + t.Src) tmpl := template.Must(template.ParseFiles(t.Src)) if err = tmpl.Execute(temp, t.Vars); err != nil { return err @@ -104,6 +109,7 @@ func (t *TemplateResource) createStageFile() error { func (t *TemplateResource) sync() error { staged := t.StageFile.Name() defer os.Remove(staged) + log.Debug("Comparing canidate config to " + t.Dest) ok, err := sameConfig(staged, t.Dest) if err != nil { log.Error(err.Error()) @@ -113,12 +119,13 @@ func (t *TemplateResource) sync() error { return nil } if !ok { - log.Info("syncing " + t.Dest) + log.Info("Syncing target config " + t.Dest) if t.CheckCmd != "" { if err := t.check(); err != nil { return errors.New("Config check failed: " + err.Error()) } } + log.Debug("Overwriting target config " + t.Dest) if err := os.Rename(staged, t.Dest); err != nil { return err } @@ -128,7 +135,7 @@ func (t *TemplateResource) sync() error { } } } else { - log.Info(t.Dest + " in sync") + log.Info("Target config " + t.Dest + " in sync") } return nil } @@ -222,12 +229,18 @@ func ProcessTemplateResources(c etcdutil.EtcdClient) []error { runErrors = append(runErrors, errors.New("An etcd client is required")) return runErrors } + log.Debug("Loading template resources from confdir " + config.ConfDir()) + if !isFileExist(config.ConfDir()) { + log.Warning(fmt.Sprintf("Cannot load template resources confdir '%s' does not exist", config.ConfDir())) + return runErrors + } paths, err := filepath.Glob(filepath.Join(config.ConfigDir(), "*toml")) if err != nil { runErrors = append(runErrors, err) return runErrors } for _, p := range paths { + log.Debug("Processing template resource " + p) t, err := NewTemplateResourceFromPath(p, c) if err != nil { runErrors = append(runErrors, err) @@ -239,6 +252,7 @@ func ProcessTemplateResources(c etcdutil.EtcdClient) []error { log.Error(err.Error()) continue } + log.Debug("Processing of template resource " + p + " complete successfully") } return runErrors } From 6ab7c72ccaea4a7975352839ddabff4180240aa8 Mon Sep 17 00:00:00 2001 From: Kelsey Hightower Date: Sun, 3 Nov 2013 15:53:57 -0800 Subject: [PATCH 27/30] Add more logging statements --- confd.go | 5 +++-- resource/template/resource.go | 9 +++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/confd.go b/confd.go index 4821be7b7..833afe386 100644 --- a/confd.go +++ b/confd.go @@ -37,6 +37,7 @@ func main() { } } // Initialize the global configuration. + log.Debug("Loading confd configuration") if err := config.LoadConfig(configFile); err != nil { log.Fatal(err.Error()) } @@ -45,10 +46,10 @@ func main() { log.SetQuiet(config.Quiet()) log.SetVerbose(config.Verbose()) log.SetDebug(config.Debug()) - log.Info("Starting confd") + log.Notice("Starting confd") // Create the etcd client upfront and use it for the life of the process. // The etcdClient is an http.Client and designed to be reused. - log.Debug("Connecting to " + strings.Join(config.EtcdNodes(), ", ")) + log.Notice("etcd nodes set to " + strings.Join(config.EtcdNodes(), ", ")) etcdClient, err := etcdutil.NewEtcdClient(config.EtcdNodes(), config.ClientCert(), config.ClientKey()) if err != nil { log.Fatal(err.Error()) diff --git a/resource/template/resource.go b/resource/template/resource.go index da0dc77a7..5ddb2c3af 100644 --- a/resource/template/resource.go +++ b/resource/template/resource.go @@ -109,17 +109,17 @@ func (t *TemplateResource) createStageFile() error { func (t *TemplateResource) sync() error { staged := t.StageFile.Name() defer os.Remove(staged) - log.Debug("Comparing canidate config to " + t.Dest) + log.Debug("Comparing candidate config to " + t.Dest) ok, err := sameConfig(staged, t.Dest) if err != nil { log.Error(err.Error()) } if config.Noop() { - log.Warning("In noop mode, not updating " + t.Dest) + log.Warning("Noop mode enabled " + t.Dest + " will not be modified") return nil } if !ok { - log.Info("Syncing target config " + t.Dest) + log.Info("Target config " + t.Dest + " out of sync") if t.CheckCmd != "" { if err := t.check(); err != nil { return errors.New("Config check failed: " + err.Error()) @@ -134,6 +134,7 @@ func (t *TemplateResource) sync() error { return err } } + log.Info("Target config " + t.Dest + " has been updated") } else { log.Info("Target config " + t.Dest + " in sync") } @@ -252,7 +253,7 @@ func ProcessTemplateResources(c etcdutil.EtcdClient) []error { log.Error(err.Error()) continue } - log.Debug("Processing of template resource " + p + " complete successfully") + log.Debug("Processing of template resource " + p + " complete") } return runErrors } From 4fa992cf05f70c2e5df58ab74cb91f6212aa1e86 Mon Sep 17 00:00:00 2001 From: Kelsey Hightower Date: Sun, 3 Nov 2013 16:05:33 -0800 Subject: [PATCH 28/30] log a message if SRV domain is set --- config/config.go | 1 + 1 file changed, 1 insertion(+) diff --git a/config/config.go b/config/config.go index 885f6efb8..31761b59c 100644 --- a/config/config.go +++ b/config/config.go @@ -193,6 +193,7 @@ func setEtcdHosts() error { // If a domain name is given then lookup the etcd SRV record, and override // all other etcd node settings. if config.Confd.SRVDomain != "" { + log.Info("SRV domain set to " + config.Confd.SRVDomain) etcdHosts, err := getEtcdHostsFromSRV(config.Confd.SRVDomain) if err != nil { return errors.New("Cannot get etcd hosts from SRV records " + err.Error()) From 59fbc9625cf927bfe8dbc4eb5a6be364a985a4b8 Mon Sep 17 00:00:00 2001 From: Kelsey Hightower Date: Sun, 3 Nov 2013 19:23:39 -0800 Subject: [PATCH 29/30] Update README --- README.md | 218 +++++++++++++----------------------------------------- 1 file changed, 52 insertions(+), 166 deletions(-) diff --git a/README.md b/README.md index 8e2fe15bb..d347d9020 100644 --- a/README.md +++ b/README.md @@ -15,207 +15,93 @@ README version 0.2.0 * Mailing list: [Google Groups](https://groups.google.com/forum/#!forum/confd-users) * Website: [www.confd.io](http://www.confd.io) -## Getting Started +## Quick Start -### Installing confd +Before we begin be sure to [download and install confd](https://github.com/kelseyhightower/confd/wiki/Installation). -Download the latest binary from [Github](https://github.com/kelseyhightower/confd/releases). +### Add keys to etcd -### Building - -You can build confd from source: - -``` -git clone https://github.com/kelseyhightower/confd.git -cd confd -go build -``` - -This will produce the `confd` binary in the current directory. - -## Usage - -The following commands will process all the [template resources](https://github.com/kelseyhightower/confd#template-resources) found under `/etc/confd/conf.d`. - -### Poll the etcd cluster in 30 second intervals - -The "/production" string will be prefixed to keys when querying etcd at http://127.0.0.1:4001. +This guide assumes you have a working [etcd](https://github.com/coreos/etcd#getting-started) server up and running and the ability to add new keys. Using the `etcdctl` command line tool add the following keys and values to etcd: ``` -confd -interval 30 -prefix '/production' -node 'http://127.0.0.1:4001' +etcdctl set /myapp/database/url db.example.com +etcdctl set /myapp/database/user rob ``` -### Same as above in noop mode - -``` -confd -interval 30 -prefix '/production' -node 'http://127.0.0.1:4001' -noop -``` - -### Single run without polling - -Using default settings run one time and exit. - -``` -confd -onetime -``` - -### Client authentication - -Same as above but authenticate with client certificates. +### Create the confdir -``` -confd -onetime -client-key /etc/confd/ssl/client.key -client-cert /etc/confd/ssl/client.crt -``` - -### Lookup etcd nodes using SRV records +The confdir is where template resource configs and source templates are stored. The default confdir is `/etc/confd`. Create the confdir by executing the following command: -``` -dig SRV _etcd._tcp.confd.io -... -;; ANSWER SECTION: -_etcd._tcp.confd.io. 300 IN SRV 1 50 4001 etcd0.confd.io. -_etcd._tcp.confd.io. 300 IN SRV 2 50 4001 etcd1.confd.io. +```Bash +sudo mkdir -p /etc/confd/{conf.d,templates} ``` -``` -confd -srv-domain example.com -etcd-scheme https -``` +You don't have to use the default `confdir` location. For example you can create the confdir under your home directory. Then you tell confd to use the new `confdir` via the `-confdir` flag. -confd would connect to the nodes at `["https://etcd0.confd.io:4001", "https://etcd1.confd.io:4001"]` - -## Configuration - -The confd configuration file is written in [TOML](https://github.com/mojombo/toml) -and loaded from `/etc/confd/confd.toml` by default. - -Optional: - -* `debug` (bool) - Enable debug logging. -* `client_cert` (string) The cert file of the client. -* `client_key` (string) The key file of the client. -* `confdir` (string) - The path to confd configs. The default is /etc/confd. -* `etcd_nodes` (array of strings) - An array of etcd cluster nodes. The default - is ["http://127.0.0.1:4001"]. -* `etcd_scheme` (string) - The etcd scheme to use. Must be 'http' or 'https' -* `interval` (int) - The number of seconds to wait between calls to etcd. The - default is 600. -* `noop` (bool) - Enable noop mode. Process all template resource, but don't update target config. -* `prefix` (string) - The prefix string to prefix to keys when making calls to - etcd. The default is "/". -* `quiet` (bool) - Enable quiet logging. Only error messages are printed. -* `srv_domain` (string) - The domain to query for etcd SRV records. -* `verbose` (bool) - Enable verbose logging. - -Example: - -```TOML -[confd] -confdir = "/etc/confd" -interval = 600 -prefix = "/" -etcd_nodes = [ - "http://127.0.0.1:4001", -] -client_cert = "/etc/confd/ssl/client.crt" -client_key = "/etc/confd/ssl/client.key" +```Bash +mkdir -p ~/confd/{conf.d,templates} ``` -## Template Resources - -Template resources are written in TOML and define a single template resource. -Template resources are stored under the `confdir/conf.d` directory. - -Required: - -* `dest` (string) - output file where the template should be rendered. -* `keys` (array of strings) - An array of etcd keys. Keys will be looked up - with the configured prefix. -* `src` (string) - relative path of a the [configuration template](https://github.com/kelseyhightower/confd#templates). +### Create a template resource config -Optional: +Template resources are defined in [TOML](https://github.com/mojombo/toml) config files under the `confdir` (i.e. ~/confd/conf.d/*.toml). -* `group` (string) - name of the group that should own the file. -* `mode` (string) - mode the file should be in. -* `owner` (string) - name of the user that should own the file. -* `reload_cmd` (string) - command to reload config. -* `check_cmd` (string) - command to check config. Use `{{ .src }}` to reference - the rendered source template. +Lets create a simple template resource to manage the `/tmp/myconfig.conf` configuration file. -Example: - -```TOML +```toml [template] -src = "nginx.conf.tmpl" -dest = "/etc/nginx/nginx.conf" -owner = "root" -group = "root" -mode = "0644" +src = "myconfig.conf.tmpl" +dest = "/tmp/myconfig.conf" keys = [ - "/nginx", + "/myapp/database/url", + "/myapp/database/user", ] -check_cmd = "/usr/sbin/nginx -t -c {{ .src }}" -reload_cmd = "/usr/sbin/service nginx restart" ``` -## Templates +Save the file under the `confdir` directory, i.e. `~/confd/conf.d/myconfig.toml` -Templates define a single application configration template. -Templates are stored under the `confdir/templates` directory. +### Create the source template -Templates are written in Go's [`text/template`](http://golang.org/pkg/text/template/). +Source templates are plain old [Golang text templates](http://golang.org/pkg/text/template/#pkg-overview), and are stored under the `confdir` templates directory. Create the following source template and save it as `~/confd/templates/myconfig.conf.tmpl` -Etcd keys are treated as paths and automatically transformed into keys for retrieval in templates. Underscores are used in place of forward slashes. _Values retrived from Etcd are never modified._ -For example `/foo/bar` becomes `foo_bar`. +``` +# This a comment +[myconfig] +database_url = {{ .myapp_database_url }} +database_user = {{ .myapp_database_user }} +``` -`foo_bar` is accessed as `{{ .foo_bar }}` +### Processing template resources +confd supports two modes of operation, daemon and onetime mode. In daemon mode, confd runs in the foreground and processing template resources every 5 mins by default. For this tutorial we are going to use onetime mode. + +Assuming you etcd server is running at http://127.0.0.1:4001 you can run the following command to process the `~/confd/conf.d/myconfig.toml` template resource: -Example: ``` -$ etcdctl set /nginx/domain 'example.com' -$ etcdctl set /nginx/root '/var/www/example_dotcom' -$ etcdctl set /nginx/worker_processes '2' +confd -verbose -onetime -node 'http://127.0.0.1:4001' -confdir ~/confd ``` - - -`$ cat /etc/confd/templates/nginx.conf.tmpl`: +Output: ``` -worker_processes {{ .nginx_worker_processes }}; - -server { - listen 80; - server_name www.{{ .nginx_domain }}; - access_log /var/log/nginx/{{ .nginx_domain }}.access.log; - error_log /var/log/nginx/{{ .nginx_domain }}.log; - - location / { - root {{ .nginx_root }}; - index index.html index.htm; - } -} +2013-11-03T18:00:47-08:00 confd[21294]: NOTICE Starting confd +2013-11-03T18:00:47-08:00 confd[21294]: NOTICE etcd nodes set to http://127.0.0.1:4001 +2013-11-03T18:00:47-08:00 confd[21294]: INFO Target config /tmp/myconfig.conf out of sync +2013-11-03T18:00:47-08:00 confd[21294]: INFO Target config /tmp/myconfig.conf has been updated ``` -Will produce `/etc/nginx/nginx.conf`: +The `dest` config should now be in sync with the template resource configuration. + ``` -worker_processes 2; - -server { - listen 80; - server_name www.example.com; - access_log /var/log/nginx/example.com.access.log; - error_log /var/log/nginx/example.com.error.log; - - location / { - root /var/www/example_dotcom; - index index.html index.htm; - } -} +cat /tmp/myconfig.conf ``` -Go's [`text/template`](http://golang.org/pkg/text/template/) package is very powerful. For more details on it's capabilities see its [documentation.](http://golang.org/pkg/text/template/) - -## Extras +Output: +``` +# This a comment +[myconfig] +database_url = db.example.com +database_user = rob +``` -### Configuration Management +## Next steps -- [confd-cookbook](https://github.com/rjocoleman/confd-cookbook) +Checkout the [confd wiki](https://github.com/kelseyhightower/confd/wiki/_pages) for more docs. From 587f94d6a82eb740f00201dbb31f3c572ae99f04 Mon Sep 17 00:00:00 2001 From: Kelsey Hightower Date: Sun, 3 Nov 2013 19:26:54 -0800 Subject: [PATCH 30/30] Update README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d347d9020..4ad436bbe 100644 --- a/README.md +++ b/README.md @@ -104,4 +104,4 @@ database_user = rob ## Next steps -Checkout the [confd wiki](https://github.com/kelseyhightower/confd/wiki/_pages) for more docs. +Checkout the [confd wiki](https://github.com/kelseyhightower/confd/wiki/_pages) for more docs and [usage examples](https://github.com/kelseyhightower/confd/wiki/Usage-Examples).