From cde4632b2954b510e72efe2a7373d2c1d37456e9 Mon Sep 17 00:00:00 2001 From: Richard Li <98242479+RichhLi@users.noreply.github.com> Date: Mon, 3 Jul 2023 21:13:21 -0700 Subject: [PATCH] support using json for registry config (#1680) * support using json for registry config Signed-off-by: Richard Li * simplify code Signed-off-by: Richard Li * Update pkg/app/bootstrap.go Co-authored-by: Keming Signed-off-by: Richard Li <98242479+RichhLi@users.noreply.github.com> --------- Signed-off-by: Richard Li Signed-off-by: Richard Li <98242479+RichhLi@users.noreply.github.com> Co-authored-by: Richard Li Co-authored-by: Keming --- pkg/app/bootstrap.go | 119 +++++++++++++++++++++++-- pkg/driver/docker/docker.go | 3 +- pkg/driver/nerdctl/nerdctl.go | 2 +- pkg/util/buildkitutil/buildkit.go | 40 ++++++--- pkg/util/buildkitutil/buildkit_test.go | 94 +++++++++++++++---- 5 files changed, 217 insertions(+), 41 deletions(-) diff --git a/pkg/app/bootstrap.go b/pkg/app/bootstrap.go index 2dc421937..067b74a04 100644 --- a/pkg/app/bootstrap.go +++ b/pkg/app/bootstrap.go @@ -15,6 +15,7 @@ package app import ( + "encoding/json" "fmt" "os" "path/filepath" @@ -75,6 +76,11 @@ var CommandBootstrap = &cli.Command{ Name: "registry", Usage: "Specify the registry to pull the image from", Aliases: []string{"r"}, + Value: "docker.io", + }, + &cli.StringFlag{ + Name: "registry-config", + Usage: "Path to a JSON file containing registry configuration. Cannot be used with 'registry' or 'registry-ca-keypair'", }, }, @@ -91,6 +97,9 @@ func bootstrap(clicontext *cli.Context) error { }, { "registry CA keypair", registryCA, + }, { + "registry json config", + registryJSONConfig, }, { "autocomplete", autocomplete, @@ -112,16 +121,25 @@ func bootstrap(clicontext *cli.Context) error { } func registryCA(clicontext *cli.Context) error { + configFile := clicontext.String("registry-config") ca := clicontext.String("registry-ca-keypair") + registry := clicontext.String("registry") + if len(ca) == 0 { return nil } + + // We only need this check in registryCA because it is called before registryJSONConfig + if len(configFile) > 0 && len(ca) > 0 { + return errors.New("only one of `registry-config` and `registry-ca-keypair` can be used") + } + mirror := clicontext.String("dockerhub-mirror") if len(mirror) == 0 { return errors.New("`registry-ca-keypair` should be used with `dockerhub-mirror`") } - // parse ca/key/cert + // Parse ca/key/cert kvPairs := strings.Split(ca, ",") if len(kvPairs) != 3 { return errors.New("`registry-ca-keypair` requires ca/key/cert 3 part separated by ','") @@ -146,7 +164,8 @@ func registryCA(clicontext *cli.Context) error { if !exist { return errors.Newf("file %s doesn't exist", kv[1]) } - path, err := fileutil.ConfigFile(fmt.Sprintf("registry_%s.pem", kv[0])) + + path, err := fileutil.ConfigFile(fmt.Sprintf("%s_%s.pem", registry, kv[0])) if err != nil { return errors.Wrap(err, "failed to get the envd config file path") } @@ -166,6 +185,69 @@ func registryCA(clicontext *cli.Context) error { return nil } +func registryJSONConfig(clicontext *cli.Context) error { + configFile := clicontext.String("registry-config") + if len(configFile) == 0 { + return nil + } + + // Check if file exists + exist, err := fileutil.FileExists(configFile) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to parse file path %s", configFile)) + } + if !exist { + return errors.Newf("file %s doesn't exist", configFile) + } + + config := buildkitutil.BuildkitConfig{} + configJson, err := os.ReadFile(configFile) + if err != nil { + return errors.Wrap(err, "Failed to read registry config file") + } + if err := json.Unmarshal(configJson, &config); err != nil { + return errors.Wrap(err, "Failed to parse registry config file") + } + + // Check for required keys in each registry + for i, registry := range config.Registries { + if registry.Name == "" { + return errors.Newf("`name` key is required in the config for registry at index %d", i) + } + + // Check for optional keys and if they exist, ensure they point to existing files + optionalKeys := map[string]string{"ca": registry.Ca, "cert": registry.Cert, "key": registry.Key} + for key, value := range optionalKeys { + if value != "" { + exist, err := fileutil.FileExists(value) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to parse file path %s", value)) + } + if !exist { + return errors.Newf("file %s doesn't exist", value) + } + + // Read the file + content, err := os.ReadFile(value) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to read the %s file for registry %s", key, registry.Name)) + } + + // Write the content to the envd config directory + envdConfigPath, err := fileutil.ConfigFile(fmt.Sprintf("%s_%s.pem", registry.Name, key)) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to get the envd config file path for %s of registry %s", key, registry.Name)) + } + + if err = os.WriteFile(envdConfigPath, content, 0644); err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to store the %s file for registry %s", key, registry.Name)) + } + } + } + } + return nil +} + func sshKey(clicontext *cli.Context) error { sshKeyPair := clicontext.StringSlice("ssh-keypair") @@ -240,7 +322,7 @@ func sshKey(clicontext *cli.Context) error { return nil default: - return errors.Errorf("Invalid ssh-keypair flag") + return errors.Newf("Invalid ssh-keypair flag") } } @@ -289,14 +371,33 @@ func buildkit(clicontext *cli.Context) error { } logrus.Debug("bootstrap the buildkitd container") - var bkClient buildkitd.Client - config := buildkitutil.BuildkitConfig{ - Registry: clicontext.String("registry"), - Mirror: clicontext.String("dockerhub-mirror"), - UseHTTP: clicontext.Bool("use-http"), - SetCA: clicontext.IsSet("registry-ca-keypair"), + // Populate the BuildkitConfig struct + config := buildkitutil.BuildkitConfig{} + + configFile := clicontext.String("registry-config") + if len(configFile) != 0 { + configJson, err := os.ReadFile(configFile) + if err != nil { + return errors.Wrap(err, "Failed to read registry config file") + } + if err := json.Unmarshal(configJson, &config); err != nil { + return errors.Wrap(err, "Failed to parse registry config file") + } + } else if len(clicontext.String("registry-ca-keypair")) != 0 { + // The values of Ca, Cert, and Key don't actually matter since we already copied their contents to the envd config directory and mounted to `/etc/registry`. + // So instead of parsing registry-ca-keypair again, we'll just put the default value. + // This is to ensure that buildkitConfigTemplate parses properly. + config.Registries = append(config.Registries, buildkitutil.Registry{ + Name: clicontext.String("registry"), + Ca: "/etc/registry", + Cert: "/etc/registry", + Key: "/etc/registry", + UseHttp: clicontext.Bool("use-http"), + Mirror: clicontext.String("dockerhub-mirror"), + }) } + var bkClient buildkitd.Client if c.Builder == types.BuilderTypeMoby { bkClient, err = buildkitd.NewMobyClient(clicontext.Context, c.Builder, c.BuilderAddress, &config) diff --git a/pkg/driver/docker/docker.go b/pkg/driver/docker/docker.go index 67f0bbc42..866b01d1e 100644 --- a/pkg/driver/docker/docker.go +++ b/pkg/driver/docker/docker.go @@ -207,13 +207,14 @@ func (c dockerClient) StartBuildkitd(ctx context.Context, tag, name string, bc * AutoRemove: true, } - if bc.SetCA { + if len(bc.Registries) > 0 { hostConfig.Mounts = append(hostConfig.Mounts, mount.Mount{ Type: mount.TypeBind, Source: fileutil.DefaultConfigDir, Target: buildkitdCertPath, }) } + cfg, err := bc.String() if err != nil { return "", errors.Wrap(err, "failed to generate buildkit config") diff --git a/pkg/driver/nerdctl/nerdctl.go b/pkg/driver/nerdctl/nerdctl.go index 3f14fc7e4..fb99e1559 100644 --- a/pkg/driver/nerdctl/nerdctl.go +++ b/pkg/driver/nerdctl/nerdctl.go @@ -83,7 +83,7 @@ func (nc *nerdctlClient) StartBuildkitd(ctx context.Context, tag, name string, b if !existed { buildkitdCmd := "buildkitd" // TODO: support mirror CA keypair - if bc.Registry != "" || bc.Mirror != "" || bc.UseHTTP { + if len(bc.Registries) > 0 { cfg, err := bc.String() if err != nil { return "", errors.Wrap(err, "failed to generate buildkit config") diff --git a/pkg/util/buildkitutil/buildkit.go b/pkg/util/buildkitutil/buildkit.go index 161dbd966..e5998eb6a 100644 --- a/pkg/util/buildkitutil/buildkit.go +++ b/pkg/util/buildkitutil/buildkit.go @@ -20,21 +20,37 @@ import ( ) const buildkitConfigTemplate = ` -[registry."{{ if .Registry }}{{ .Registry }}{{ else }}docker.io{{ end }}"]{{ if .Mirror }} - mirrors = ["{{ .Mirror }}"]{{ end }} - http = {{ .UseHTTP }} - {{ if .SetCA}}ca=["/etc/registry/ca.pem"] - [[registry."{{ if .Registry }}{{ .Registry }}{{ else }}docker.io{{ end }}".keypair]] - key="/etc/registry/key.pem" - cert="/etc/registry/cert.pem" - {{ end }} +[registry] +{{- range $registry := .Registries }} + [registry."{{ if $registry.Name }}{{ $registry.Name }}{{ else }}docker.io{{ end }}"] + {{- if $registry.UseHttp }} + http = true + {{- end }} + {{- if $registry.Mirror }} + mirrors = ["{{ $registry.Mirror }}"] + {{- end }} + {{- if $registry.Ca }} + ca=["/etc/registry/{{ $registry.Name }}_ca.pem"] + {{- end }} + {{- if and $registry.Cert $registry.Key }} + [[registry."{{ if $registry.Name }}{{ $registry.Name }}{{ else }}docker.io{{ end }}".keypair]] + key="/etc/registry/{{ $registry.Name }}_key.pem" + cert="/etc/registry/{{ $registry.Name }}_cert.pem" + {{- end }} +{{- end }} ` +type Registry struct { + Name string `json:"name"` + Ca string `json:"ca"` + Cert string `json:"cert"` + Key string `json:"key"` + UseHttp bool `json:"use_http"` + Mirror string `json:"mirror"` +} + type BuildkitConfig struct { - Registry string - Mirror string - UseHTTP bool - SetCA bool + Registries []Registry `json:"registries"` } func (c *BuildkitConfig) String() (string, error) { diff --git a/pkg/util/buildkitutil/buildkit_test.go b/pkg/util/buildkitutil/buildkit_test.go index f613e1d16..147e2c9c0 100644 --- a/pkg/util/buildkitutil/buildkit_test.go +++ b/pkg/util/buildkitutil/buildkit_test.go @@ -28,36 +28,94 @@ func TestBuildkitWithRegistry(t *testing.T) { }{ { BuildkitConfig{ - Registry: "registry.example.com", - Mirror: "https://mirror.example.com", - UseHTTP: true, + Registries: []Registry{ + { + Name: "registry.example.com", + Ca: "/etc/registry/ca.pem", + Cert: "/etc/registry/cert.pem", + Key: "/etc/registry/key.pem", + UseHttp: false, + Mirror: "https://mirror.example.com", + }, + }, }, ` -[registry."registry.example.com"] - mirrors = ["https://mirror.example.com"] - http = true +[registry] + [registry."registry.example.com"] + mirrors = ["https://mirror.example.com"] + ca=["/etc/registry/registry.example.com_ca.pem"] + [[registry."registry.example.com".keypair]] + key="/etc/registry/registry.example.com_key.pem" + cert="/etc/registry/registry.example.com_cert.pem" `, }, { BuildkitConfig{ - Registry: "registry.example.com", - SetCA: true, + Registries: []Registry{ + { + Name: "registry.example.com", + UseHttp: true, + Mirror: "https://mirror.example.com", + }, + { + Name: "docker.io", + Mirror: "https://mirror.example.com", + }, + }, }, ` -[registry."registry.example.com"] - http = false - ca=["/etc/registry/ca.pem"] - [[registry."registry.example.com".keypair]] - key="/etc/registry/key.pem" - cert="/etc/registry/cert.pem" +[registry] + [registry."registry.example.com"] + http = true + mirrors = ["https://mirror.example.com"] + [registry."docker.io"] + mirrors = ["https://mirror.example.com"] `, }, { - BuildkitConfig{}, + BuildkitConfig{ + Registries: []Registry{}, + }, ` -[registry."docker.io"] - http = false - `, +[registry] +`, + }, + { + BuildkitConfig{ + Registries: []Registry{ + { + Name: "registry1.example.com", + Ca: "/etc/registry/ca1.pem", + Cert: "/etc/registry/cert1.pem", + Key: "/etc/registry/key1.pem", + UseHttp: true, + Mirror: "https://mirror.example.com", + }, + { + Name: "registry2.example.com", + Ca: "/etc/registry/ca2.pem", + Cert: "/etc/registry/cert2.pem", + Key: "/etc/registry/key2.pem", + Mirror: "https://mirror.example.com", + }, + }, + }, + ` +[registry] + [registry."registry1.example.com"] + http = true + mirrors = ["https://mirror.example.com"] + ca=["/etc/registry/registry1.example.com_ca.pem"] + [[registry."registry1.example.com".keypair]] + key="/etc/registry/registry1.example.com_key.pem" + cert="/etc/registry/registry1.example.com_cert.pem" + [registry."registry2.example.com"] + mirrors = ["https://mirror.example.com"] + ca=["/etc/registry/registry2.example.com_ca.pem"] + [[registry."registry2.example.com".keypair]] + key="/etc/registry/registry2.example.com_key.pem" + cert="/etc/registry/registry2.example.com_cert.pem" +`, }, }