added support for private Docker registries

pkg/app/bootstrap.go

@@ -15,6 +15,7 @@
 package app
 
 import (
+	"encoding/json"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -55,32 +56,33 @@ var CommandBootstrap = &cli.Command{
 			Usage:   "Dockerhub mirror to use",
 			Aliases: []string{"m"},
 		},
-		&cli.StringFlag{
-			Name:    "registry-ca-keypair",
-			Usage:   "Specify the ca/key/cert file path for the private registry (format: 'ca=/etc/config/ca.pem,key=/etc/config/key.pem,cert=/etc/config/cert.pem')",
-			Aliases: []string{"ca"},
-		},
 		&cli.StringSliceFlag{
 			Name: "ssh-keypair",
 			Usage: fmt.Sprintf("Manually specify ssh key pair as `publicKey,privateKey`. Envd will generate a keypair at %s and %s if not specified",
 				sshconfig.GetPublicKeyOrPanic(), sshconfig.GetPrivateKeyOrPanic()),
 			Aliases: []string{"k"},
 		},
-		&cli.BoolFlag{
-			Name:  "use-http",
-			Usage: "Use HTTP instead of HTTPS for the registry",
-			Value: false,
-		},
 		&cli.StringFlag{
-			Name:    "registry",
-			Usage:   "Specify the registry to pull the image from",
-			Aliases: []string{"r"},
+			Name:  "registry-config",
+			Usage: "Path to a JSON file containing registry configuration",
 		},
 	},
 
 	Action: bootstrap,
 }
 
+type Registry struct {
+	Name    string `json:"name"`
+	Ca      string `json:"ca"`
+	Cert    string `json:"cert"`
+	Key     string `json:"key"`
+	UseHttp bool   `json:"use_http"`
+}
+
+type RegistriesData struct {
+	Registries []Registry `json:"registries"`
+}
+
 func bootstrap(clicontext *cli.Context) error {
 	stages := []struct {
 		Name string
@@ -89,8 +91,8 @@ func bootstrap(clicontext *cli.Context) error {
 		"SSH Key",
 		sshKey,
 	}, {
-		"registry CA keypair",
-		registryCA,
+		"registry-config",
+		registryConfig,
 	}, {
 		"autocomplete",
 		autocomplete,
@@ -111,57 +113,65 @@ func bootstrap(clicontext *cli.Context) error {
 	return nil
 }
 
-func registryCA(clicontext *cli.Context) error {
-	ca := clicontext.String("registry-ca-keypair")
-	if len(ca) == 0 {
+func registryConfig(clicontext *cli.Context) error {
+	configFile := clicontext.String("registry-config")
+	if len(configFile) == 0 {
 		return nil
 	}
-	mirror := clicontext.String("dockerhub-mirror")
-	if len(mirror) == 0 {
-		return errors.New("`registry-ca-keypair` should be used with `dockerhub-mirror`")
+
+	// Check if file exists
+	exist, err := fileutil.FileExists(configFile)
+	if err != nil {
+		return errors.Wrap(err, fmt.Sprintf("failed to parse file path %s", configFile))
+	}
+	if !exist {
+		return errors.Newf("file %s doesn't exist", configFile)
 	}
 
-	// parse ca/key/cert
-	kvPairs := strings.Split(ca, ",")
-	if len(kvPairs) != 3 {
-		return errors.New("`registry-ca-keypair` requires ca/key/cert 3 part separated by ','")
+	var data RegistriesData
+	configJson, err := os.ReadFile(configFile)
+	if err != nil {
+		return errors.Wrap(err, "Failed to read registry config file")
 	}
-	names := []string{"ca", "cert", "key"}
-	for _, pair := range kvPairs {
-		kv := strings.SplitN(pair, "=", 2)
-		index := -1
-		for i, name := range names {
-			if name == kv[0] {
-				index = i
-				break
-			}
-		}
-		if index == -1 {
-			return errors.Newf("parse error: `%s` is not a valid ca/key/cert key or it's duplicated")
-		}
-		exist, err := fileutil.FileExists(kv[1])
-		if err != nil {
-			return errors.Wrap(err, fmt.Sprintf("failed to parse file path %s", pair))
-		}
-		if !exist {
-			return errors.Newf("file %s doesn't exist", kv[1])
-		}
-		path, err := fileutil.ConfigFile(fmt.Sprintf("registry_%s.pem", kv[0]))
-		if err != nil {
-			return errors.Wrap(err, "failed to get the envd config file path")
-		}
-		content, err := os.ReadFile(kv[1])
-		if err != nil {
-			return errors.Wrap(err, "failed to read the file")
-		}
-		if err = os.WriteFile(path, content, 0644); err != nil {
-			return errors.Wrap(err, "failed to store the CA file")
-		}
-		names = append(names[:index], names[index+1:]...)
+	if err := json.Unmarshal(configJson, &data); err != nil {
+		return errors.Wrap(err, "Failed to parse registry config file")
 	}
 
-	if len(names) != 0 {
-		return errors.Newf("registry %s are not provided", names)
+	// Check for required keys in each registry
+	for i, registry := range data.Registries {
+		if registry.Name == "" {
+			return errors.Newf("`name` key is required in the config for registry at index %d", i)
+		}
+
+		// Check for optional keys and if they exist, ensure they point to existing files
+		optionalKeys := map[string]string{"ca": registry.Ca, "cert": registry.Cert, "key": registry.Key}
+		for key, value := range optionalKeys {
+			if value != "" {
+				exist, err := fileutil.FileExists(value)
+				if err != nil {
+					return errors.Wrap(err, fmt.Sprintf("failed to parse file path %s", value))
+				}
+				if !exist {
+					return errors.Newf("file %s doesn't exist", value)
+				}
+
+				// Read the file
+				content, err := os.ReadFile(value)
+				if err != nil {
+					return errors.Wrap(err, fmt.Sprintf("failed to read the %s file for registry %s", key, registry.Name))
+				}
+
+				// Write the content to the envd config directory
+				envdConfigPath, err := fileutil.ConfigFile(fmt.Sprintf("%s_%s.pem", registry.Name, key))
+				if err != nil {
+					return errors.Wrap(err, fmt.Sprintf("failed to get the envd config file path for %s of registry %s", key, registry.Name))
+				}
+
+				if err = os.WriteFile(envdConfigPath, content, 0644); err != nil {
+					return errors.Wrap(err, fmt.Sprintf("failed to store the %s file for registry %s", key, registry.Name))
+				}
+			}
+		}
 	}
 	return nil
 }
@@ -240,7 +250,7 @@ func sshKey(clicontext *cli.Context) error {
 		return nil
 
 	default:
-		return errors.Errorf("Invalid ssh-keypair flag")
+		return errors.Newf("Invalid ssh-keypair flag")
 	}
 }
 
@@ -290,11 +300,29 @@ func buildkit(clicontext *cli.Context) error {
 
 	logrus.Debug("bootstrap the buildkitd container")
 	var bkClient buildkitd.Client
+	var data RegistriesData
+
+	// Populate the BuildkitConfig struct
 	config := buildkitutil.BuildkitConfig{
-		Registry: clicontext.String("registry"),
-		Mirror:   clicontext.String("dockerhub-mirror"),
-		UseHTTP:  clicontext.Bool("use-http"),
-		SetCA:    clicontext.IsSet("registry-ca-keypair"),
+		Mirror: clicontext.String("dockerhub-mirror"),
+	}
+
+	configFile := clicontext.String("registry-config")
+	if len(configFile) != 0 {
+		configJson, err := os.ReadFile(configFile)
+		if err != nil {
+			return errors.Wrap(err, "Failed to read registry config file")
+		}
+		if err := json.Unmarshal(configJson, &data); err != nil {
+			return errors.Wrap(err, "Failed to parse registry config file")
+		}
+		for _, registry := range data.Registries {
+			config.RegistryName = append(config.RegistryName, registry.Name)
+			config.CaPath = append(config.CaPath, registry.Ca)
+			config.CertPath = append(config.CertPath, registry.Cert)
+			config.KeyPath = append(config.KeyPath, registry.Key)
+			config.UseHTTP = append(config.UseHTTP, registry.UseHttp)
+		}
 	}
 
 	if c.Builder == types.BuilderTypeMoby {

pkg/driver/docker/docker.go

@@ -207,13 +207,14 @@ func (c dockerClient) StartBuildkitd(ctx context.Context, tag, name string, bc *
 		AutoRemove: true,
 	}
 
-	if bc.SetCA {
+	if len(bc.RegistryName) > 0 {
 		hostConfig.Mounts = append(hostConfig.Mounts, mount.Mount{
 			Type:   mount.TypeBind,
 			Source: fileutil.DefaultConfigDir,
 			Target: buildkitdCertPath,
 		})
 	}
+
 	cfg, err := bc.String()
 	if err != nil {
 		return "", errors.Wrap(err, "failed to generate buildkit config")

pkg/driver/nerdctl/nerdctl.go

@@ -83,7 +83,7 @@ func (nc *nerdctlClient) StartBuildkitd(ctx context.Context, tag, name string, b
 	if !existed {
 		buildkitdCmd := "buildkitd"
 		// TODO: support mirror CA keypair
-		if bc.Registry != "" || bc.Mirror != "" || bc.UseHTTP {
+		if len(bc.RegistryName) > 0 || bc.Mirror != "" || len(bc.UseHTTP) > 0 {
 			cfg, err := bc.String()
 			if err != nil {
 				return "", errors.Wrap(err, "failed to generate buildkit config")

pkg/util/buildkitutil/buildkit.go

@@ -20,21 +20,33 @@ import (
 )
 
 const buildkitConfigTemplate = `
-[registry."{{ if .Registry }}{{ .Registry }}{{ else }}docker.io{{ end }}"]{{ if .Mirror }}
-  mirrors = ["{{ .Mirror }}"]{{ end }}
-  http = {{ .UseHTTP }}
-  {{ if .SetCA}}ca=["/etc/registry/ca.pem"]
-  [[registry."{{ if .Registry }}{{ .Registry }}{{ else }}docker.io{{ end }}".keypair]]
-	key="/etc/registry/key.pem"
-	cert="/etc/registry/cert.pem"
-  {{ end }}
+[registry]
+{{- range $index, $value := .RegistryName }}
+  [registry."{{ if $value }}{{ $value }}{{ else }}docker.io{{ end }}"]
+    {{- if index $.UseHTTP $index }}
+    http = true
+    {{- end }}
+    {{- if $.Mirror }}
+    mirrors = ["{{ $.Mirror }}"]
+    {{- end }}
+    {{- if index $.CaPath $index }}
+    ca=["/etc/registry/{{ $value }}_ca.pem"]
+    {{- end }}
+    {{- if and (index $.CertPath $index) (index $.KeyPath $index) }}
+    [[registry."{{ if $value }}{{ $value }}{{ else }}docker.io{{ end }}".keypair]]
+      key="/etc/registry/{{ $value }}_key.pem"
+      cert="/etc/registry/{{ $value }}_cert.pem"
+    {{- end }}
+{{- end }}
 `
 
 type BuildkitConfig struct {
-	Registry string
-	Mirror   string
-	UseHTTP  bool
-	SetCA    bool
+	RegistryName []string
+	CaPath       []string
+	CertPath     []string
+	KeyPath      []string
+	UseHTTP      []bool
+	Mirror       string
 }
 
 func (c *BuildkitConfig) String() (string, error) {

pkg/util/buildkitutil/buildkit_test.go

@@ -28,36 +28,79 @@ func TestBuildkitWithRegistry(t *testing.T) {
 	}{
 		{
 			BuildkitConfig{
-				Registry: "registry.example.com",
-				Mirror:   "https://mirror.example.com",
-				UseHTTP:  true,
+				RegistryName: []string{"registry.example.com"},
+				CaPath:       []string{"/etc/registry/ca.pem"},
+				CertPath:     []string{"/etc/registry/cert.pem"},
+				KeyPath:      []string{"/etc/registry/key.pem"},
+				UseHTTP:      []bool{false},
+				Mirror:       "https://mirror.example.com",
 			},
 			`
-[registry."registry.example.com"]
-  mirrors = ["https://mirror.example.com"]
-  http = true
+[registry]
+  [registry."registry.example.com"]
+    mirrors = ["https://mirror.example.com"]
+    ca=["/etc/registry/registry.example.com_ca.pem"]
+    [[registry."registry.example.com".keypair]]
+      key="/etc/registry/registry.example.com_key.pem"
+      cert="/etc/registry/registry.example.com_cert.pem"
 `,
 		},
 		{
 			BuildkitConfig{
-				Registry: "registry.example.com",
-				SetCA:    true,
+				RegistryName: []string{"registry.example.com", "docker.io"},
+				CaPath:       []string{"", ""},
+				CertPath:     []string{"", ""},
+				KeyPath:      []string{"", ""},
+				UseHTTP:      []bool{true, false},
+				Mirror:       "https://mirror.example.com",
 			},
 			`
-[registry."registry.example.com"]
-  http = false
-  ca=["/etc/registry/ca.pem"]
-  [[registry."registry.example.com".keypair]]
-	key="/etc/registry/key.pem"
-	cert="/etc/registry/cert.pem"
+[registry]
+  [registry."registry.example.com"]
+    http = true
+    mirrors = ["https://mirror.example.com"]
+  [registry."docker.io"]
+    mirrors = ["https://mirror.example.com"]
 `,
 		},
 		{
-			BuildkitConfig{},
+			BuildkitConfig{
+				RegistryName: []string{},
+				CaPath:       []string{},
+				CertPath:     []string{},
+				KeyPath:      []string{},
+				UseHTTP:      []bool{},
+				Mirror:       "",
+			},
 			`
-[registry."docker.io"]
-  http = false
-			`,
+[registry]
+`,
+		},
+		{
+			BuildkitConfig{
+				RegistryName: []string{"registry1.example.com", "registry2.example.com"},
+				CaPath:       []string{"/etc/registry/ca1.pem", "/etc/registry/ca2.pem"},
+				CertPath:     []string{"/etc/registry/cert1.pem", "/etc/registry/cert2.pem"},
+				KeyPath:      []string{"/etc/registry/key1.pem", "/etc/registry/key2.pem"},
+				UseHTTP:      []bool{true, false},
+				Mirror:       "https://mirror.example.com",
+			},
+			`
+[registry]
+  [registry."registry1.example.com"]
+    http = true
+    mirrors = ["https://mirror.example.com"]
+    ca=["/etc/registry/registry1.example.com_ca.pem"]
+    [[registry."registry1.example.com".keypair]]
+      key="/etc/registry/registry1.example.com_key.pem"
+      cert="/etc/registry/registry1.example.com_cert.pem"
+  [registry."registry2.example.com"]
+    mirrors = ["https://mirror.example.com"]
+    ca=["/etc/registry/registry2.example.com_ca.pem"]
+    [[registry."registry2.example.com".keypair]]
+      key="/etc/registry/registry2.example.com_key.pem"
+      cert="/etc/registry/registry2.example.com_cert.pem"
+`,
 		},
 	}