feat: user could set airgapped install script path (#140)

Signed-off-by: nasusoba <zhengrunyu2020@outlook.com>

user could customize etcd proxy image

Signed-off-by: nasusoba <zhengrunyu2020@outlook.com>

add test for resolveEtcdProxyFile

bootstrap/api/v1beta1/conversion.go

@@ -43,6 +43,8 @@ func (c *KThreesConfig) ConvertTo(dstRaw ctrlconversion.Hub) error {
 	dst.Spec.ServerConfig.DeprecatedDisableExternalCloudProvider = restored.Spec.ServerConfig.DeprecatedDisableExternalCloudProvider
 	dst.Spec.ServerConfig.DisableCloudController = restored.Spec.ServerConfig.DisableCloudController
 	dst.Spec.ServerConfig.SystemDefaultRegistry = restored.Spec.ServerConfig.SystemDefaultRegistry
+	dst.Spec.ServerConfig.EtcdProxyImage = restored.Spec.ServerConfig.EtcdProxyImage
+	dst.Spec.AgentConfig.AirGappedInstallScriptPath = restored.Spec.AgentConfig.AirGappedInstallScriptPath
 	return nil
 }
 
@@ -96,6 +98,8 @@ func (r *KThreesConfigTemplate) ConvertTo(dstRaw ctrlconversion.Hub) error {
 	dst.Spec.Template.Spec.ServerConfig.DeprecatedDisableExternalCloudProvider = restored.Spec.Template.Spec.ServerConfig.DeprecatedDisableExternalCloudProvider
 	dst.Spec.Template.Spec.ServerConfig.DisableCloudController = restored.Spec.Template.Spec.ServerConfig.DisableCloudController
 	dst.Spec.Template.Spec.ServerConfig.SystemDefaultRegistry = restored.Spec.Template.Spec.ServerConfig.SystemDefaultRegistry
+	dst.Spec.Template.Spec.ServerConfig.EtcdProxyImage = restored.Spec.Template.Spec.ServerConfig.EtcdProxyImage
+	dst.Spec.Template.Spec.AgentConfig.AirGappedInstallScriptPath = restored.Spec.Template.Spec.AgentConfig.AirGappedInstallScriptPath
 	return nil
 }
 
@@ -159,3 +163,8 @@ func Convert_v1beta2_KThreesServerConfig_To_v1beta1_KThreesServerConfig(in *boot
 
 	return nil
 }
+
+// Convert_v1beta2_KThreesAgentConfig_To_v1beta1_KThreesAgentConfig is an autogenerated conversion function.
+func Convert_v1beta2_KThreesAgentConfig_To_v1beta1_KThreesAgentConfig(in *bootstrapv1beta2.KThreesAgentConfig, out *KThreesAgentConfig, s conversion.Scope) error { //nolint: stylecheck
+	return autoConvert_v1beta2_KThreesAgentConfig_To_v1beta1_KThreesAgentConfig(in, out, s)
+}

bootstrap/api/v1beta2/kthreesconfig_types.go

@@ -124,6 +124,10 @@ type KThreesServerConfig struct {
 	// SystemDefaultRegistry defines private registry to be used for all system images
 	// +optional
 	SystemDefaultRegistry string `json:"systemDefaultRegistry,omitempty"`
+
+	// Customized etcd proxy image for management cluster to communicate with workload cluster etcd (default: "alpine/socat")
+	// +optional
+	EtcdProxyImage string `json:"etcdProxyImage,omitempty"`
 }
 
 type KThreesAgentConfig struct {
@@ -154,10 +158,16 @@ type KThreesAgentConfig struct {
 
 	// AirGapped is a boolean value to define if the bootstrapping should be air-gapped,
 	// basically supposing that online container registries and k3s install scripts are not reachable.
-	// User should prepare docker image, k3s binary, and put the install script in `/opt/install.sh`
+	// User should prepare docker image, k3s binary, and put the install script in AirGappedInstallScriptPath (default path: "/opt/install.sh")
 	// on all nodes in the air-gap environment.
 	// +optional
 	AirGapped bool `json:"airGapped,omitempty"`
+
+	// AirGappedInstallScriptPath is the path to the install script in the air-gapped environment.
+	// The install script should be prepared by the user. The value is only
+	// used when AirGapped is set to true (default: "/opt/install.sh").
+	// +optional
+	AirGappedInstallScriptPath string `json:"airGappedInstallScriptPath,omitempty"`
 }
 
 // KThreesConfigStatus defines the observed state of KThreesConfig.

bootstrap/config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigs.yaml

@@ -326,9 +326,15 @@ spec:
                     description: |-
                       AirGapped is a boolean value to define if the bootstrapping should be air-gapped,
                       basically supposing that online container registries and k3s install scripts are not reachable.
-                      User should prepare docker image, k3s binary, and put the install script in `/opt/install.sh`
+                      User should prepare docker image, k3s binary, and put the install script in AirGappedInstallScriptPath (default path: "/opt/install.sh")
                       on all nodes in the air-gap environment.
                     type: boolean
+                  airGappedInstallScriptPath:
+                    description: |-
+                      AirGappedInstallScriptPath is the path to the install script in the air-gapped environment.
+                      The install script should be prepared by the user. The value is only
+                      used when AirGapped is set to true (default: "/opt/install.sh").
+                    type: string
                   kubeProxyArgs:
                     description: KubeProxyArgs Customized flag for kube-proxy process
                     items:
@@ -471,6 +477,10 @@ spec:
                       the ''cloud-provider=external'' kubelet argument. (default:
                       false)'
                     type: boolean
+                  etcdProxyImage:
+                    description: 'Customized etcd proxy image for management cluster
+                      to communicate with workload cluster etcd (default: "alpine/socat")'
+                    type: string
                   httpsListenPort:
                     description: 'HTTPSListenPort HTTPS listen port (default: 6443)'
                     type: string

bootstrap/config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigtemplates.yaml

@@ -281,9 +281,15 @@ spec:
                             description: |-
                               AirGapped is a boolean value to define if the bootstrapping should be air-gapped,
                               basically supposing that online container registries and k3s install scripts are not reachable.
-                              User should prepare docker image, k3s binary, and put the install script in `/opt/install.sh`
+                              User should prepare docker image, k3s binary, and put the install script in AirGappedInstallScriptPath (default path: "/opt/install.sh")
                               on all nodes in the air-gap environment.
                             type: boolean
+                          airGappedInstallScriptPath:
+                            description: |-
+                              AirGappedInstallScriptPath is the path to the install script in the air-gapped environment.
+                              The install script should be prepared by the user. The value is only
+                              used when AirGapped is set to true (default: "/opt/install.sh").
+                            type: string
                           kubeProxyArgs:
                             description: KubeProxyArgs Customized flag for kube-proxy
                               process
@@ -432,6 +438,11 @@ spec:
                               the ''cloud-provider=external'' kubelet argument. (default:
                               false)'
                             type: boolean
+                          etcdProxyImage:
+                            description: 'Customized etcd proxy image for management
+                              cluster to communicate with workload cluster etcd (default:
+                              "alpine/socat")'
+                            type: string
                           httpsListenPort:
                             description: 'HTTPSListenPort HTTPS listen port (default:
                               6443)'

bootstrap/controllers/kthreesconfig_controller.go

@@ -17,9 +17,11 @@ limitations under the License.
 package controllers
 
 import (
+	"bytes"
 	"context"
 	"errors"
 	"fmt"
+	"html/template"
 	"time"
 
 	"github.com/go-logr/logr"
@@ -249,23 +251,24 @@ func (r *KThreesConfigReconciler) joinControlplane(ctx context.Context, scope *S
 	}
 
 	if scope.Config.Spec.IsEtcdEmbedded() {
-		etcdProxyFile := bootstrapv1.File{
-			Path:        etcd.EtcdProxyDaemonsetYamlLocation,
-			Content:     etcd.EtcdProxyDaemonsetYaml,
-			Owner:       "root:root",
-			Permissions: "0640",
+		etcdProxyFile, err := r.resolveEtcdProxyFile(scope.Config)
+		if err != nil {
+			conditions.MarkFalse(scope.Config, bootstrapv1.DataSecretAvailableCondition, bootstrapv1.DataSecretGenerationFailedReason, clusterv1.ConditionSeverityWarning, err.Error())
+			return fmt.Errorf("failed to resolve etcd proxy file: %w", err)
 		}
-		files = append(files, etcdProxyFile)
+
+		files = append(files, *etcdProxyFile)
 	}
 
 	cpInput := &cloudinit.ControlPlaneInput{
 		BaseUserData: cloudinit.BaseUserData{
-			PreK3sCommands:  scope.Config.Spec.PreK3sCommands,
-			PostK3sCommands: scope.Config.Spec.PostK3sCommands,
-			AdditionalFiles: files,
-			ConfigFile:      workerConfigFile,
-			K3sVersion:      scope.Config.Spec.Version,
-			AirGapped:       scope.Config.Spec.AgentConfig.AirGapped,
+			PreK3sCommands:             scope.Config.Spec.PreK3sCommands,
+			PostK3sCommands:            scope.Config.Spec.PostK3sCommands,
+			AdditionalFiles:            files,
+			ConfigFile:                 workerConfigFile,
+			K3sVersion:                 scope.Config.Spec.Version,
+			AirGapped:                  scope.Config.Spec.AgentConfig.AirGapped,
+			AirGappedInstallScriptPath: scope.Config.Spec.AgentConfig.AirGappedInstallScriptPath,
 		},
 	}
 
@@ -320,12 +323,13 @@ func (r *KThreesConfigReconciler) joinWorker(ctx context.Context, scope *Scope)
 
 	winput := &cloudinit.WorkerInput{
 		BaseUserData: cloudinit.BaseUserData{
-			PreK3sCommands:  scope.Config.Spec.PreK3sCommands,
-			PostK3sCommands: scope.Config.Spec.PostK3sCommands,
-			AdditionalFiles: files,
-			ConfigFile:      workerConfigFile,
-			K3sVersion:      scope.Config.Spec.Version,
-			AirGapped:       scope.Config.Spec.AgentConfig.AirGapped,
+			PreK3sCommands:             scope.Config.Spec.PreK3sCommands,
+			PostK3sCommands:            scope.Config.Spec.PostK3sCommands,
+			AdditionalFiles:            files,
+			ConfigFile:                 workerConfigFile,
+			K3sVersion:                 scope.Config.Spec.Version,
+			AirGapped:                  scope.Config.Spec.AgentConfig.AirGapped,
+			AirGappedInstallScriptPath: scope.Config.Spec.AgentConfig.AirGappedInstallScriptPath,
 		},
 	}
 
@@ -380,6 +384,38 @@ func (r *KThreesConfigReconciler) resolveSecretFileContent(ctx context.Context,
 	return data, nil
 }
 
+func (r *KThreesConfigReconciler) resolveEtcdProxyFile(cfg *bootstrapv1.KThreesConfig) (*bootstrapv1.File, error) {
+	// Parse the template
+	tpl, err := template.New("etcd-proxy").Parse(etcd.EtcdProxyDaemonsetYamlTemplate)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse etcd-proxy template: %w", err)
+	}
+
+	// If user has set the systemDefaultRegistry, will prefix the image with it.
+	systemDefaultRegistry := cfg.Spec.ServerConfig.SystemDefaultRegistry
+	if systemDefaultRegistry != "" {
+		systemDefaultRegistry = fmt.Sprintf("%s/", systemDefaultRegistry)
+	}
+
+	// Render the template, the image name will be ${EtcdProxyImage} if the user
+	// has set it, otherwise it will be ${SystemDefaultRegistry}alpine/socat
+	var buf bytes.Buffer
+	err = tpl.Execute(&buf, map[string]string{
+		"EtcdProxyImage":        cfg.Spec.ServerConfig.EtcdProxyImage,
+		"SystemDefaultRegistry": systemDefaultRegistry,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to render etcd-proxy template: %w", err)
+	}
+
+	return &bootstrapv1.File{
+		Path:        etcd.EtcdProxyDaemonsetYamlLocation,
+		Content:     buf.String(),
+		Owner:       "root:root",
+		Permissions: "0640",
+	}, nil
+}
+
 func (r *KThreesConfigReconciler) handleClusterNotInitialized(ctx context.Context, scope *Scope) (_ ctrl.Result, reterr error) {
 	// initialize the DataSecretAvailableCondition if missing.
 	// this is required in order to avoid the condition's LastTransitionTime to flicker in case of errors surfacing
@@ -465,23 +501,23 @@ func (r *KThreesConfigReconciler) handleClusterNotInitialized(ctx context.Contex
 	}
 
 	if scope.Config.Spec.IsEtcdEmbedded() {
-		etcdProxyFile := bootstrapv1.File{
-			Path:        etcd.EtcdProxyDaemonsetYamlLocation,
-			Content:     etcd.EtcdProxyDaemonsetYaml,
-			Owner:       "root:root",
-			Permissions: "0640",
+		etcdProxyFile, err := r.resolveEtcdProxyFile(scope.Config)
+		if err != nil {
+			conditions.MarkFalse(scope.Config, bootstrapv1.DataSecretAvailableCondition, bootstrapv1.DataSecretGenerationFailedReason, clusterv1.ConditionSeverityWarning, err.Error())
+			return ctrl.Result{}, fmt.Errorf("failed to resolve etcd proxy file: %w", err)
 		}
-		files = append(files, etcdProxyFile)
+		files = append(files, *etcdProxyFile)
 	}
 
 	cpinput := &cloudinit.ControlPlaneInput{
 		BaseUserData: cloudinit.BaseUserData{
-			PreK3sCommands:  scope.Config.Spec.PreK3sCommands,
-			PostK3sCommands: scope.Config.Spec.PostK3sCommands,
-			AdditionalFiles: files,
-			ConfigFile:      initConfigFile,
-			K3sVersion:      scope.Config.Spec.Version,
-			AirGapped:       scope.Config.Spec.AgentConfig.AirGapped,
+			PreK3sCommands:             scope.Config.Spec.PreK3sCommands,
+			PostK3sCommands:            scope.Config.Spec.PostK3sCommands,
+			AdditionalFiles:            files,
+			ConfigFile:                 initConfigFile,
+			K3sVersion:                 scope.Config.Spec.Version,
+			AirGapped:                  scope.Config.Spec.AgentConfig.AirGapped,
+			AirGappedInstallScriptPath: scope.Config.Spec.AgentConfig.AirGappedInstallScriptPath,
 		},
 		Certificates: certificates,
 	}

bootstrap/controllers/kthreesconfig_controller_test.go

@@ -0,0 +1,55 @@
+/*
+
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controllers
+
+import (
+	"testing"
+
+	. "github.com/onsi/gomega"
+
+	bootstrapv1 "github.com/k3s-io/cluster-api-k3s/bootstrap/api/v1beta2"
+)
+
+func TestKThreesConfigReconciler_ResolveEtcdProxyFile(t *testing.T) {
+	g := NewWithT(t)
+	// If EtcdProxyImage is set, it should override the system default registry
+	config := &bootstrapv1.KThreesConfig{
+		Spec: bootstrapv1.KThreesConfigSpec{
+			ServerConfig: bootstrapv1.KThreesServerConfig{
+				EtcdProxyImage:        "etcd-proxy-image",
+				SystemDefaultRegistry: "system-default-registry",
+			},
+		},
+	}
+	r := &KThreesConfigReconciler{}
+	etcdProxyFile, err := r.resolveEtcdProxyFile(config)
+	g.Expect(err).ToNot(HaveOccurred())
+	g.Expect(etcdProxyFile.Content).To(ContainSubstring("etcd-proxy-image"), "generated etcd proxy image should contain EtcdProxyImage")
+	g.Expect(etcdProxyFile.Content).ToNot(ContainSubstring("system-default-registry"), "system-default-registry should be overwritten by EtcdProxyImage")
+
+	// If EtcdProxyImage is not set, the system default registry should be used
+	config2 := &bootstrapv1.KThreesConfig{
+		Spec: bootstrapv1.KThreesConfigSpec{
+			ServerConfig: bootstrapv1.KThreesServerConfig{
+				SystemDefaultRegistry: "system-default-registry2",
+			},
+		},
+	}
+	etcdProxyFile, err = r.resolveEtcdProxyFile(config2)
+	g.Expect(err).ToNot(HaveOccurred())
+	g.Expect(etcdProxyFile.Content).To(ContainSubstring("system-default-registry2/"), "generated etcd proxy image should be prefixed with SystemDefaultRegistry")
+}

controlplane/api/v1beta1/conversion.go

@@ -94,9 +94,11 @@ func (in *KThreesControlPlane) ConvertTo(dstRaw ctrlconversion.Hub) error {
 	dst.Spec.KThreesConfigSpec.ServerConfig.DeprecatedDisableExternalCloudProvider = restored.Spec.KThreesConfigSpec.ServerConfig.DeprecatedDisableExternalCloudProvider
 	dst.Spec.KThreesConfigSpec.ServerConfig.DisableCloudController = restored.Spec.KThreesConfigSpec.ServerConfig.DisableCloudController
 	dst.Spec.KThreesConfigSpec.ServerConfig.SystemDefaultRegistry = restored.Spec.KThreesConfigSpec.ServerConfig.SystemDefaultRegistry
+	dst.Spec.KThreesConfigSpec.ServerConfig.EtcdProxyImage = restored.Spec.KThreesConfigSpec.ServerConfig.EtcdProxyImage
 	dst.Spec.MachineTemplate.NodeVolumeDetachTimeout = restored.Spec.MachineTemplate.NodeVolumeDetachTimeout
 	dst.Spec.MachineTemplate.NodeDeletionTimeout = restored.Spec.MachineTemplate.NodeDeletionTimeout
 	dst.Status.Version = restored.Status.Version
+	dst.Spec.KThreesConfigSpec.AgentConfig.AirGappedInstallScriptPath = restored.Spec.KThreesConfigSpec.AgentConfig.AirGappedInstallScriptPath
 	return nil
 }