feat(kaniko): Optimize kaniko build by 50% using compression and add …

…progress (#9476) * feat(kaniko): Optimize kaniko build using compression and add progress Signed-off-by: Suleiman Dibirov <idsulik@gmail.com> * add new config for kaniko BuildContextCompressionLevel Signed-off-by: Suleiman Dibirov <idsulik@gmail.com> * linters fix Signed-off-by: Suleiman Dibirov <idsulik@gmail.com> * fixed defaults_test.go Signed-off-by: Suleiman Dibirov <idsulik@gmail.com> --------- Signed-off-by: Suleiman Dibirov <idsulik@gmail.com>
GoogleContainerTools · Aug 30, 2024 · 1bc72a7 · 1bc72a7
1 parent ca9d2f1
commit 1bc72a7
Show file tree

Hide file tree

Showing 7 changed files with 68 additions and 19 deletions.
diff --git a/docs-v2/content/en/schemas/v4beta12.json b/docs-v2/content/en/schemas/v4beta12.json
@@ -2644,6 +2644,12 @@
             "{\"key1\": \"value1\", \"key2\": \"value2\", \"key3\": \"'{{.ENV_VARIABLE}}'\"}"
           ]
         },
+        "buildContextCompressionLevel": {
+          "type": "integer",
+          "description": "gzip compression level for the build context.",
+          "x-intellij-html-description": "gzip compression level for the build context.",
+          "default": "1"
+        },
         "cache": {
           "$ref": "#/definitions/KanikoCache",
           "description": "configures Kaniko caching. If a cache is specified, Kaniko will use a remote cache which will speed up builds.",
@@ -2919,7 +2925,8 @@
         "contextSubPath",
         "ignorePaths",
         "copyMaxRetries",
-        "copyTimeout"
+        "copyTimeout",
+        "buildContextCompressionLevel"
       ],
       "additionalProperties": false,
       "type": "object",

diff --git a/pkg/skaffold/build/cluster/kaniko.go b/pkg/skaffold/build/cluster/kaniko.go
@@ -18,12 +18,15 @@ package cluster
 
 import (
 	"bytes"
+	"compress/gzip"
 	"context"
 	"errors"
 	"fmt"
 	"io"
 	"time"
 
+	"github.com/docker/docker/pkg/progress"
+	"github.com/docker/docker/pkg/streamformatter"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
@@ -33,6 +36,7 @@ import (
 	"github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/docker"
 	"github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/kubernetes"
 	kubernetesclient "github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/kubernetes/client"
+	"github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/output"
 	"github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/output/log"
 	"github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/platform"
 	"github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/schema/latest"
@@ -44,7 +48,8 @@ const (
 )
 
 func (b *Builder) buildWithKaniko(ctx context.Context, out io.Writer, workspace string, artifactName string, artifact *latest.KanikoArtifact, tag string, requiredImages map[string]*string, platforms platform.Matcher) (string, error) {
-	log.Entry(ctx).Info("Start building with kaniko for artifact")
+	output.Default.Fprintf(out, "Start building with kaniko for artifact\n")
+
 	start := time.Now()
 	defer func() {
 		log.Entry(ctx).Infof("Building with kaniko completed in %s", time.Since(start))
@@ -101,7 +106,7 @@ func (b *Builder) buildWithKaniko(ctx context.Context, out io.Writer, workspace
 		}
 	}()
 
-	if err := b.setupKanikoBuildContext(ctx, workspace, artifactName, artifact, pods, pod.Name); err != nil {
+	if err := b.setupKanikoBuildContext(ctx, out, workspace, artifactName, artifact, pods, pod.Name); err != nil {
 		return "", fmt.Errorf("copying sources: %w", err)
 	}
 
@@ -122,7 +127,7 @@ func (b *Builder) buildWithKaniko(ctx context.Context, out io.Writer, workspace
 	return docker.RemoteDigest(tag, b.cfg, nil)
 }
 
-func (b *Builder) copyKanikoBuildContext(ctx context.Context, workspace string, artifactName string, artifact *latest.KanikoArtifact, podName string) error {
+func (b *Builder) copyKanikoBuildContext(ctx context.Context, out io.Writer, workspace string, artifactName string, artifact *latest.KanikoArtifact, podName string) error {
 	copyTimeout, err := time.ParseDuration(artifact.CopyTimeout)
 
 	if err != nil {
@@ -132,23 +137,45 @@ func (b *Builder) copyKanikoBuildContext(ctx context.Context, workspace string,
 	ctx, cancel := context.WithTimeout(ctx, copyTimeout)
 	defer cancel()
 	errs := make(chan error, 1)
-	buildCtx, buildCtxWriter := io.Pipe()
+	buildCtxReader, buildCtxWriter := io.Pipe()
+	gzipWriter, err := gzip.NewWriterLevel(buildCtxWriter, *artifact.BuildContextCompressionLevel)
+
+	if err != nil {
+		return fmt.Errorf("creating gzip writer: %w", err)
+	}
+
 	go func() {
-		err := docker.CreateDockerTarContext(ctx, buildCtxWriter, docker.NewBuildConfig(
+		defer func() {
+			closeErr := gzipWriter.Close()
+			if closeErr != nil {
+				log.Entry(ctx).Debugf("closing gzip writer: %v", closeErr)
+			}
+			closeErr = buildCtxWriter.Close() // it's safe to close the writer multiple times
+			if closeErr != nil {
+				log.Entry(ctx).Debugf("closing build context writer: %v", closeErr)
+			}
+		}()
+
+		err := docker.CreateDockerTarContext(ctx, gzipWriter, docker.NewBuildConfig(
 			kaniko.GetContext(artifact, workspace), artifactName, artifact.DockerfilePath, artifact.BuildArgs), b.cfg)
 		if err != nil {
-			buildCtxWriter.CloseWithError(fmt.Errorf("creating docker context: %w", err))
+			closeErr := buildCtxWriter.CloseWithError(fmt.Errorf("creating docker context: %w", err))
+			if closeErr != nil {
+				log.Entry(ctx).Debugf("closing build context writer: %v", closeErr)
+			}
 			errs <- err
 			return
 		}
-		buildCtxWriter.Close()
 	}()
 
+	progressOutput := streamformatter.NewProgressOutput(out)
+	progressReader := progress.NewProgressReader(buildCtxReader, progressOutput, 0, "", "Sending build context to Kaniko pod")
 	// Send context by piping into `tar`.
 	// In case of an error, retry and print the command's output. (The `err` itself is useless: exit status 1).
-	var out bytes.Buffer
-	if err := b.kubectlcli.Run(ctx, buildCtx, &out, "exec", "-i", podName, "-c", initContainer, "-n", b.Namespace, "--", "tar", "-xf", "-", "-C", kaniko.DefaultEmptyDirMountPath); err != nil {
-		errRun := fmt.Errorf("uploading build context: %s", out.String())
+	var cmdOut bytes.Buffer
+	if err := b.kubectlcli.Run(ctx,
+		progressReader, &cmdOut, "exec", "-i", podName, "-c", initContainer, "-n", b.Namespace, "--", "tar", "-zxf", "-", "-C", kaniko.DefaultEmptyDirMountPath); err != nil {
+		errRun := fmt.Errorf("uploading build context: %s", cmdOut.String())
 		select {
 		case errTar := <-errs:
 			if errTar != nil {
@@ -165,7 +192,7 @@ func (b *Builder) copyKanikoBuildContext(ctx context.Context, workspace string,
 // first copy over the buildcontext tarball into the init container tmp dir via kubectl cp
 // Via kubectl exec, we extract the tarball to the empty dir
 // Then, via kubectl exec, create the /tmp/complete file via kubectl exec to complete the init container
-func (b *Builder) setupKanikoBuildContext(ctx context.Context, workspace string, artifactName string, artifact *latest.KanikoArtifact, pods corev1.PodInterface, podName string) error {
+func (b *Builder) setupKanikoBuildContext(ctx context.Context, out io.Writer, workspace string, artifactName string, artifact *latest.KanikoArtifact, pods corev1.PodInterface, podName string) error {
 	if err := kubernetes.WaitForPodInitialized(ctx, pods, podName); err != nil {
 		return fmt.Errorf("waiting for pod to initialize: %w", err)
 	}
@@ -178,7 +205,7 @@ func (b *Builder) setupKanikoBuildContext(ctx context.Context, workspace string,
 	}
 
 	err = wait.Poll(time.Second, timeout*time.Duration(*artifact.CopyMaxRetries+1), func() (bool, error) {
-		if err := b.copyKanikoBuildContext(ctx, workspace, artifactName, artifact, podName); err != nil {
+		if err := b.copyKanikoBuildContext(ctx, out, workspace, artifactName, artifact, podName); err != nil {
 			if errors.Is(ctx.Err(), context.Canceled) {
 				return false, err
 			}

diff --git a/pkg/skaffold/build/kaniko/types.go b/pkg/skaffold/build/kaniko/types.go
@@ -113,4 +113,6 @@ const (
 	DefaultCopyMaxRetries = 3
 	// DefaultCopyTimeout for kaniko pod
 	DefaultCopyTimeout = "5m"
+	// DefaultBuildContextCompressionLevel for kaniko pod
+	DefaultBuildContextCompressionLevel = 1 // BestSpeed
 )
diff --git a/pkg/skaffold/schema/defaults/defaults.go b/pkg/skaffold/schema/defaults/defaults.go
@@ -364,6 +364,7 @@ func setKanikoArtifactDefaults(a *latest.KanikoArtifact) {
 	a.DigestFile = valueOrDefault(a.DigestFile, constants.DefaultKanikoDigestFile)
 	a.CopyMaxRetries = valueOrDefaultInt(a.CopyMaxRetries, kaniko.DefaultCopyMaxRetries)
 	a.CopyTimeout = valueOrDefault(a.CopyTimeout, kaniko.DefaultCopyTimeout)
+	a.BuildContextCompressionLevel = valueOrDefaultInt(a.BuildContextCompressionLevel, kaniko.DefaultBuildContextCompressionLevel)
 }
 
 func valueOrDefault(v, def string) string {

diff --git a/pkg/skaffold/schema/defaults/defaults_test.go b/pkg/skaffold/schema/defaults/defaults_test.go
@@ -136,6 +136,7 @@ func TestSetDefaults(t *testing.T) {
 	testutil.CheckDeepEqual(t, "eights", cfg.Build.Artifacts[7].ImageName)
 	testutil.CheckDeepEqual(t, 3, *cfg.Build.Artifacts[7].KanikoArtifact.CopyMaxRetries)
 	testutil.CheckDeepEqual(t, "5m", cfg.Build.Artifacts[7].KanikoArtifact.CopyTimeout)
+	testutil.CheckDeepEqual(t, 1, *cfg.Build.Artifacts[7].KanikoArtifact.BuildContextCompressionLevel)
 }
 
 func TestSetDefaultsOnCluster(t *testing.T) {

diff --git a/pkg/skaffold/schema/latest/config.go b/pkg/skaffold/schema/latest/config.go
@@ -1536,6 +1536,15 @@ type KanikoArtifact struct {
 	// CopyTimeout is the timeout for copying build contexts to a cluster.
 	// Defaults to 5 minutes (`5m`).
 	CopyTimeout string `yaml:"copyTimeout,omitempty"`
+
+	// BuildContextCompressionLevel is the gzip compression level for the build context.
+	// Defaults to `1`.
+	// 0: NoCompression
+	// 1: BestSpeed
+	// 9: BestCompression
+	// -1: DefaultCompression
+	// -2: HuffmanOnly
+	BuildContextCompressionLevel *int `yaml:"buildContextCompressionLevel,omitempty"`
 }
 
 // DockerArtifact describes an artifact built from a Dockerfile,

diff --git a/pkg/skaffold/schema/versions_test.go b/pkg/skaffold/schema/versions_test.go
@@ -621,17 +621,19 @@ func withBazelArtifact() func(*latest.BuildConfig) {
 func withKanikoArtifact() func(*latest.BuildConfig) {
 	return func(cfg *latest.BuildConfig) {
 		copyMaxRetries := 3
+		compressionLevel := 1
 		cfg.Artifacts = append(cfg.Artifacts, &latest.Artifact{
 			ImageName: "image1",
 			Workspace: "./examples/app1",
 			ArtifactType: latest.ArtifactType{
 				KanikoArtifact: &latest.KanikoArtifact{
-					DockerfilePath: "Dockerfile",
-					InitImage:      constants.DefaultBusyboxImage,
-					Image:          kaniko.DefaultImage,
-					DigestFile:     "/dev/termination-log",
-					CopyMaxRetries: &copyMaxRetries,
-					CopyTimeout:    "5m",
+					DockerfilePath:               "Dockerfile",
+					InitImage:                    constants.DefaultBusyboxImage,
+					Image:                        kaniko.DefaultImage,
+					DigestFile:                   "/dev/termination-log",
+					CopyMaxRetries:               &copyMaxRetries,
+					CopyTimeout:                  "5m",
+					BuildContextCompressionLevel: &compressionLevel,
 				},
 			},
 		})