From 1bc72a7c53d99f6c135e2278ebf8f04671a20452 Mon Sep 17 00:00:00 2001 From: Suleiman Dibirov <3595194+idsulik@users.noreply.github.com> Date: Fri, 30 Aug 2024 18:24:24 +0300 Subject: [PATCH] feat(kaniko): Optimize kaniko build by 50% using compression and add progress (#9476) * feat(kaniko): Optimize kaniko build using compression and add progress Signed-off-by: Suleiman Dibirov * add new config for kaniko BuildContextCompressionLevel Signed-off-by: Suleiman Dibirov * linters fix Signed-off-by: Suleiman Dibirov * fixed defaults_test.go Signed-off-by: Suleiman Dibirov --------- Signed-off-by: Suleiman Dibirov --- docs-v2/content/en/schemas/v4beta12.json | 9 +++- pkg/skaffold/build/cluster/kaniko.go | 51 ++++++++++++++----- pkg/skaffold/build/kaniko/types.go | 2 + pkg/skaffold/schema/defaults/defaults.go | 1 + pkg/skaffold/schema/defaults/defaults_test.go | 1 + pkg/skaffold/schema/latest/config.go | 9 ++++ pkg/skaffold/schema/versions_test.go | 14 ++--- 7 files changed, 68 insertions(+), 19 deletions(-) diff --git a/docs-v2/content/en/schemas/v4beta12.json b/docs-v2/content/en/schemas/v4beta12.json index c03e49403ab..644821c8641 100755 --- a/docs-v2/content/en/schemas/v4beta12.json +++ b/docs-v2/content/en/schemas/v4beta12.json @@ -2644,6 +2644,12 @@ "{\"key1\": \"value1\", \"key2\": \"value2\", \"key3\": \"'{{.ENV_VARIABLE}}'\"}" ] }, + "buildContextCompressionLevel": { + "type": "integer", + "description": "gzip compression level for the build context.", + "x-intellij-html-description": "gzip compression level for the build context.", + "default": "1" + }, "cache": { "$ref": "#/definitions/KanikoCache", "description": "configures Kaniko caching. If a cache is specified, Kaniko will use a remote cache which will speed up builds.", @@ -2919,7 +2925,8 @@ "contextSubPath", "ignorePaths", "copyMaxRetries", - "copyTimeout" + "copyTimeout", + "buildContextCompressionLevel" ], "additionalProperties": false, "type": "object", diff --git a/pkg/skaffold/build/cluster/kaniko.go b/pkg/skaffold/build/cluster/kaniko.go index ef527cd8769..80f3690907e 100644 --- a/pkg/skaffold/build/cluster/kaniko.go +++ b/pkg/skaffold/build/cluster/kaniko.go @@ -18,12 +18,15 @@ package cluster import ( "bytes" + "compress/gzip" "context" "errors" "fmt" "io" "time" + "github.com/docker/docker/pkg/progress" + "github.com/docker/docker/pkg/streamformatter" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" @@ -33,6 +36,7 @@ import ( "github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/docker" "github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/kubernetes" kubernetesclient "github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/kubernetes/client" + "github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/output" "github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/output/log" "github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/platform" "github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/schema/latest" @@ -44,7 +48,8 @@ const ( ) func (b *Builder) buildWithKaniko(ctx context.Context, out io.Writer, workspace string, artifactName string, artifact *latest.KanikoArtifact, tag string, requiredImages map[string]*string, platforms platform.Matcher) (string, error) { - log.Entry(ctx).Info("Start building with kaniko for artifact") + output.Default.Fprintf(out, "Start building with kaniko for artifact\n") + start := time.Now() defer func() { log.Entry(ctx).Infof("Building with kaniko completed in %s", time.Since(start)) @@ -101,7 +106,7 @@ func (b *Builder) buildWithKaniko(ctx context.Context, out io.Writer, workspace } }() - if err := b.setupKanikoBuildContext(ctx, workspace, artifactName, artifact, pods, pod.Name); err != nil { + if err := b.setupKanikoBuildContext(ctx, out, workspace, artifactName, artifact, pods, pod.Name); err != nil { return "", fmt.Errorf("copying sources: %w", err) } @@ -122,7 +127,7 @@ func (b *Builder) buildWithKaniko(ctx context.Context, out io.Writer, workspace return docker.RemoteDigest(tag, b.cfg, nil) } -func (b *Builder) copyKanikoBuildContext(ctx context.Context, workspace string, artifactName string, artifact *latest.KanikoArtifact, podName string) error { +func (b *Builder) copyKanikoBuildContext(ctx context.Context, out io.Writer, workspace string, artifactName string, artifact *latest.KanikoArtifact, podName string) error { copyTimeout, err := time.ParseDuration(artifact.CopyTimeout) if err != nil { @@ -132,23 +137,45 @@ func (b *Builder) copyKanikoBuildContext(ctx context.Context, workspace string, ctx, cancel := context.WithTimeout(ctx, copyTimeout) defer cancel() errs := make(chan error, 1) - buildCtx, buildCtxWriter := io.Pipe() + buildCtxReader, buildCtxWriter := io.Pipe() + gzipWriter, err := gzip.NewWriterLevel(buildCtxWriter, *artifact.BuildContextCompressionLevel) + + if err != nil { + return fmt.Errorf("creating gzip writer: %w", err) + } + go func() { - err := docker.CreateDockerTarContext(ctx, buildCtxWriter, docker.NewBuildConfig( + defer func() { + closeErr := gzipWriter.Close() + if closeErr != nil { + log.Entry(ctx).Debugf("closing gzip writer: %v", closeErr) + } + closeErr = buildCtxWriter.Close() // it's safe to close the writer multiple times + if closeErr != nil { + log.Entry(ctx).Debugf("closing build context writer: %v", closeErr) + } + }() + + err := docker.CreateDockerTarContext(ctx, gzipWriter, docker.NewBuildConfig( kaniko.GetContext(artifact, workspace), artifactName, artifact.DockerfilePath, artifact.BuildArgs), b.cfg) if err != nil { - buildCtxWriter.CloseWithError(fmt.Errorf("creating docker context: %w", err)) + closeErr := buildCtxWriter.CloseWithError(fmt.Errorf("creating docker context: %w", err)) + if closeErr != nil { + log.Entry(ctx).Debugf("closing build context writer: %v", closeErr) + } errs <- err return } - buildCtxWriter.Close() }() + progressOutput := streamformatter.NewProgressOutput(out) + progressReader := progress.NewProgressReader(buildCtxReader, progressOutput, 0, "", "Sending build context to Kaniko pod") // Send context by piping into `tar`. // In case of an error, retry and print the command's output. (The `err` itself is useless: exit status 1). - var out bytes.Buffer - if err := b.kubectlcli.Run(ctx, buildCtx, &out, "exec", "-i", podName, "-c", initContainer, "-n", b.Namespace, "--", "tar", "-xf", "-", "-C", kaniko.DefaultEmptyDirMountPath); err != nil { - errRun := fmt.Errorf("uploading build context: %s", out.String()) + var cmdOut bytes.Buffer + if err := b.kubectlcli.Run(ctx, + progressReader, &cmdOut, "exec", "-i", podName, "-c", initContainer, "-n", b.Namespace, "--", "tar", "-zxf", "-", "-C", kaniko.DefaultEmptyDirMountPath); err != nil { + errRun := fmt.Errorf("uploading build context: %s", cmdOut.String()) select { case errTar := <-errs: if errTar != nil { @@ -165,7 +192,7 @@ func (b *Builder) copyKanikoBuildContext(ctx context.Context, workspace string, // first copy over the buildcontext tarball into the init container tmp dir via kubectl cp // Via kubectl exec, we extract the tarball to the empty dir // Then, via kubectl exec, create the /tmp/complete file via kubectl exec to complete the init container -func (b *Builder) setupKanikoBuildContext(ctx context.Context, workspace string, artifactName string, artifact *latest.KanikoArtifact, pods corev1.PodInterface, podName string) error { +func (b *Builder) setupKanikoBuildContext(ctx context.Context, out io.Writer, workspace string, artifactName string, artifact *latest.KanikoArtifact, pods corev1.PodInterface, podName string) error { if err := kubernetes.WaitForPodInitialized(ctx, pods, podName); err != nil { return fmt.Errorf("waiting for pod to initialize: %w", err) } @@ -178,7 +205,7 @@ func (b *Builder) setupKanikoBuildContext(ctx context.Context, workspace string, } err = wait.Poll(time.Second, timeout*time.Duration(*artifact.CopyMaxRetries+1), func() (bool, error) { - if err := b.copyKanikoBuildContext(ctx, workspace, artifactName, artifact, podName); err != nil { + if err := b.copyKanikoBuildContext(ctx, out, workspace, artifactName, artifact, podName); err != nil { if errors.Is(ctx.Err(), context.Canceled) { return false, err } diff --git a/pkg/skaffold/build/kaniko/types.go b/pkg/skaffold/build/kaniko/types.go index d190ab4a40f..69dbf348445 100644 --- a/pkg/skaffold/build/kaniko/types.go +++ b/pkg/skaffold/build/kaniko/types.go @@ -113,4 +113,6 @@ const ( DefaultCopyMaxRetries = 3 // DefaultCopyTimeout for kaniko pod DefaultCopyTimeout = "5m" + // DefaultBuildContextCompressionLevel for kaniko pod + DefaultBuildContextCompressionLevel = 1 // BestSpeed ) diff --git a/pkg/skaffold/schema/defaults/defaults.go b/pkg/skaffold/schema/defaults/defaults.go index 7b61e5037df..580f3859380 100644 --- a/pkg/skaffold/schema/defaults/defaults.go +++ b/pkg/skaffold/schema/defaults/defaults.go @@ -364,6 +364,7 @@ func setKanikoArtifactDefaults(a *latest.KanikoArtifact) { a.DigestFile = valueOrDefault(a.DigestFile, constants.DefaultKanikoDigestFile) a.CopyMaxRetries = valueOrDefaultInt(a.CopyMaxRetries, kaniko.DefaultCopyMaxRetries) a.CopyTimeout = valueOrDefault(a.CopyTimeout, kaniko.DefaultCopyTimeout) + a.BuildContextCompressionLevel = valueOrDefaultInt(a.BuildContextCompressionLevel, kaniko.DefaultBuildContextCompressionLevel) } func valueOrDefault(v, def string) string { diff --git a/pkg/skaffold/schema/defaults/defaults_test.go b/pkg/skaffold/schema/defaults/defaults_test.go index 38985c2c56f..cdcd73a4441 100644 --- a/pkg/skaffold/schema/defaults/defaults_test.go +++ b/pkg/skaffold/schema/defaults/defaults_test.go @@ -136,6 +136,7 @@ func TestSetDefaults(t *testing.T) { testutil.CheckDeepEqual(t, "eights", cfg.Build.Artifacts[7].ImageName) testutil.CheckDeepEqual(t, 3, *cfg.Build.Artifacts[7].KanikoArtifact.CopyMaxRetries) testutil.CheckDeepEqual(t, "5m", cfg.Build.Artifacts[7].KanikoArtifact.CopyTimeout) + testutil.CheckDeepEqual(t, 1, *cfg.Build.Artifacts[7].KanikoArtifact.BuildContextCompressionLevel) } func TestSetDefaultsOnCluster(t *testing.T) { diff --git a/pkg/skaffold/schema/latest/config.go b/pkg/skaffold/schema/latest/config.go index cf999b2851d..fa654731042 100644 --- a/pkg/skaffold/schema/latest/config.go +++ b/pkg/skaffold/schema/latest/config.go @@ -1536,6 +1536,15 @@ type KanikoArtifact struct { // CopyTimeout is the timeout for copying build contexts to a cluster. // Defaults to 5 minutes (`5m`). CopyTimeout string `yaml:"copyTimeout,omitempty"` + + // BuildContextCompressionLevel is the gzip compression level for the build context. + // Defaults to `1`. + // 0: NoCompression + // 1: BestSpeed + // 9: BestCompression + // -1: DefaultCompression + // -2: HuffmanOnly + BuildContextCompressionLevel *int `yaml:"buildContextCompressionLevel,omitempty"` } // DockerArtifact describes an artifact built from a Dockerfile, diff --git a/pkg/skaffold/schema/versions_test.go b/pkg/skaffold/schema/versions_test.go index acb55dc9f32..c5839fa7809 100644 --- a/pkg/skaffold/schema/versions_test.go +++ b/pkg/skaffold/schema/versions_test.go @@ -621,17 +621,19 @@ func withBazelArtifact() func(*latest.BuildConfig) { func withKanikoArtifact() func(*latest.BuildConfig) { return func(cfg *latest.BuildConfig) { copyMaxRetries := 3 + compressionLevel := 1 cfg.Artifacts = append(cfg.Artifacts, &latest.Artifact{ ImageName: "image1", Workspace: "./examples/app1", ArtifactType: latest.ArtifactType{ KanikoArtifact: &latest.KanikoArtifact{ - DockerfilePath: "Dockerfile", - InitImage: constants.DefaultBusyboxImage, - Image: kaniko.DefaultImage, - DigestFile: "/dev/termination-log", - CopyMaxRetries: ©MaxRetries, - CopyTimeout: "5m", + DockerfilePath: "Dockerfile", + InitImage: constants.DefaultBusyboxImage, + Image: kaniko.DefaultImage, + DigestFile: "/dev/termination-log", + CopyMaxRetries: ©MaxRetries, + CopyTimeout: "5m", + BuildContextCompressionLevel: &compressionLevel, }, }, })