-
Notifications
You must be signed in to change notification settings - Fork 6
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Create some StepAction to fetch and upload caches…
And experiment with a small go project, GOCACHE and GOMODCACHE. Signed-off-by: Vincent Demeester <vdemeest@redhat.com>
- Loading branch information
1 parent
089624e
commit 4bf5b94
Showing
17 changed files
with
1,210 additions
and
51 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| defaultBaseImage: registry.access.redhat.com/ubi8/ubi-minimal |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,53 +1,86 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "io/fs" | ||
| "os" | ||
| "path/filepath" | ||
|
|
||
| "github.com/moby/patternmatcher" | ||
| "github.com/openshift-pipelines/tekton-caches/internal/fetch" | ||
| "github.com/openshift-pipelines/tekton-caches/internal/hash" | ||
| "github.com/spf13/cobra" | ||
| ) | ||
|
|
||
| const ( | ||
| filesFlag = "hashfiles" | ||
| targetFlag = "target" | ||
| folderFlag = "folder" | ||
| workingdirFlag = "workingdir" | ||
| filesFlag = "hashfiles" | ||
| patternsFlag = "pattern" | ||
| sourceFlag = "source" | ||
| folderFlag = "folder" | ||
| ) | ||
|
|
||
| func fetchCmd() *cobra.Command { | ||
| cmd := &cobra.Command{ | ||
| Use: "fetch", | ||
| RunE: func(cmd *cobra.Command, args []string) error { | ||
| files, err := cmd.Flags().GetString(filesFlag) | ||
| target, err := cmd.Flags().GetString(sourceFlag) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| target, err := cmd.Flags().GetString(targetFlag) | ||
| folder, err := cmd.Flags().GetString(folderFlag) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| folder, err := cmd.Flags().GetString(folderFlag) | ||
| workingdir, err := cmd.Flags().GetString(workingdirFlag) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| // FIXME error out if empty | ||
|
|
||
| matches, err := filepath.Glob(files) | ||
| patterns, err := cmd.Flags().GetStringArray(patternsFlag) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| matches := glob(workingdir, func(s string) bool { | ||
| m, err := patternmatcher.Matches(s, patterns) | ||
| if err != nil { | ||
| fmt.Fprintf(os.Stderr, "error trying to match files with '%v': %s", patterns, err) | ||
| return false | ||
| } | ||
| return m | ||
| }) | ||
| if len(matches) == 0 { | ||
| return fmt.Errorf("Didn't match any files with %v", patterns) | ||
| } else { | ||
| fmt.Fprintf(os.Stderr, "Matched the following files: %v\n", matches) | ||
| } | ||
| // TODO: Hash files based of matches | ||
| hashStr, err := hash.Compute(matches) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| return fetch.Try(cmd.Context(), hashStr, target, folder) | ||
|
|
||
| // FIXME: Wrap the error. | ||
| // If not, warn and do not fail | ||
| // fmt.Fprintf(os.Stderr, "Repository %s doesn't exists or isn't reachable, fetching no cache.\n", cacheImageRef) | ||
| return fetch.Fetch(cmd.Context(), hashStr, target, folder) | ||
| }, | ||
| } | ||
|
|
||
| cmd.Flags().String(filesFlag, "", "Files pattern to compute the hash from") | ||
| cmd.Flags().String(targetFlag, "", "Cache oci image target reference") | ||
| cmd.Flags().StringArray(patternsFlag, []string{}, "Files pattern to compute the hash from") | ||
| cmd.Flags().String(sourceFlag, "", "Cache source reference") | ||
| cmd.Flags().String(folderFlag, "", "Folder where to extract the content of the cache if it exists") | ||
| cmd.Flags().String(workingdirFlag, ".", "Working dir from where the files patterns needs to be taken") | ||
|
|
||
| return cmd | ||
| } | ||
|
|
||
| func glob(root string, fn func(string) bool) []string { | ||
| var files []string | ||
| filepath.WalkDir(root, func(s string, d fs.DirEntry, e error) error { | ||
| if fn(s) { | ||
| files = append(files, s) | ||
| } | ||
| return nil | ||
| }) | ||
| return files | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,65 @@ | ||
| apiVersion: tekton.dev/v1alpha1 | ||
| kind: StepAction | ||
| metadata: | ||
| name: cache-fetch | ||
| annotations: | ||
| tekton.dev/pipelines.minVersion: "0.56.0" | ||
| tekton.dev/tags: "cache" | ||
| spec: | ||
| params: | ||
| - name: patterns | ||
| description: | | ||
| Regular expression to select files to include to compute the hash. | ||
| For example, in the case of a Go project, you can use `go.mod` for this, so the value would be "**/go.sum" (to work with possible sub go modules as well). | ||
| type: array | ||
| - name: source | ||
| description: | | ||
| The source from where the cache should be fetched. It's a URI with the scheme defining the "provider". In addition, one can add a {{hash}} variable to use the computed hash in the reference (oci image tags, path in s3, …) | ||
| Currently supported: | ||
| - oci:// (e.g. oci://quay.io/vdemeester/go-cache:{{hash}} | ||
| - s3:// (e.g. s3:// | ||
| type: string | ||
| - name: cachePath | ||
| description: | | ||
| Path where to extract the cache content. | ||
| It can refer any folder, backed by a workspace or a volume, or nothing. | ||
| type: string | ||
| - name: workingdir | ||
| description: | | ||
| The working dir from where the files patterns needs to be taken | ||
| type: string | ||
| results: # Any result to "publish" ? | ||
| - name: fetched | ||
| description: | | ||
| Whether a cache was fetched or not (true/false). This step won't fail if it didn't manage to fetch cache. This results allows the next step to act whether something was fetched or not. | ||
| env: | ||
| - name: PARAM_SOURCE | ||
| value: $(params.source) | ||
| - name: PARAM_CACHE_PATH | ||
| value: $(params.cachePath) | ||
| - name: PARAM_WORKINGDIR | ||
| value: $(params.workingdir) | ||
| # FIXME: use a released version once something is released :) | ||
| image: ghcr.io/openshift-pipelines/tekton-caches/cache:latest | ||
| args: ["$(params.patterns[*])"] | ||
| script: | | ||
| #!/bin/sh | ||
| PATTERN_FLAGS="" | ||
| echo "Patterns: $*" | ||
| for p in $*; do | ||
| PATTERN_FLAGS="${PATTERN_FLAGS} --pattern ${p}" | ||
| done | ||
| set -x | ||
| /ko-app/cache fetch ${PATTERN_FLAGS} \ | ||
| --source ${PARAM_SOURCE} \ | ||
| --folder ${PARAM_CACHE_PATH} \ | ||
| --workingdir ${PARAM_WORKINGDIR} | ||
| if [ $? -eq 0 ]; then | ||
| echo -n true > $(step.results.fetched.path) | ||
| else | ||
| echo -n false > $(step.results.fetched.path) | ||
| fi | ||
| exit 0 |
Oops, something went wrong.