Skip to content

Commit

Permalink
Add push artifact work around.
Browse files Browse the repository at this point in the history 
The workaround identifies artifacts in workspace and pushes them to s3
storage.

Signed-off-by: Humair Khan <humair88@hotmail.com>
  • Loading branch information
@HumairAK
HumairAK committed Jul 24, 2023
1 parent ddb30e9 commit 1838ca8
Show file tree
Hide file tree
Showing 5 changed files with 50 additions and 15 deletions.
13 changes: 10 additions & 3 deletions config/internal/apiserver/artifact_script.yaml.tmpl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,16 @@ data:
artifact_script: |-
#!/usr/bin/env sh
push_artifact() {
if [ -f "$2" ]; then
tar -cvzf $1.tgz $2
aws s3 --endpoint {{.ObjectStorageConnection.Endpoint}} cp $1.tgz s3://{{.ObjectStorageConnection.Bucket}}/artifacts/$PIPELINERUN/$PIPELINETASK/$1.tgz
workspace_dir=$(echo $(context.taskRun.name) | sed -e "s/$(context.pipeline.name)-//g")
workspace_dest=/workspace/${workspace_dir}/artifacts/$(context.pipelineRun.name)/$(context.taskRun.name)
artifact_name=$(basename $2)
if [ -f "$workspace_dest/$artifact_name" ]; then
echo sending to: ${workspace_dest}/${artifact_name}
tar -cvzf $1.tgz -C ${workspace_dest} ${artifact_name}
aws s3 --endpoint ${ARTIFACT_ENDPOINT} cp $1.tgz s3://${ARTIFACT_BUCKET}/artifacts/$PIPELINERUN/$PIPELINETASK/$1.tgz
elif [ -f "$2" ]; then
tar -cvzf $1.tgz -C $(dirname $2) ${artifact_name}
aws s3 --endpoint ${ARTIFACT_ENDPOINT} cp $1.tgz s3://${ARTIFACT_BUCKET}/artifacts/$PIPELINERUN/$PIPELINETASK/$1.tgz
else
echo "$2 file does not exist. Skip artifact tracking for $1"
fi
Expand Down
13 changes: 10 additions & 3 deletions controllers/testdata/declarative/case_0/expected/created/configmap_artifact_script.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,16 @@ data:
artifact_script: |-
#!/usr/bin/env sh
push_artifact() {
if [ -f "$2" ]; then
tar -cvzf $1.tgz $2
aws s3 --endpoint http://minio-testdsp0.default.svc.cluster.local:9000 cp $1.tgz s3://mlpipeline/artifacts/$PIPELINERUN/$PIPELINETASK/$1.tgz
workspace_dir=$(echo $(context.taskRun.name) | sed -e "s/$(context.pipeline.name)-//g")
workspace_dest=/workspace/${workspace_dir}/artifacts/$(context.pipelineRun.name)/$(context.taskRun.name)
artifact_name=$(basename $2)
if [ -f "$workspace_dest/$artifact_name" ]; then
echo sending to: ${workspace_dest}/${artifact_name}
tar -cvzf $1.tgz -C ${workspace_dest} ${artifact_name}
aws s3 --endpoint ${ARTIFACT_ENDPOINT} cp $1.tgz s3://${ARTIFACT_BUCKET}/artifacts/$PIPELINERUN/$PIPELINETASK/$1.tgz
elif [ -f "$2" ]; then
tar -cvzf $1.tgz -C $(dirname $2) ${artifact_name}
aws s3 --endpoint ${ARTIFACT_ENDPOINT} cp $1.tgz s3://${ARTIFACT_BUCKET}/artifacts/$PIPELINERUN/$PIPELINETASK/$1.tgz
else
echo "$2 file does not exist. Skip artifact tracking for $1"
fi
Expand Down
13 changes: 10 additions & 3 deletions controllers/testdata/declarative/case_2/expected/created/configmap_artifact_script.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,16 @@ data:
artifact_script: |-
#!/usr/bin/env sh
push_artifact() {
if [ -f "$2" ]; then
tar -cvzf $1.tgz $2
aws s3 --endpoint http://minio-testdsp2.default.svc.cluster.local:9000 cp $1.tgz s3://mlpipeline/artifacts/$PIPELINERUN/$PIPELINETASK/$1.tgz
workspace_dir=$(echo $(context.taskRun.name) | sed -e "s/$(context.pipeline.name)-//g")
workspace_dest=/workspace/${workspace_dir}/artifacts/$(context.pipelineRun.name)/$(context.taskRun.name)
artifact_name=$(basename $2)
if [ -f "$workspace_dest/$artifact_name" ]; then
echo sending to: ${workspace_dest}/${artifact_name}
tar -cvzf $1.tgz -C ${workspace_dest} ${artifact_name}
aws s3 --endpoint ${ARTIFACT_ENDPOINT} cp $1.tgz s3://${ARTIFACT_BUCKET}/artifacts/$PIPELINERUN/$PIPELINETASK/$1.tgz
elif [ -f "$2" ]; then
tar -cvzf $1.tgz -C $(dirname $2) ${artifact_name}
aws s3 --endpoint ${ARTIFACT_ENDPOINT} cp $1.tgz s3://${ARTIFACT_BUCKET}/artifacts/$PIPELINERUN/$PIPELINETASK/$1.tgz
else
echo "$2 file does not exist. Skip artifact tracking for $1"
fi
Expand Down
13 changes: 10 additions & 3 deletions controllers/testdata/declarative/case_4/expected/created/configmap_artifact_script.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,16 @@ data:
artifact_script: |-
#!/usr/bin/env sh
push_artifact() {
if [ -f "$2" ]; then
tar -cvzf $1.tgz $2
aws s3 --endpoint http://minio-testdsp4.default.svc.cluster.local:9000 cp $1.tgz s3://mlpipeline/artifacts/$PIPELINERUN/$PIPELINETASK/$1.tgz
workspace_dir=$(echo $(context.taskRun.name) | sed -e "s/$(context.pipeline.name)-//g")
workspace_dest=/workspace/${workspace_dir}/artifacts/$(context.pipelineRun.name)/$(context.taskRun.name)
artifact_name=$(basename $2)
if [ -f "$workspace_dest/$artifact_name" ]; then
echo sending to: ${workspace_dest}/${artifact_name}
tar -cvzf $1.tgz -C ${workspace_dest} ${artifact_name}
aws s3 --endpoint ${ARTIFACT_ENDPOINT} cp $1.tgz s3://${ARTIFACT_BUCKET}/artifacts/$PIPELINERUN/$PIPELINETASK/$1.tgz
elif [ -f "$2" ]; then
tar -cvzf $1.tgz -C $(dirname $2) ${artifact_name}
aws s3 --endpoint ${ARTIFACT_ENDPOINT} cp $1.tgz s3://${ARTIFACT_BUCKET}/artifacts/$PIPELINERUN/$PIPELINETASK/$1.tgz
else
echo "$2 file does not exist. Skip artifact tracking for $1"
fi
Expand Down
13 changes: 10 additions & 3 deletions controllers/testdata/declarative/case_5/expected/created/configmap_artifact_script.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,16 @@ data:
artifact_script: |-
#!/usr/bin/env sh
push_artifact() {
if [ -f "$2" ]; then
tar -cvzf $1.tgz $2
aws s3 --endpoint http://minio-testdsp5.default.svc.cluster.local:9000 cp $1.tgz s3://mlpipeline/artifacts/$PIPELINERUN/$PIPELINETASK/$1.tgz
workspace_dir=$(echo $(context.taskRun.name) | sed -e "s/$(context.pipeline.name)-//g")
workspace_dest=/workspace/${workspace_dir}/artifacts/$(context.pipelineRun.name)/$(context.taskRun.name)
artifact_name=$(basename $2)
if [ -f "$workspace_dest/$artifact_name" ]; then
echo sending to: ${workspace_dest}/${artifact_name}
tar -cvzf $1.tgz -C ${workspace_dest} ${artifact_name}
aws s3 --endpoint ${ARTIFACT_ENDPOINT} cp $1.tgz s3://${ARTIFACT_BUCKET}/artifacts/$PIPELINERUN/$PIPELINETASK/$1.tgz
elif [ -f "$2" ]; then
tar -cvzf $1.tgz -C $(dirname $2) ${artifact_name}
aws s3 --endpoint ${ARTIFACT_ENDPOINT} cp $1.tgz s3://${ARTIFACT_BUCKET}/artifacts/$PIPELINERUN/$PIPELINETASK/$1.tgz
else
echo "$2 file does not exist. Skip artifact tracking for $1"
fi
Expand Down

0 comments on commit 1838ca8

Please sign in to comment.