Replace sample iris-dataset pipeline with a v2 one

opendatahub-io · Oct 31, 2023 · 440031b · 440031b
1 parent 76e17de
commit 440031b
Show file tree

Hide file tree

Showing 15 changed files with 2,378 additions and 1,065 deletions.
diff --git a/config/internal/apiserver/sample-pipeline/sample-pipeline.yaml.tmpl b/config/internal/apiserver/sample-pipeline/sample-pipeline.yaml.tmpl
diff --git a/controllers/dspipeline_params.go b/controllers/dspipeline_params.go
@@ -39,6 +39,7 @@ type DSPAParams struct {
 	Name                 string
 	Namespace            string
 	Owner                mf.Owner
+	DSPVersion           string
 	APIServer            *dspa.APIServer
 	APIServerServiceName string
 	OAuthProxy           string
@@ -432,6 +433,7 @@ func setResourcesDefault(defaultValue dspa.ResourceRequirements, value **dspa.Re
 func (p *DSPAParams) ExtractParams(ctx context.Context, dsp *dspa.DataSciencePipelinesApplication, client client.Client, log logr.Logger) error {
 	p.Name = dsp.Name
 	p.Namespace = dsp.Namespace
+	p.DSPVersion = dsp.Spec.DSPVersion
 	p.Owner = dsp
 	p.APIServer = dsp.Spec.APIServer.DeepCopy()
 	p.APIServerServiceName = fmt.Sprintf("%s-%s", config.DSPServicePrefix, p.Name)

diff --git a/controllers/testdata/declarative/case_2/deploy/cr.yaml b/controllers/testdata/declarative/case_2/deploy/cr.yaml
@@ -3,6 +3,7 @@ kind: DataSciencePipelinesApplication
 metadata:
   name: testdsp2
 spec:
+  dspVersion: v2
   apiServer:
     deploy: true
     image: api-server:test2

diff --git a/controllers/testdata/declarative/case_2/expected/created/sample-pipeline.yaml.tmpl b/controllers/testdata/declarative/case_2/expected/created/sample-pipeline.yaml.tmpl
diff --git a/controllers/testdata/declarative/case_6/config.yaml b/controllers/testdata/declarative/case_6/config.yaml
@@ -0,0 +1,10 @@
+# When a complete DSPA is deployed with (defaults specified)
+Images:
+  ApiServer: api-server:test6
+  Artifact: artifact-manager:test6
+  PersistentAgent: persistenceagent:test6
+  ScheduledWorkflow: scheduledworkflow:test6
+  Cache: ubi-minimal:test6
+  MoveResultsImage: busybox:test6
+  MariaDB: mariadb:test6
+  OAuthProxy: oauth-proxy:test6
diff --git a/controllers/testdata/declarative/case_6/deploy/cr.yaml b/controllers/testdata/declarative/case_6/deploy/cr.yaml
@@ -0,0 +1,89 @@
+apiVersion: datasciencepipelinesapplications.opendatahub.io/v1alpha1
+kind: DataSciencePipelinesApplication
+metadata:
+  name: testdsp6
+spec:
+  apiServer:
+    deploy: true
+    image: api-server:test6
+    applyTektonCustomResource: true
+    archiveLogs: false
+    artifactImage: artifact-manager:test6
+    cacheImage: ubi-minimal:test6
+    moveResultsImage: busybox:test6
+    injectDefaultScript: true
+    stripEOF: true
+    enableOauth: true
+    enableSamplePipeline: true
+    terminateStatus: Cancelled
+    trackArtifacts: true
+    dbConfigConMaxLifetimeSec: 125
+    collectMetrics: true
+    autoUpdatePipelineDefaultVersion: true
+    resources:
+      requests:
+        cpu: "1231m"
+        memory: "1Gi"
+      limits:
+        cpu: "2522m"
+        memory: "5Gi"
+  persistenceAgent:
+    deploy: true
+    image: persistenceagent:test6
+    numWorkers: 5
+    resources:
+      requests:
+        cpu: "1233m"
+        memory: "1Gi"
+      limits:
+        cpu: "2524m"
+        memory: "5Gi"
+  scheduledWorkflow:
+    deploy: true
+    image: scheduledworkflow:test6
+    cronScheduleTimezone: EST
+    resources:
+      requests:
+        cpu: "1235m"
+        memory: "1Gi"
+      limits:
+        cpu: "2526m"
+        memory: "5Gi"
+  mlpipelineUI:
+    deploy: true
+    image: frontend:test6
+    configMap: some-test-configmap
+    resources:
+      requests:
+        cpu: "1239m"
+        memory: "1Gi"
+      limits:
+        cpu: "2530m"
+        memory: "5Gi"
+  database:
+    mariaDB:
+      deploy: true
+      image: mariadb:test6
+      username: testuser
+      pipelineDBName: randomDBName
+      pvcSize: 32Gi
+      resources:
+        requests:
+          cpu: "1212m"
+          memory: "1Gi"
+        limits:
+          cpu: "2554m"
+          memory: "5Gi"
+  objectStorage:
+    minio:
+      deploy: true
+      image: minio:test6
+      bucket: mlpipeline
+      pvcSize: 40Gi
+      resources:
+        requests:
+          cpu: "1334m"
+          memory: "1Gi"
+        limits:
+          cpu: "2535m"
+          memory: "5Gi"
diff --git a/controllers/testdata/declarative/case_6/expected/apiserver_deployment.yaml b/controllers/testdata/declarative/case_6/expected/apiserver_deployment.yaml
@@ -0,0 +1,214 @@
+apiVersion: apps/v1ds-pipeline-testdsp6
+kind: Deployment
+metadata:
+  name: ds-pipeline-testdsp6
+  namespace: default
+  labels:
+    app: ds-pipeline-testdsp6
+    component: data-science-pipelines
+    dspa: testdsp2
+spec:
+  selector:
+    matchLabels:
+      app: ds-pipeline-testdsp6
+      component: data-science-pipelines
+      dspa: testdsp2
+  template:
+    metadata:
+      labels:
+        app: ds-pipeline-testdsp6
+        component: data-science-pipelines
+        dspa: testdsp2
+    spec:
+      containers:
+        - env:
+            - name: POD_NAMESPACE
+              value: "default"
+            - name: DBCONFIG_USER
+              value: "testuser"
+            - name: DBCONFIG_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  key: "password"
+                  name: "ds-pipeline-db-testdsp2"
+            - name: DBCONFIG_DBNAME
+              value: "randomDBName"
+            - name: DBCONFIG_HOST
+              value: "mariadb-testdsp2.default.svc.cluster.local"
+            - name: DBCONFIG_PORT
+              value: "3306"
+            - name: ARTIFACT_BUCKET
+              value: "mlpipeline"
+            - name: ARTIFACT_ENDPOINT
+              value: "http://minio-testdsp2.default.svc.cluster.local:9000"
+            - name: ARTIFACT_SCRIPT
+              valueFrom:
+                configMapKeyRef:
+                  key: "artifact_script"
+                  name: "ds-pipeline-artifact-script-testdsp2"
+            - name: ARTIFACT_IMAGE
+              value: "artifact-manager:test2"
+            - name: ARCHIVE_LOGS
+              value: "false"
+            - name: EXECUTIONTYPE
+              value: PipelineRun
+            - name: TRACK_ARTIFACTS
+              value: "true"
+            - name: STRIP_EOF
+              value: "true"
+            - name: PIPELINE_RUNTIME
+              value: "tekton"
+            - name: DEFAULTPIPELINERUNNERSERVICEACCOUNT
+              value: "pipeline-runner-testdsp2"
+            - name: INJECT_DEFAULT_SCRIPT
+              value: "true"
+            - name: APPLY_TEKTON_CUSTOM_RESOURCE
+              value: "true"
+            - name: TERMINATE_STATUS
+              value: "Cancelled"
+            - name: AUTO_UPDATE_PIPELINE_DEFAULT_VERSION
+              value: "true"
+            - name: DBCONFIG_CONMAXLIFETIMESEC
+              value: "125"
+            - name: ML_PIPELINE_VISUALIZATIONSERVER_SERVICE_HOST
+              value: "ds-pipeline-visualizationserver"
+            - name: ML_PIPELINE_VISUALIZATIONSERVER_SERVICE_PORT
+              value: "8888"
+            - name: OBJECTSTORECONFIG_BUCKETNAME
+              value: "mlpipeline"
+            - name: OBJECTSTORECONFIG_ACCESSKEY
+              valueFrom:
+                secretKeyRef:
+                  key: "accesskey"
+                  name: "mlpipeline-minio-artifact"
+            - name: OBJECTSTORECONFIG_SECRETACCESSKEY
+              valueFrom:
+                secretKeyRef:
+                  key: "secretkey"
+                  name: "mlpipeline-minio-artifact"
+            - name: OBJECTSTORECONFIG_SECURE
+              value: "false"
+            - name: MINIO_SERVICE_SERVICE_HOST
+              value: "minio-testdsp2.default.svc.cluster.local"
+            - name: MINIO_SERVICE_SERVICE_PORT
+              value: "9000"
+            - name: CACHE_IMAGE
+              value: "ubi-minimal:test2"
+            - name: MOVERESULTS_IMAGE
+              value: "busybox:test2"
+            - name: METADATA_GRPC_SERVICE_SERVICE_HOST
+              value: "ds-pipeline-metadata-grpc-testdsp2.default.svc.cluster.local"
+            - name: METADATA_GRPC_SERVICE_SERVICE_PORT
+              value: "8080"
+            - name: ML_PIPELINE_SERVICE_HOST
+              value: ds-pipeline-testdsp6.default.svc.cluster.local
+            - name: ML_PIPELINE_SERVICE_PORT_GRPC
+              value: "8887"
+          image: api-server:test2
+          imagePullPolicy: Always
+          name: ds-pipeline-api-server
+          ports:
+            - containerPort: 8888
+              name: http
+              protocol: TCP
+            - containerPort: 8887
+              name: grpc
+              protocol: TCP
+          livenessProbe:
+            exec:
+              command:
+                - wget
+                - -q
+                - -S
+                - -O
+                - '-'
+                - http://localhost:8888/apis/v1beta1/healthz
+            initialDelaySeconds: 3
+            periodSeconds: 5
+            timeoutSeconds: 2
+          readinessProbe:
+            exec:
+              command:
+                - wget
+                - -q
+                - -S
+                - -O
+                - '-'
+                - http://localhost:8888/apis/v1beta1/healthz
+            initialDelaySeconds: 3
+            periodSeconds: 5
+            timeoutSeconds: 2
+          resources:
+            requests:
+              cpu: 1231m
+              memory: 1Gi
+            limits:
+              cpu: 2522m
+              memory: 5Gi
+          volumeMounts:
+            - mountPath: /config/sample_config.json
+              name: sample-config
+              subPath: sample_config.json
+            - mountPath: /samples/
+              name: sample-pipeline
+        - name: oauth-proxy
+          args:
+            - --https-address=:8443
+            - --provider=openshift
+            - --openshift-service-account=ds-pipeline-testdsp6
+            - --upstream=http://localhost:8888
+            - --tls-cert=/etc/tls/private/tls.crt
+            - --tls-key=/etc/tls/private/tls.key
+            - --cookie-secret=SECRET
+            - '--openshift-delegate-urls={"/": {"group":"route.openshift.io","resource":"routes","verb":"get","name":"ds-pipeline-testdsp6","namespace":"default"}}'
+            - '--openshift-sar={"namespace":"default","resource":"routes","resourceName":"ds-pipeline-testdsp6","verb":"get","resourceAPIGroup":"route.openshift.io"}'
+            - --skip-auth-regex='(^/metrics|^/apis/v1beta1/healthz)'
+          image: oauth-proxy:test2
+          ports:
+            - containerPort: 8443
+              name: oauth
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /oauth/healthz
+              port: oauth
+              scheme: HTTPS
+            initialDelaySeconds: 30
+            timeoutSeconds: 1
+            periodSeconds: 5
+            successThreshold: 1
+            failureThreshold: 3
+          readinessProbe:
+            httpGet:
+              path: /oauth/healthz
+              port: oauth
+              scheme: HTTPS
+            initialDelaySeconds: 5
+            timeoutSeconds: 1
+            periodSeconds: 5
+            successThreshold: 1
+            failureThreshold: 3
+          resources:
+            limits:
+              cpu: 100m
+              memory: 256Mi
+            requests:
+              cpu: 100m
+              memory: 256Mi
+          volumeMounts:
+            - mountPath: /etc/tls/private
+              name: proxy-tls
+      volumes:
+        - name: proxy-tls
+          secret:
+            secretName: ds-pipelines-proxy-tls-testdsp2
+            defaultMode: 420
+        - configMap:
+            defaultMode: 420
+            name: sample-config-testdsp2
+          name: sample-config
+        - configMap:
+            defaultMode: 420
+            name: sample-pipeline-testdsp2
+          name: sample-pipeline
+      serviceAccountName: ds-pipeline-testdsp6
diff --git a/controllers/testdata/declarative/case_6/expected/configmap_artifact_script.yaml b/controllers/testdata/declarative/case_6/expected/configmap_artifact_script.yaml
@@ -0,0 +1,35 @@
+apiVersion: v1
+data:
+  artifact_script: |-
+    #!/usr/bin/env sh
+    push_artifact() {
+        workspace_dir=$(echo $(context.taskRun.name) | sed -e "s/$(context.pipeline.name)-//g")
+        workspace_dest=/workspace/${workspace_dir}/artifacts/$(context.pipelineRun.name)/$(context.taskRun.name)
+        artifact_name=$(basename $2)
+        if [ -f "$workspace_dest/$artifact_name" ]; then
+            echo sending to: ${workspace_dest}/${artifact_name}
+            tar -cvzf $1.tgz -C ${workspace_dest} ${artifact_name}
+            aws s3 --endpoint http://minio-testdsp2.default.svc.cluster.local:9000 cp $1.tgz s3://mlpipeline/artifacts/$PIPELINERUN/$PIPELINETASK/$1.tgz
+        elif [ -f "$2" ]; then
+            tar -cvzf $1.tgz -C $(dirname $2) ${artifact_name}
+            aws s3 --endpoint http://minio-testdsp2.default.svc.cluster.local:9000 cp $1.tgz s3://mlpipeline/artifacts/$PIPELINERUN/$PIPELINETASK/$1.tgz
+        else
+            echo "$2 file does not exist. Skip artifact tracking for $1"
+        fi
+    }
+    push_log() {
+        cat /var/log/containers/$PODNAME*$NAMESPACE*step-main*.log > step-main.log
+        push_artifact main-log step-main.log
+    }
+    strip_eof() {
+        if [ -f "$2" ]; then
+            awk 'NF' $2 | head -c -1 > $1_temp_save && cp $1_temp_save $2
+        fi
+    }
+kind: ConfigMap
+metadata:
+  name: ds-pipeline-artifact-script-testdsp2
+  namespace: default
+  labels:
+    app: ds-pipeline-testdsp6
+    component: data-science-pipelines