Merge pull request #422 from rimolive/dspv2-rmartine

Last changes for DSPv2 PoC

api/v1alpha1/dspipeline_types.go

@@ -40,9 +40,8 @@ type DSPASpec struct {
 	// ObjectStorage specifies Object Store configurations, used for DS Pipelines artifact passing and storage. Specify either the your own External Storage (e.g. AWS S3), or use the default Minio deployment (unsupported, primarily for development, and testing) .
 	// +kubebuilder:validation:Required
 	*ObjectStorage `json:"objectStorage"`
-	// +kubebuilder:validation:Optional
-	// +kubebuilder:default:={deploy: false}
-	*MLMD `json:"mlmd"`
+	// +kubebuilder:default:={deploy: true}
+	*MLMD `json:"mlmd,omitempty"`
 	// +kubebuilder:validation:Optional
 	// +kubebuilder:default:={deploy: false}
 	*CRDViewer `json:"crdviewer"`
@@ -231,8 +230,8 @@ type Minio struct {
 }
 
 type MLMD struct {
-	// Enable DS Pipelines Operator management of MLMD. Setting Deploy to false disables operator reconciliation. Default: false
-	// +kubebuilder:default:=false
+	// Enable DS Pipelines Operator management of MLMD. Setting Deploy to false disables operator reconciliation. Default: true
+	// +kubebuilder:default:=true
 	// +kubebuilder:validation:Optional
 	Deploy  bool `json:"deploy"`
 	*Envoy  `json:"envoy,omitempty"`

config/crd/bases/datasciencepipelinesapplications.opendatahub.io_datasciencepipelinesapplications.yaml

@@ -285,13 +285,13 @@ spec:
                 type: string
               mlmd:
                 default:
-                  deploy: false
+                  deploy: true
                 properties:
                   deploy:
-                    default: false
+                    default: true
                     description: 'Enable DS Pipelines Operator management of MLMD.
                       Setting Deploy to false disables operator reconciliation. Default:
-                      false'
+                      true'
                     type: boolean
                   envoy:
                     properties:

config/internal/apiserver/default/deployment.yaml.tmpl

@@ -98,12 +98,16 @@ spec:
               value: "{{.APIServer.MoveResultsImage}}"
             {{ if .MLMD.Deploy }}
             - name: METADATA_GRPC_SERVICE_SERVICE_HOST
-              value: "ds-pipeline-metadata-grpc-{{.Name}}"
+              value: "ds-pipeline-metadata-grpc-{{.Name}}.{{.Namespace}}.svc.cluster.local"
             {{ if.MLMD.GRPC.Port }}
             - name: METADATA_GRPC_SERVICE_SERVICE_PORT
               value: "{{.MLMD.GRPC.Port}}"
             {{ end }}
             {{ end }}
+            - name: ML_PIPELINE_SERVICE_HOST
+              value: ds-pipeline-{{.Name}}.{{.Namespace}}.svc.cluster.local
+            - name: ML_PIPELINE_SERVICE_PORT_GRPC
+              value: "8887"
           image: {{.APIServer.Image}}
           imagePullPolicy: Always
           name: ds-pipeline-api-server

config/internal/common/default/policy.yaml.tmpl

@@ -25,10 +25,7 @@ spec:
         - namespaceSelector:
             matchLabels:
               kubernetes.io/metadata.name: redhat-ods-monitoring
-        - podSelector:
-            matchLabels:
-              app.kubernetes.io/name: kfp-driver
-          namespaceSelector:
+        - namespaceSelector:
             matchLabels:
               kubernetes.io/metadata.name: openshift-pipelines
         - podSelector:
@@ -72,3 +69,13 @@ spec:
           port: 8888
         - protocol: TCP
           port: 8887
+    - ports:
+        - protocol: TCP
+          port: 8080
+      from:
+        - podSelector:
+            matchLabels:
+              app.kubernetes.io/name: data-science-pipelines-operator-driver
+          namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: openshift-pipelines

config/v2/driver/deployment.yaml

@@ -42,6 +42,18 @@ spec:
           value: config-observability
         - name: METRICS_DOMAIN
           value: tekton.dev/pipeline
+        - name: ML_PIPELINE_SERVICE_HOST
+          value: ds-pipeline-sample.data-science-pipelines-application-v2.svc.cluster.local
+        - name: ML_PIPELINE_SERVICE_PORT_GRPC
+          value: '8887'
+        - name: MINIO_SERVICE_SERVICE_HOST
+          value: minio-sample.data-science-pipelines-application-v2.svc.cluster.local
+        - name: MINIO_SERVICE_SERVICE_PORT
+          value: '9000'
+        - name: METADATA_GRPC_SERVICE_SERVICE_HOST
+          value: ds-pipeline-metadata-grpc-sample.data-science-pipelines-application-v2.svc.cluster.local
+        - name: METADATA_GRPC_SERVICE_SERVICE_PORT
+          value: '8080'
         image: quay.io/internaldatahub/tekton-driver:2.0.0
         imagePullPolicy: Always
         name: driver

config/v2/exithandler/clusterrole.leaderelection.yaml

@@ -4,7 +4,7 @@ metadata:
   labels:
     app.kubernetes.io/instance: default
     app.kubernetes.io/part-of: kfp-tekton
-  name: leader-election-clusterrole
+  name: exithandler-leader-election-clusterrole
 rules:
 - apiGroups:
   - coordination.k8s.io

config/v2/pipelineloop/controller/deployment.yaml

@@ -46,7 +46,7 @@ spec:
           value: config-observability
         - name: METRICS_DOMAIN
           value: tekton.dev/pipeline
-        image: quay.io/internaldatahub/tektonpipelineloop-controller:2.0.0
+        image: quay.io/internaldatahub/tekton-pipelineloop-controller:2.0.0
         name: pipelineloop-controller
         securityContext:
           allowPrivilegeEscalation: false

config/v2/pipelineloop/webhook/deployment.yaml

@@ -50,7 +50,7 @@ spec:
           value: tektonpipelineloop-webhook-certs
         - name: METRICS_DOMAIN
           value: tekton.dev/pipeline
-        image: quay.io/internaldatahub/tektonpipelineloop-webhook:2.0.0
+        image: quay.io/internaldatahub/tekton-pipelineloop-webhook:2.0.0
         name: webhook
         ports:
         - containerPort: 9090

config/v2/tektoncrds/kustomization.yaml

@@ -1,4 +1,2 @@
 resources:
 - crd.yaml
-- scc.anyuid.yaml
-- scc.privileged.yaml

controllers/dspipeline_params.go

@@ -111,13 +111,6 @@ func (p *DSPAParams) ObjectStorageHealthCheckDisabled(dsp *dspa.DataSciencePipel
 	return false
 }
 
-func (p *DSPAParams) UsingMLMD(dsp *dspa.DataSciencePipelinesApplication) bool {
-	if dsp.Spec.MLMD != nil {
-		return dsp.Spec.MLMD.Deploy
-	}
-	return false
-}
-
 func passwordGen(n int) string {
 	rand.Seed(time.Now().UnixNano())
 	var chars = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890")

controllers/mlmd.go

@@ -26,15 +26,18 @@ func (r *DSPAReconciler) ReconcileMLMD(dsp *dspav1alpha1.DataSciencePipelinesApp
 
 	log := r.Log.WithValues("namespace", dsp.Namespace).WithValues("dspa_name", dsp.Name)
 
-	if params.UsingMLMD(dsp) {
-		log.Info("Applying ML-Metadata (MLMD) Resources")
+	if !dsp.Spec.MLMD.Deploy {
+		r.Log.Info("Skipping Application of ML-Metadata (MLMD) Resources")
+		return nil
+	}
 
-		err := r.ApplyDir(dsp, params, mlmdTemplatesDir)
-		if err != nil {
-			return err
-		}
+	log.Info("Applying ML-Metadata (MLMD) Resources")
 
-		log.Info("Finished applying MLMD Resources")
+	err := r.ApplyDir(dsp, params, mlmdTemplatesDir)
+	if err != nil {
+		return err
 	}
+
+	log.Info("Finished applying MLMD Resources")
 	return nil
 }

controllers/mlmd_test.go

@@ -211,6 +211,9 @@ func TestDefaultDeployBehaviorMLMD(t *testing.T) {
 				// possibly due to test setup - Investigate.
 				ArchiveLogs: true,
 			},
+			MLMD: &dspav1alpha1.MLMD{
+				Deploy: true,
+			},
 			Database: &dspav1alpha1.Database{
 				DisableHealthCheck: false,
 				MariaDB: &dspav1alpha1.MariaDB{
@@ -261,18 +264,18 @@ func TestDefaultDeployBehaviorMLMD(t *testing.T) {
 	// Ensure MLMD-Envoy resources still doesn't exist
 	deployment = &appsv1.Deployment{}
 	created, err = reconciler.IsResourceCreated(ctx, deployment, expectedMLMDEnvoyName, testNamespace)
-	assert.False(t, created)
+	assert.True(t, created)
 	assert.Nil(t, err)
 
 	// Ensure MLMD-GRPC resources still doesn't exist
 	deployment = &appsv1.Deployment{}
 	created, err = reconciler.IsResourceCreated(ctx, deployment, expectedMLMDGRPCName, testNamespace)
-	assert.False(t, created)
+	assert.True(t, created)
 	assert.Nil(t, err)
 
 	// Ensure MLMD-Writer resources still doesn't exist
 	deployment = &appsv1.Deployment{}
 	created, err = reconciler.IsResourceCreated(ctx, deployment, expectedMLMDWriterName, testNamespace)
-	assert.False(t, created)
+	assert.True(t, created)
 	assert.Nil(t, err)
 }

controllers/testdata/declarative/case_0/expected/created/apiserver_deployment.yaml

@@ -96,6 +96,14 @@ spec:
               value: "ubi-minimal:test0"
             - name: MOVERESULTS_IMAGE
               value: "busybox:test0"
+            - name: METADATA_GRPC_SERVICE_SERVICE_HOST
+              value: "ds-pipeline-metadata-grpc-testdsp0.default.svc.cluster.local"
+            - name: METADATA_GRPC_SERVICE_SERVICE_PORT
+              value: "8080"
+            - name: ML_PIPELINE_SERVICE_HOST
+              value: ds-pipeline-testdsp0.default.svc.cluster.local
+            - name: ML_PIPELINE_SERVICE_PORT_GRPC
+              value: "8887"
           image: api-server:test0
           imagePullPolicy: Always
           name: ds-pipeline-api-server

controllers/testdata/declarative/case_2/expected/created/apiserver_deployment.yaml

@@ -96,6 +96,14 @@ spec:
               value: "ubi-minimal:test2"
             - name: MOVERESULTS_IMAGE
               value: "busybox:test2"
+            - name: METADATA_GRPC_SERVICE_SERVICE_HOST
+              value: "ds-pipeline-metadata-grpc-testdsp2.default.svc.cluster.local"
+            - name: METADATA_GRPC_SERVICE_SERVICE_PORT
+              value: "8080"
+            - name: ML_PIPELINE_SERVICE_HOST
+              value: ds-pipeline-testdsp2.default.svc.cluster.local
+            - name: ML_PIPELINE_SERVICE_PORT_GRPC
+              value: "8887"
           image: api-server:test2
           imagePullPolicy: Always
           name: ds-pipeline-api-server

controllers/testdata/declarative/case_3/expected/created/apiserver_deployment.yaml

@@ -96,6 +96,14 @@ spec:
               value: ubi-minimal:test3
             - name: MOVERESULTS_IMAGE
               value: busybox:test3
+            - name: METADATA_GRPC_SERVICE_SERVICE_HOST
+              value: "ds-pipeline-metadata-grpc-testdsp3.default.svc.cluster.local"
+            - name: METADATA_GRPC_SERVICE_SERVICE_PORT
+              value: "8080"
+            - name: ML_PIPELINE_SERVICE_HOST
+              value: ds-pipeline-testdsp3.default.svc.cluster.local
+            - name: ML_PIPELINE_SERVICE_PORT_GRPC
+              value: "8887"
           image: api-server:test3
           imagePullPolicy: Always
           name: ds-pipeline-api-server

controllers/testdata/declarative/case_4/expected/created/apiserver_deployment.yaml

@@ -96,6 +96,14 @@ spec:
               value: "this-ubi-minimal-image-from-cr-should-be-used:test4"
             - name: MOVERESULTS_IMAGE
               value: "this-busybox-image-from-cr-should-be-used:test4"
+            - name: METADATA_GRPC_SERVICE_SERVICE_HOST
+              value: "ds-pipeline-metadata-grpc-testdsp4.default.svc.cluster.local"
+            - name: METADATA_GRPC_SERVICE_SERVICE_PORT
+              value: "8080"
+            - name: ML_PIPELINE_SERVICE_HOST
+              value: ds-pipeline-testdsp4.default.svc.cluster.local
+            - name: ML_PIPELINE_SERVICE_PORT_GRPC
+              value: "8887"
           image: this-apiserver-image-from-cr-should-be-used:test4
           imagePullPolicy: Always
           name: ds-pipeline-api-server

controllers/testdata/declarative/case_5/expected/created/apiserver_deployment.yaml

@@ -97,9 +97,13 @@ spec:
             - name: MOVERESULTS_IMAGE
               value: "busybox:test5"
             - name: METADATA_GRPC_SERVICE_SERVICE_HOST
-              value: ds-pipeline-metadata-grpc-testdsp5
+              value: "ds-pipeline-metadata-grpc-testdsp5.default.svc.cluster.local"
             - name: METADATA_GRPC_SERVICE_SERVICE_PORT
               value: "1337"
+            - name: ML_PIPELINE_SERVICE_HOST
+              value: ds-pipeline-testdsp5.default.svc.cluster.local
+            - name: ML_PIPELINE_SERVICE_PORT_GRPC
+              value: "8887"
           image: api-server:test5
           imagePullPolicy: Always
           name: ds-pipeline-api-server