Merge pull request #615 from VaniHaripriya/RHOAIENG-1676-Fix

Add integration tests for external storage/db connections
opendatahub-io · Mar 22, 2024 · db78e2b · db78e2b
2 parents 259315b + 0a6a268
commit db78e2b
Show file tree

Hide file tree

Showing 13 changed files with 363 additions and 11 deletions.
diff --git a/.github/resources/mariadb/deployment.yaml b/.github/resources/mariadb/deployment.yaml
@@ -0,0 +1,74 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: mariadb
+  namespace: test-mariadb
+  labels:
+    app: mariadb
+    component: data-science-pipelines
+spec:
+  strategy:
+    type: Recreate  # Need this since backing PVC is ReadWriteOnce, which creates resource lock condition in default Rolling strategy
+  selector:
+    matchLabels:
+      app: mariadb
+      component: data-science-pipelines
+  template:
+    metadata:
+      labels:
+        app: mariadb
+        component: data-science-pipelines
+    spec:
+      containers:
+        - name: mariadb
+          image: quay.io/centos7/mariadb-103-centos7:ea07c0dade9571d78a272b453fd2dea92077dc7f
+          ports:
+            - containerPort: 3306
+          readinessProbe:
+            exec:
+              command:
+                - /bin/sh
+                - "-i"
+                - "-c"
+                - >-
+                  MYSQL_PWD=$MYSQL_PASSWORD mysql -h 127.0.0.1 -u $MYSQL_USER -D
+                  $MYSQL_DATABASE -e 'SELECT 1'
+            failureThreshold: 3
+            initialDelaySeconds: 5
+            periodSeconds: 10
+            successThreshold: 1
+            timeoutSeconds: 1
+          livenessProbe:
+            failureThreshold: 3
+            initialDelaySeconds: 30
+            periodSeconds: 10
+            successThreshold: 1
+            tcpSocket:
+              port: 3306
+            timeoutSeconds: 1
+          env:
+            - name: MYSQL_USER
+              value: "mlpipeline"
+            - name: MYSQL_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  key: "password"
+                  name: "ds-pipeline-db-test"
+            - name: MYSQL_DATABASE
+              value: "mlpipeline"
+            - name: MYSQL_ALLOW_EMPTY_PASSWORD
+              value: "true"
+          resources:
+            limits:
+              cpu: 20m
+              memory: 500Mi
+            requests:
+              cpu: 20m
+              memory: 100m
+          volumeMounts:
+            - name: mariadb-persistent-storage
+              mountPath: /var/lib/mysql
+      volumes:
+        - name: mariadb-persistent-storage
+          persistentVolumeClaim:
+            claimName: mariadb-test
diff --git a/.github/resources/mariadb/kustomization.yaml b/.github/resources/mariadb/kustomization.yaml
@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: test-mariadb
+resources:
+  - deployment.yaml
+  - pvc.yaml
+  - secret.yaml
+  - service.yaml
diff --git a/.github/resources/mariadb/pvc.yaml b/.github/resources/mariadb/pvc.yaml
@@ -0,0 +1,11 @@
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: mariadb-test
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 500Mi
+  volumeMode: Filesystem
diff --git a/.github/resources/mariadb/secret.yaml b/.github/resources/mariadb/secret.yaml
@@ -0,0 +1,11 @@
+kind: Secret
+apiVersion: v1
+metadata:
+  name: ds-pipeline-db-test
+  namespace: test-mariadb
+  labels:
+    app: mariadb
+    component: data-science-pipelines
+stringData:
+  password: password
+type: Opaque
diff --git a/.github/resources/mariadb/service.yaml b/.github/resources/mariadb/service.yaml
@@ -0,0 +1,16 @@
+kind: Service
+apiVersion: v1
+metadata:
+  name: mariadb
+  namespace: test-mariadb
+  labels:
+    app: mariadb
+    component: data-science-pipelines
+spec:
+  ports:
+    - port: 3306
+      protocol: TCP
+      targetPort: 3306
+  selector:
+    app: mariadb
+    component: data-science-pipelines
diff --git a/.github/resources/minio/deployment.yaml b/.github/resources/minio/deployment.yaml
@@ -0,0 +1,54 @@
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: minio
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: minio
+  template:
+    metadata:
+      labels:
+        app: minio
+    spec:
+      volumes:
+        - name: data
+          persistentVolumeClaim:
+            claimName: minio
+      containers:
+        - resources:
+            limits:
+              cpu: 20m
+              memory: 500Mi
+            requests:
+              cpu: 20m
+              memory: 100m
+          name: minio
+          env:
+            - name: MINIO_ROOT_USER
+              valueFrom:
+                secretKeyRef:
+                  name: minio
+                  key: accesskey
+            - name: MINIO_ROOT_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: minio
+                  key: secretkey
+          ports:
+            - containerPort: 9000
+              protocol: TCP
+          imagePullPolicy: IfNotPresent
+          volumeMounts:
+            - name: data
+              mountPath: /data
+              subPath: minio
+          image: 'quay.io/minio/minio:RELEASE.2023-10-16T04-13-43Z'
+          args:
+            - server
+            - /data
+            - --console-address
+            - ":9001"
+  strategy:
+    type: Recreate
diff --git a/.github/resources/minio/kustomization.yaml b/.github/resources/minio/kustomization.yaml
@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: test-minio
+resources:
+  - deployment.yaml
+  - service.yaml
+  - pvc.yaml
+  - secret.yaml
diff --git a/.github/resources/minio/pvc.yaml b/.github/resources/minio/pvc.yaml
@@ -0,0 +1,11 @@
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: minio
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 500Mi
+  volumeMode: Filesystem
diff --git a/.github/resources/minio/secret.yaml b/.github/resources/minio/secret.yaml
@@ -0,0 +1,8 @@
+kind: Secret
+apiVersion: v1
+metadata:
+  name: minio
+stringData:
+  accesskey: accesskey
+  secretkey: secretkey
+type: Opaque
diff --git a/.github/resources/minio/service.yaml b/.github/resources/minio/service.yaml
@@ -0,0 +1,16 @@
+kind: Service
+apiVersion: v1
+metadata:
+  name: minio
+spec:
+  ports:
+    - name: https
+      protocol: TCP
+      port: 9000
+      targetPort: 9000
+    - name: console
+      protocol: TCP
+      port: 9001
+      targetPort: 9001
+  selector:
+    app: minio
diff --git a/.github/workflows/kind-integration.yml b/.github/workflows/kind-integration.yml
@@ -23,9 +23,15 @@ concurrency:
 env:
   IMAGE_REPO_DSPO: data-science-pipelines-operator
   DSPA_NAMESPACE: test-dspa
+  DSPA_EXTERNAL_NAMESPACE: test-dspa-external
+  MINIO_NAMESPACE: test-minio
+  MARIADB_NAMESPACE: test-mariadb
   DSPA_NAME: test-dspa
+  DSPA_EXTERNAL_NAME: test-dspa-external
   DSPA_DEPLOY_WAIT_TIMEOUT: 300
   INTEGRATION_TESTS_DIR: ${{ github.workspace }}/tests
+  DSPA_PATH: ${{ github.workspace }}/tests/resources/dspa-lite.yaml
+  DSPA_EXTERNAL_PATH: ${{ github.workspace }}/tests/resources/dspa-external-lite.yaml
 jobs:
   dspo-tests:
     runs-on: ubuntu-20.04
@@ -76,21 +82,64 @@ jobs:
           make podman-push -e IMG="${DSPO_IMAGE}"
           make deploy-kind -e IMG="${DSPO_IMAGE}"
 
-      - name: Wait for DSPO
+      - name: Create Minio Namespace
+        run: |
+          kubectl create namespace ${{ env.MINIO_NAMESPACE }}
+
+      - name: Deploy Minio
+        working-directory: ${{ github.workspace }}/.github/resources/minio
+        run: |
+          kustomize build . |  oc -n ${{ env.MINIO_NAMESPACE }} apply -f -
+
+      - name: Create MariaDB Namespace
+        run: |
+          kubectl create namespace ${{ env.MARIADB_NAMESPACE }}
+
+      - name: Deploy MariaDB
+        working-directory: ${{ github.workspace }}/.github/resources/mariadb
+        run: |
+          kustomize build . |  oc -n ${{ env.MARIADB_NAMESPACE }} apply -f -
+
+      - name: Wait for Dependencies (DSPO, Minio, Mariadb)
         run: |
           kubectl wait -n opendatahub --timeout=60s --for=condition=Available=true deployment data-science-pipelines-operator-controller-manager
+          kubectl wait -n ${{ env.MARIADB_NAMESPACE }} --timeout=60s --for=condition=Available=true deployment mariadb
+          kubectl wait -n ${{ env.MINIO_NAMESPACE }} --timeout=60s --for=condition=Available=true deployment minio
 
       - name: Create DSPA Namespace
         run: |
           kubectl create namespace ${{ env.DSPA_NAMESPACE }}
 
+      - name: Create Namespace for DSPA with External connections
+        run: |
+          kubectl create namespace ${{ env.DSPA_EXTERNAL_NAMESPACE }}
+
+      - name: Create Minio Secrets
+        run: |
+          kubectl create secret generic minio --from-literal=accesskey=accesskey --from-literal=secretkey=secretkey -n ${{ env.DSPA_EXTERNAL_NAMESPACE }}
+
+      - name: Create MariaDB Secrets
+        run: |
+          password=$(kubectl get secret ds-pipeline-db-test -n test-mariadb -o jsonpath="{.data.password}" | base64 --decode)
+          kubectl create secret generic ds-pipeline-db-test --from-literal=password="$password" -n ${{ env.DSPA_EXTERNAL_NAMESPACE }}
+
       - name: Run tests
         working-directory: ${{ github.workspace }}
         env:
           NAMESPACE: ${{ env.DSPA_NAMESPACE }}
           DSPA_NAME: ${{ env.DSPA_NAME }}
+          DSPA_PATH: ${{ env.DSPA_PATH }}
+        run: |
+            make integrationtest K8SAPISERVERHOST=$(oc whoami --show-server) DSPANAMESPACE=${NAMESPACE} DSPAPATH=${DSPA_PATH}
+
+      - name: Run tests for DSPA with External Connections
+        working-directory: ${{ github.workspace }}
+        env:
+          NAMESPACE: ${{ env.DSPA_EXTERNAL_NAMESPACE }}
+          DSPA_NAME: ${{ env.DSPA_EXTERNAL_NAME }}
+          DSPA_EXTERNAL_PATH: ${{ env.DSPA_EXTERNAL_PATH }}
         run: |
-          make integrationtest K8SAPISERVERHOST=$(oc whoami --show-server) DSPANAMESPACE=${NAMESPACE}
+            make integrationtest K8SAPISERVERHOST=$(oc whoami --show-server) DSPANAMESPACE=${NAMESPACE} DSPAPATH=${DSPA_EXTERNAL_PATH}
 
       - name: Clean up
         run: |

diff --git a/tests/dspa_v2_test.go b/tests/dspa_v2_test.go
@@ -31,6 +31,24 @@ import (
 
 func (suite *IntegrationTestSuite) TestDSPADeployment() {
 	podCount := 8
+	if suite.DSPA.Spec.ObjectStorage.ExternalStorage != nil {
+		podCount = podCount - 1
+	}
+	if suite.DSPA.Spec.Database.ExternalDB != nil {
+		podCount = podCount - 1
+	}
+	deployments := []string{
+		fmt.Sprintf("ds-pipeline-%s", suite.DSPA.Name),
+		fmt.Sprintf("ds-pipeline-persistenceagent-%s", suite.DSPA.Name),
+		fmt.Sprintf("ds-pipeline-scheduledworkflow-%s", suite.DSPA.Name),
+	}
+
+	if suite.DSPA.Spec.ObjectStorage.ExternalStorage == nil && suite.DSPA.Spec.Database.ExternalDB == nil {
+		deployments = append(deployments,
+			fmt.Sprintf("mariadb-%s", suite.DSPA.Name),
+			fmt.Sprintf("minio-%s", suite.DSPA.Name),
+		)
+	}
 	suite.T().Run("with default MariaDB and Minio", func(t *testing.T) {
 		t.Run(fmt.Sprintf("should have %d pods", podCount), func(t *testing.T) {
 			podList := &corev1.PodList{}
@@ -41,15 +59,6 @@ func (suite *IntegrationTestSuite) TestDSPADeployment() {
 			require.NoError(t, err)
 			assert.Equal(t, podCount, len(podList.Items))
 		})
-
-		deployments := []string{
-			fmt.Sprintf("ds-pipeline-%s", suite.DSPA.Name),
-			fmt.Sprintf("ds-pipeline-persistenceagent-%s", suite.DSPA.Name),
-			fmt.Sprintf("ds-pipeline-scheduledworkflow-%s", suite.DSPA.Name),
-			fmt.Sprintf("mariadb-%s", suite.DSPA.Name),
-			fmt.Sprintf("minio-%s", suite.DSPA.Name),
-		}
-
 		for _, deployment := range deployments {
 			t.Run(fmt.Sprintf("should have a ready %s deployment", deployment), func(t *testing.T) {
 				testUtil.TestForSuccessfulDeployment(t, suite.Ctx, suite.DSPANamespace, deployment, suite.Clientmgr.k8sClient)