add service ca bundle for pod to pod tls

Signed-off-by: Humair Khan <HumairAK@users.noreply.github.com>

controllers/config/defaults.go

@@ -42,6 +42,9 @@ const (
 	CustomDSPTrustedCAConfigMapNamePrefix = "dsp-trusted-ca"
 	CustomDSPTrustedCAConfigMapKey        = "dsp-ca.crt"
 
+	OpenshiftServiceCAConfigMapName = "openshift-service-ca.crt"
+	OpenshiftServiceCAConfigMapKey  = "service-ca.crt"
+
 	DefaultSystemSSLCertFile     = "SSL_CERT_FILE"
 	DefaultSystemSSLCertFilePath = "/etc/pki/tls/certs/ca-bundle.crt" // Fedora/RHEL 6
 

controllers/dspipeline_params.go

@@ -649,7 +649,7 @@ func (p *DSPAParams) ExtractParams(ctx context.Context, dsp *dspa.DataSciencePip
 		// Track whether the "ca-bundle.crt" configmap key from odh-trusted-ca bundle
 		// was found, this will be used to decide whether we need to account for this
 		// ourselves later or not.
-		odhTrustedCABundleAdded := false
+		wellKnownCABundleAdded := false
 
 		// Check for cert bundle provided by the platform instead of by the DSPA user
 		// If it exists, include this cert for tls verifications
@@ -677,7 +677,7 @@ func (p *DSPAParams) ExtractParams(ctx context.Context, dsp *dspa.DataSciencePip
 			// however if a user creates this, they may accidentally leave this out, so we need to account for this
 			_, ok := odhTrustedCABundleConfigMap.Data[config.GlobalODHCaBundleConfigMapSystemBundleKey]
 			if ok {
-				odhTrustedCABundleAdded = true
+				wellKnownCABundleAdded = true
 			}
 		}
 
@@ -699,6 +699,22 @@ func (p *DSPAParams) ExtractParams(ctx context.Context, dsp *dspa.DataSciencePip
 			}
 		}
 
+		// If PodToPodTLS is enabled, we need to include service-ca ca-bundles to recognize the certs
+		// that are signed by service-ca. These can be accessed via "openshift-service-ca.crt"
+		// configmap.
+		if p.PodToPodTLS {
+			serviceCA, serviceCACfgErr := util.GetConfigMap(ctx, config.OpenshiftServiceCAConfigMapName, p.Namespace, client)
+			if serviceCACfgErr != nil {
+				log.Info(fmt.Sprintf("Encountered error when attempting to fetch ConfigMap: [%s]. Error: %v", config.OpenshiftServiceCAConfigMapName, serviceCA))
+				return serviceCACfgErr
+			}
+			serviceCABundle := util.GetConfigMapValue(config.OpenshiftServiceCAConfigMapKey, serviceCA)
+			if serviceCABundle == "" {
+				return fmt.Errorf("expected key %s from configmap %s not found", config.OpenshiftServiceCAConfigMapKey, config.OpenshiftServiceCAConfigMapName)
+			}
+			p.APICustomPemCerts = append(p.APICustomPemCerts, []byte(serviceCABundle))
+		}
+
 		if p.APIServer.CABundleFileMountPath != "" {
 			p.CustomCABundleRootMountPath = p.APIServer.CABundleFileMountPath
 		}
@@ -722,7 +738,7 @@ func (p *DSPAParams) ExtractParams(ctx context.Context, dsp *dspa.DataSciencePip
 			// We need to ensure system certs are always part of this new configmap
 			// We can either source this from odh-trusted-ca-bundle cfgmap if provided,
 			// or fetch one from "config-trusted-cabundle" configmap, which is always present in an ocp ns
-			if !odhTrustedCABundleAdded {
+			if !wellKnownCABundleAdded {
 				certs, sysCertsErr := util.GetSystemCerts()
 				if sysCertsErr != nil {
 					return sysCertsErr

controllers/dspipeline_params_test.go

@@ -173,6 +173,38 @@ func TestExtractParams_CABundle(t *testing.T) {
 			},
 			SSLCertFileEnv: "testdata/tls/dummy-ca-bundle.crt",
 		},
+
+		{
+			msg:                         "pod to pod tls enabled",
+			dsp:                         testutil.CreateDSPAWithAPIServerPodtoPodTlsEnabled(),
+			CustomCABundleRootMountPath: "/dsp-custom-certs",
+			CustomSSLCertDir:            strPtr("/dsp-custom-certs:/etc/ssl/certs:/etc/pki/tls/certs"),
+			PiplinesCABundleMountPath:   "/dsp-custom-certs/dsp-ca.crt",
+			APICustomPemCerts:           [][]byte{[]byte("service-ca-contents")},
+			CustomCABundle:              &dspav1alpha1.CABundle{ConfigMapKey: "dsp-ca.crt", ConfigMapName: "dsp-trusted-ca-testdspa"},
+			ConfigMapPreReq: []*v1.ConfigMap{
+				{
+					ObjectMeta: metav1.ObjectMeta{Name: "openshift-service-ca.crt", Namespace: "testnamespace"},
+					Data:       map[string]string{"service-ca.crt": "service-ca-contents"},
+				},
+			},
+		},
+		{
+			msg:                         "pod to pod tls enabled with sys certs",
+			dsp:                         testutil.CreateDSPAWithAPIServerPodtoPodTlsEnabled(),
+			CustomCABundleRootMountPath: "/dsp-custom-certs",
+			CustomSSLCertDir:            strPtr("/dsp-custom-certs:/etc/ssl/certs:/etc/pki/tls/certs"),
+			PiplinesCABundleMountPath:   "/dsp-custom-certs/dsp-ca.crt",
+			APICustomPemCerts:           [][]byte{[]byte("service-ca-contents"), []byte("dummycontent")},
+			CustomCABundle:              &dspav1alpha1.CABundle{ConfigMapKey: "dsp-ca.crt", ConfigMapName: "dsp-trusted-ca-testdspa"},
+			ConfigMapPreReq: []*v1.ConfigMap{
+				{
+					ObjectMeta: metav1.ObjectMeta{Name: "openshift-service-ca.crt", Namespace: "testnamespace"},
+					Data:       map[string]string{"service-ca.crt": "service-ca-contents"},
+				},
+			},
+			SSLCertFileEnv: "testdata/tls/dummy-ca-bundle.crt",
+		},
 	}
 
 	for _, test := range tt {
@@ -199,19 +231,19 @@ func TestExtractParams_CABundle(t *testing.T) {
 			}
 
 			actualCustomCABundleRootMountPath := actualParams.CustomCABundleRootMountPath
-			assert.Equal(t, actualCustomCABundleRootMountPath, test.CustomCABundleRootMountPath)
+			assert.Equal(t, test.CustomCABundleRootMountPath, actualCustomCABundleRootMountPath)
 
 			actualCustomSSLCertDir := actualParams.CustomSSLCertDir
-			assert.Equal(t, actualCustomSSLCertDir, test.CustomSSLCertDir)
+			assert.Equal(t, test.CustomSSLCertDir, actualCustomSSLCertDir)
 
 			actualPipelinesCABundleMountPath := actualParams.PiplinesCABundleMountPath
-			assert.Equal(t, actualPipelinesCABundleMountPath, test.PiplinesCABundleMountPath)
+			assert.Equal(t, test.PiplinesCABundleMountPath, actualPipelinesCABundleMountPath)
 
 			actualAPICustomPemCerts := actualParams.APICustomPemCerts
-			assert.Equal(t, actualAPICustomPemCerts, test.APICustomPemCerts)
+			assert.Equal(t, test.APICustomPemCerts, actualAPICustomPemCerts)
 
 			actualCustomCABundle := actualParams.CustomCABundle
-			assert.Equal(t, actualCustomCABundle, test.CustomCABundle)
+			assert.Equal(t, test.CustomCABundle, actualCustomCABundle)
 
 			if test.ConfigMapPreReq != nil && len(test.ConfigMapPreReq) > 0 {
 				for _, cfg := range test.ConfigMapPreReq {

controllers/mlmd_test.go

@@ -131,8 +131,9 @@ func TestDeployMLMDV2(t *testing.T) {
 	// Construct DSPA Spec with MLMD Enabled
 	dspa := &dspav1alpha1.DataSciencePipelinesApplication{
 		Spec: dspav1alpha1.DSPASpec{
-			DSPVersion: "v2",
-			APIServer:  &dspav1alpha1.APIServer{},
+			DSPVersion:  "v2",
+			PodToPodTLS: boolPtr(false),
+			APIServer:   &dspav1alpha1.APIServer{},
 			MLMD: &dspav1alpha1.MLMD{
 				Deploy: true,
 			},
@@ -315,8 +316,9 @@ func TestDontDeployMLMDV2(t *testing.T) {
 	// Construct DSPA Spec with MLMD Not Enabled
 	dspa := &dspav1alpha1.DataSciencePipelinesApplication{
 		Spec: dspav1alpha1.DSPASpec{
-			DSPVersion: "v2",
-			APIServer:  &dspav1alpha1.APIServer{},
+			DSPVersion:  "v2",
+			PodToPodTLS: boolPtr(false),
+			APIServer:   &dspav1alpha1.APIServer{},
 			MLMD: &dspav1alpha1.MLMD{
 				Deploy: false,
 			},
@@ -448,8 +450,9 @@ func TestDefaultDeployBehaviorMLMDV2(t *testing.T) {
 	// Construct DSPA Spec with MLMD Spec not defined
 	dspa := &dspav1alpha1.DataSciencePipelinesApplication{
 		Spec: dspav1alpha1.DSPASpec{
-			DSPVersion: "v2",
-			APIServer:  &dspav1alpha1.APIServer{},
+			DSPVersion:  "v2",
+			PodToPodTLS: boolPtr(false),
+			APIServer:   &dspav1alpha1.APIServer{},
 			Database: &dspav1alpha1.Database{
 				DisableHealthCheck: false,
 				MariaDB: &dspav1alpha1.MariaDB{
@@ -608,8 +611,9 @@ func TestDeployEnvoyRouteV2(t *testing.T) {
 	// Construct DSPA Spec with MLMD Enabled
 	dspa := &dspav1alpha1.DataSciencePipelinesApplication{
 		Spec: dspav1alpha1.DSPASpec{
-			DSPVersion: "v2",
-			APIServer:  &dspav1alpha1.APIServer{},
+			DSPVersion:  "v2",
+			PodToPodTLS: boolPtr(false),
+			APIServer:   &dspav1alpha1.APIServer{},
 			MLMD: &dspav1alpha1.MLMD{
 				Deploy: true,
 				Envoy: &dspav1alpha1.Envoy{
@@ -750,8 +754,9 @@ func TestDontDeployEnvoyRouteV2(t *testing.T) {
 	// Construct DSPA Spec with MLMD Enabled
 	dspa := &dspav1alpha1.DataSciencePipelinesApplication{
 		Spec: dspav1alpha1.DSPASpec{
-			DSPVersion: "v2",
-			APIServer:  &dspav1alpha1.APIServer{},
+			DSPVersion:  "v2",
+			PodToPodTLS: boolPtr(false),
+			APIServer:   &dspav1alpha1.APIServer{},
 			MLMD: &dspav1alpha1.MLMD{
 				Deploy: true,
 				Envoy: &dspav1alpha1.Envoy{
@@ -811,3 +816,7 @@ func TestDontDeployEnvoyRouteV2(t *testing.T) {
 	assert.False(t, created)
 	assert.Nil(t, err)
 }
+
+func boolPtr(b bool) *bool {
+	return &b
+}

controllers/testdata/declarative/case_6/deploy/02_configmap.yaml

@@ -0,0 +1,36 @@
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: openshift-service-ca.crt
+data:
+  service-ca.crt: |
+    -----BEGIN CERTIFICATE-----
+    MIIFLTCCAxWgAwIBAgIUIvY4jV0212P/ddjuCZhcUyJfoocwDQYJKoZIhvcNAQEL
+    BQAwJjELMAkGA1UEBhMCWFgxFzAVBgNVBAMMDnJoLWRzcC1kZXZzLmlvMB4XDTI0
+    MDMwNTAxMTExN1oXDTM0MDMwMzAxMTExN1owJjELMAkGA1UEBhMCWFgxFzAVBgNV
+    BAMMDnJoLWRzcC1kZXZzLmlvMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKC
+    AgEAnCxNdQ0EUhswfu8/K6icQKc//2xpTvcp9Bn9QZ9UUy3f2UXv5hvd4W2PM/uX
+    FaZGoEzQsYagbjyuHDBxek8YOZvdRx9h7O+LLfN+DXeLbaY6tZ2AxNWwcaAmG0EH
+    nSDVORrk8/aZfFRoxgQigWyuK28YZn2SopjNyvOc8GkNjCFO4y7g4QuzWdGMgMIA
+    +whtt3EuYIwaRourKNFp4oR4InOVdPfuGezxbKRPcFfey1JEdTxGoWnHC+HDDMCf
+    R2vV8hAQB4fdvbOoz3+S7j7d8YiaFBK/P2us6Il5tsUw4kzhD2/OLzyERB7SloZk
+    NiIcSsU0USRGLb4/ybQsxu9UPIXUlKTK70HxIEIdPSPPMM84khIOuax0QXKORFHT
+    Ti9jgEfXjuX/2RPijQoCMDrqRQvDxExnTVMncqud6PeDxOWfvSG4oyZBr4HgNAap
+    wX7FWEY6SOH0e3GrH9ceI3afDO4A4YR+EE426GgHgYe8g4NTfD1D79+txmSY6VvV
+    MBwEvPo1LJVmvz23HBC60+e6Ld3WjwE+viOktt20R5Td3NPj7qcBlMDs105yiz+l
+    Ex1h/WDrAssETrelppg3Xgkkz+iY5RwiUB2BTzeiiDbN+AE6X+S5c61Izc2qAeH2
+    gVrvMDlAK6t6bQ696TzItdAs5SnXauxPjfwmK+F65SYy7z8CAwEAAaNTMFEwHQYD
+    VR0OBBYEFDj7l4fu0pXChZsXU5Cgsmr5TYq7MB8GA1UdIwQYMBaAFDj7l4fu0pXC
+    hZsXU5Cgsmr5TYq7MA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggIB
+    AGr5DblOsH7JE9JM3M4p4eiXD40B/VIACEDMYJvyr6QjmcT8+XnHkiu7OV3OJV/G
+    S4NKhleBhfpaaP2ZPGO/vUTmqXwcK78jl0WEjPrMVjs1eDoSnUNi+KwFTBypIusD
+    gSEnICXa26v1CHCQG0QB+rUrIxJqjtq+bnlw/Ns1wxTYfZBFW1ykCJuMsekPo0pN
+    yTH1eWr0eSVWgljqHKaUjKbRRTSTWvk2Sewaq004W+6QOSb3nb1+GHVMov/Q6vsz
+    j6/3B7+7wybR80UTBI/1DfTlefQaOOgEPBjQZ92NXSxMKe2J7FPD+7NHvwTNzzVD
+    jg3cmW8pbtLEyxa+C+6EN8xnmklVfyzuzVsRJvrZvzYcOgLK2ji35oq9FYGXm0yH
+    HRpQPBFkcgNedD3qrJNYKkIBiAh2SSKKA+J8eP3uD9NUOScgl2aKVz/phU5rSDwt
+    NlhRuX8sS7q4gpL9qk4jWrMb8tNeN5nYRvmJj+Slf9sQSTfvukKo+2X8GpAecQNC
+    z6OeQyN+3C2zm4cLCHHWC0ZR/iHQyHIVKlFXznWe6qA64o4x1A0GurjVMAw0Pe0v
+    WBV3KJBsYK/wijtLeip1oKobU76oE0ML/bnhV10k6usvl4n8cDmcONo5FnGoT8Pk
+    80htx6w5fanMFu4MnoBeyJhhzNfg7ywJcc2VZSM27s2B
+    -----END CERTIFICATE-----

controllers/testdata/declarative/case_6/deploy/03_cr.yaml → controllers/testdata/declarative/case_6/deploy/04_cr.yaml

@@ -12,6 +12,7 @@ metadata:
   name: testdsp6
 spec:
   dspVersion: v2
+  podToPodTLS: true
   apiServer:
     deploy: true
     enableSamplePipeline: false

controllers/testdata/declarative/case_6/expected/created/configmap_dspa_trusted_ca.yaml

@@ -96,3 +96,33 @@ data:
     lsiMw+o9r32W0fzjQRwipTLNM0lEbgWyErsVXFb67vY/rjy9ybuFlKMMOIlZpmut
     wcr1vUGA985Lhv2jire2GTlixOiqZtuQS08lGa7kkcO8sB+7MdRdgEI=
     -----END CERTIFICATE-----
+    -----BEGIN CERTIFICATE-----
+    MIIFLTCCAxWgAwIBAgIUIvY4jV0212P/ddjuCZhcUyJfoocwDQYJKoZIhvcNAQEL
+    BQAwJjELMAkGA1UEBhMCWFgxFzAVBgNVBAMMDnJoLWRzcC1kZXZzLmlvMB4XDTI0
+    MDMwNTAxMTExN1oXDTM0MDMwMzAxMTExN1owJjELMAkGA1UEBhMCWFgxFzAVBgNV
+    BAMMDnJoLWRzcC1kZXZzLmlvMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKC
+    AgEAnCxNdQ0EUhswfu8/K6icQKc//2xpTvcp9Bn9QZ9UUy3f2UXv5hvd4W2PM/uX
+    FaZGoEzQsYagbjyuHDBxek8YOZvdRx9h7O+LLfN+DXeLbaY6tZ2AxNWwcaAmG0EH
+    nSDVORrk8/aZfFRoxgQigWyuK28YZn2SopjNyvOc8GkNjCFO4y7g4QuzWdGMgMIA
+    +whtt3EuYIwaRourKNFp4oR4InOVdPfuGezxbKRPcFfey1JEdTxGoWnHC+HDDMCf
+    R2vV8hAQB4fdvbOoz3+S7j7d8YiaFBK/P2us6Il5tsUw4kzhD2/OLzyERB7SloZk
+    NiIcSsU0USRGLb4/ybQsxu9UPIXUlKTK70HxIEIdPSPPMM84khIOuax0QXKORFHT
+    Ti9jgEfXjuX/2RPijQoCMDrqRQvDxExnTVMncqud6PeDxOWfvSG4oyZBr4HgNAap
+    wX7FWEY6SOH0e3GrH9ceI3afDO4A4YR+EE426GgHgYe8g4NTfD1D79+txmSY6VvV
+    MBwEvPo1LJVmvz23HBC60+e6Ld3WjwE+viOktt20R5Td3NPj7qcBlMDs105yiz+l
+    Ex1h/WDrAssETrelppg3Xgkkz+iY5RwiUB2BTzeiiDbN+AE6X+S5c61Izc2qAeH2
+    gVrvMDlAK6t6bQ696TzItdAs5SnXauxPjfwmK+F65SYy7z8CAwEAAaNTMFEwHQYD
+    VR0OBBYEFDj7l4fu0pXChZsXU5Cgsmr5TYq7MB8GA1UdIwQYMBaAFDj7l4fu0pXC
+    hZsXU5Cgsmr5TYq7MA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggIB
+    AGr5DblOsH7JE9JM3M4p4eiXD40B/VIACEDMYJvyr6QjmcT8+XnHkiu7OV3OJV/G
+    S4NKhleBhfpaaP2ZPGO/vUTmqXwcK78jl0WEjPrMVjs1eDoSnUNi+KwFTBypIusD
+    gSEnICXa26v1CHCQG0QB+rUrIxJqjtq+bnlw/Ns1wxTYfZBFW1ykCJuMsekPo0pN
+    yTH1eWr0eSVWgljqHKaUjKbRRTSTWvk2Sewaq004W+6QOSb3nb1+GHVMov/Q6vsz
+    j6/3B7+7wybR80UTBI/1DfTlefQaOOgEPBjQZ92NXSxMKe2J7FPD+7NHvwTNzzVD
+    jg3cmW8pbtLEyxa+C+6EN8xnmklVfyzuzVsRJvrZvzYcOgLK2ji35oq9FYGXm0yH
+    HRpQPBFkcgNedD3qrJNYKkIBiAh2SSKKA+J8eP3uD9NUOScgl2aKVz/phU5rSDwt
+    NlhRuX8sS7q4gpL9qk4jWrMb8tNeN5nYRvmJj+Slf9sQSTfvukKo+2X8GpAecQNC
+    z6OeQyN+3C2zm4cLCHHWC0ZR/iHQyHIVKlFXznWe6qA64o4x1A0GurjVMAw0Pe0v
+    WBV3KJBsYK/wijtLeip1oKobU76oE0ML/bnhV10k6usvl4n8cDmcONo5FnGoT8Pk
+    80htx6w5fanMFu4MnoBeyJhhzNfg7ywJcc2VZSM27s2B
+    -----END CERTIFICATE-----

controllers/testdata/declarative/case_7/deploy/cr.yaml

@@ -4,6 +4,7 @@ metadata:
   name: testdsp7
 spec:
   dspVersion: v2
+  podToPodTLS: false
   apiServer:
     deploy: true
     image: api-server:test7

controllers/testdata/declarative/case_7/expected/created/apiserver_deployment.yaml

@@ -83,8 +83,6 @@ spec:
               value: ds-pipeline-testdsp7.default.svc.cluster.local
             - name: ML_PIPELINE_SERVICE_PORT_GRPC
               value: "8887"
-            - name: ML_PIPELINE_TLS_ENABLED
-              value: "true"
             - name: SIGNED_URL_EXPIRY_TIME_SECONDS
               value: "15"
             - name: EXECUTIONTYPE
@@ -112,8 +110,6 @@ spec:
             - --config=/config
             - -logtostderr=true
             - --sampleconfig=/config/sample_config.json
-            - --tlsCertPath=/etc/tls/private/tls.crt
-            - --tlsCertKeyPath=/etc/tls/private/tls.key
           ports:
             - containerPort: 8888
               name: http
@@ -142,8 +138,6 @@ spec:
             - name: server-config
               mountPath: /config/config.json
               subPath: config.json
-            - mountPath: /etc/tls/private
-              name: proxy-tls
             - mountPath: /config/sample_config.json
               name: sample-config
               subPath: sample_config.json
@@ -154,8 +148,7 @@ spec:
             - --https-address=:8443
             - --provider=openshift
             - --openshift-service-account=ds-pipeline-testdsp7
-            - --upstream=https://ds-pipeline-testdsp7.default.svc.cluster.local:8888
-            - --upstream-ca=/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt
+            - --upstream=http://localhost:8888
             - --tls-cert=/etc/tls/private/tls.crt
             - --tls-key=/etc/tls/private/tls.key
             - --cookie-secret=SECRET

controllers/testdata/declarative/case_7/expected/created/mlpipelines-ui_deployment.yaml

@@ -49,10 +49,6 @@ spec:
               value: ds-pipeline-testdsp7.default.svc.cluster.local
             - name: ML_PIPELINE_SERVICE_PORT
               value: '8888'
-            - name: ML_PIPELINE_SERVICE_SCHEME
-              value: 'https'
-            - name: NODE_EXTRA_CA_CERTS
-              value: '/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt'
             - name: METADATA_ENVOY_SERVICE_SERVICE_HOST
               value: ds-pipeline-md-testdsp7
             - name: METADATA_ENVOY_SERVICE_SERVICE_PORT

controllers/testdata/declarative/case_7/expected/created/persistence-agent_deployment.yaml

@@ -36,8 +36,6 @@ spec:
               value: ""
             - name: EXECUTIONTYPE
               value: Workflow
-            - name: SSL_CERT_DIR
-              value: "/etc/pki/tls/certs:/var/run/secrets/kubernetes.io/serviceaccount/"
           image: persistenceagent:test7
           imagePullPolicy: IfNotPresent
           name: ds-pipeline-persistenceagent

controllers/testdata/declarative/case_8/deploy/01_configmap.yaml

@@ -0,0 +1,36 @@
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: openshift-service-ca.crt
+data:
+  service-ca.crt: |
+    -----BEGIN CERTIFICATE-----
+    MIIFLTCCAxWgAwIBAgIUIvY4jV0212P/ddjuCZhcUyJfoocwDQYJKoZIhvcNAQEL
+    BQAwJjELMAkGA1UEBhMCWFgxFzAVBgNVBAMMDnJoLWRzcC1kZXZzLmlvMB4XDTI0
+    MDMwNTAxMTExN1oXDTM0MDMwMzAxMTExN1owJjELMAkGA1UEBhMCWFgxFzAVBgNV
+    BAMMDnJoLWRzcC1kZXZzLmlvMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKC
+    AgEAnCxNdQ0EUhswfu8/K6icQKc//2xpTvcp9Bn9QZ9UUy3f2UXv5hvd4W2PM/uX
+    FaZGoEzQsYagbjyuHDBxek8YOZvdRx9h7O+LLfN+DXeLbaY6tZ2AxNWwcaAmG0EH
+    nSDVORrk8/aZfFRoxgQigWyuK28YZn2SopjNyvOc8GkNjCFO4y7g4QuzWdGMgMIA
+    +whtt3EuYIwaRourKNFp4oR4InOVdPfuGezxbKRPcFfey1JEdTxGoWnHC+HDDMCf
+    R2vV8hAQB4fdvbOoz3+S7j7d8YiaFBK/P2us6Il5tsUw4kzhD2/OLzyERB7SloZk
+    NiIcSsU0USRGLb4/ybQsxu9UPIXUlKTK70HxIEIdPSPPMM84khIOuax0QXKORFHT
+    Ti9jgEfXjuX/2RPijQoCMDrqRQvDxExnTVMncqud6PeDxOWfvSG4oyZBr4HgNAap
+    wX7FWEY6SOH0e3GrH9ceI3afDO4A4YR+EE426GgHgYe8g4NTfD1D79+txmSY6VvV
+    MBwEvPo1LJVmvz23HBC60+e6Ld3WjwE+viOktt20R5Td3NPj7qcBlMDs105yiz+l
+    Ex1h/WDrAssETrelppg3Xgkkz+iY5RwiUB2BTzeiiDbN+AE6X+S5c61Izc2qAeH2
+    gVrvMDlAK6t6bQ696TzItdAs5SnXauxPjfwmK+F65SYy7z8CAwEAAaNTMFEwHQYD
+    VR0OBBYEFDj7l4fu0pXChZsXU5Cgsmr5TYq7MB8GA1UdIwQYMBaAFDj7l4fu0pXC
+    hZsXU5Cgsmr5TYq7MA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggIB
+    AGr5DblOsH7JE9JM3M4p4eiXD40B/VIACEDMYJvyr6QjmcT8+XnHkiu7OV3OJV/G
+    S4NKhleBhfpaaP2ZPGO/vUTmqXwcK78jl0WEjPrMVjs1eDoSnUNi+KwFTBypIusD
+    gSEnICXa26v1CHCQG0QB+rUrIxJqjtq+bnlw/Ns1wxTYfZBFW1ykCJuMsekPo0pN
+    yTH1eWr0eSVWgljqHKaUjKbRRTSTWvk2Sewaq004W+6QOSb3nb1+GHVMov/Q6vsz
+    j6/3B7+7wybR80UTBI/1DfTlefQaOOgEPBjQZ92NXSxMKe2J7FPD+7NHvwTNzzVD
+    jg3cmW8pbtLEyxa+C+6EN8xnmklVfyzuzVsRJvrZvzYcOgLK2ji35oq9FYGXm0yH
+    HRpQPBFkcgNedD3qrJNYKkIBiAh2SSKKA+J8eP3uD9NUOScgl2aKVz/phU5rSDwt
+    NlhRuX8sS7q4gpL9qk4jWrMb8tNeN5nYRvmJj+Slf9sQSTfvukKo+2X8GpAecQNC
+    z6OeQyN+3C2zm4cLCHHWC0ZR/iHQyHIVKlFXznWe6qA64o4x1A0GurjVMAw0Pe0v
+    WBV3KJBsYK/wijtLeip1oKobU76oE0ML/bnhV10k6usvl4n8cDmcONo5FnGoT8Pk
+    80htx6w5fanMFu4MnoBeyJhhzNfg7ywJcc2VZSM27s2B
+    -----END CERTIFICATE-----

controllers/testdata/declarative/case_8/deploy/02_cr.yaml

@@ -5,6 +5,7 @@ kind: DataSciencePipelinesApplication
 metadata:
   name: testdsp8
 spec:
+  podToPodTLS: true
   dspVersion: v2
   objectStorage:
     minio:

controllers/testdata/declarative/case_8/expected/created/apiserver_deployment.yaml

@@ -91,10 +91,10 @@ spec:
               value: ds-pipeline-testdsp8.default.svc.cluster.local
             - name: ML_PIPELINE_SERVICE_PORT_GRPC
               value: "8887"
-            - name: ML_PIPELINE_TLS_ENABLED
-              value: "true"
             - name: SIGNED_URL_EXPIRY_TIME_SECONDS
               value: "15"
+            - name: ML_PIPELINE_TLS_ENABLED
+              value: "true"
             - name: EXECUTIONTYPE
               value: Workflow
             - name: DB_DRIVER_NAME