add apiserver tls support

* add openshift ingress cabundle to pa/apiserver/ui * add ui tls enabled kfp server client Signed-off-by: Humair Khan <HumairAK@users.noreply.github.com>
opendatahub-io · Jul 17, 2024 · a290b86 · a290b86
1 parent bd4b501
commit a290b86
Show file tree

Hide file tree

Showing 22 changed files with 629 additions and 91 deletions.
diff --git a/api/v1alpha1/dspipeline_types.go b/api/v1alpha1/dspipeline_types.go
@@ -44,6 +44,12 @@ type DSPASpec struct {
 	// +kubebuilder:validation:Optional
 	// +kubebuilder:default:="v1"
 	DSPVersion string `json:"dspVersion,omitempty"`
+
+	// PodToPodTLS Set to "true" or "false" to enable or disable TLS communication between DSPA components (pods). Defaults to "true" to enable TLS between all pods. Only supported in DSP V2 on OpenShift.
+	// +kubebuilder:default:=true
+	// +kubebuilder:validation:Optional
+	PodToPodTLS *bool `json:"podToPodTLS"`
+
 	// WorkflowController is an argo-specific component that manages a DSPA's Workflow objects and handles the orchestration of them with the central Argo server
 	// +kubebuilder:validation:Optional
 	*WorkflowController `json:"workflowController,omitempty"`

diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
diff --git a/...ses/datasciencepipelinesapplications.opendatahub.io_datasciencepipelinesapplications.yaml b/...ses/datasciencepipelinesapplications.opendatahub.io_datasciencepipelinesapplications.yaml
@@ -754,6 +754,12 @@ spec:
                         type: object
                     type: object
                 type: object
+              podToPodTLS:
+                default: true
+                description: PodToPodTLS Set to "true" or "false" to enable or disable
+                  TLS communication between DSPA components (pods). Defaults to "true"
+                  to enable TLS between all pods. Only supported in DSP V2 on OpenShift.
+                type: boolean
               scheduledWorkflow:
                 default:
                   deploy: true

diff --git a/config/internal/apiserver/default/deployment.yaml.tmpl b/config/internal/apiserver/default/deployment.yaml.tmpl
@@ -118,6 +118,10 @@ spec:
               value: "8887"
             - name: SIGNED_URL_EXPIRY_TIME_SECONDS
               value: "{{.APIServer.ArtifactSignedURLExpirySeconds}}"
+            {{ if .PodToPodTLS }}
+            - name: ML_PIPELINE_TLS_ENABLED
+              value: "true"
+            {{ end }}
             {{ if (eq .DSPVersion "v2") }}
             ## Argo-Specific Env Vars ##
             - name: EXECUTIONTYPE
@@ -181,32 +185,32 @@ spec:
             {{ if .APIServer.EnableSamplePipeline }}
             - --sampleconfig=/config/sample_config.json
             {{ end }}
+            {{ if .PodToPodTLS }}
+            - --tlsCertPath=/etc/tls/private/tls.crt
+            - --tlsCertKeyPath=/etc/tls/private/tls.key
+            {{ end }}
           ports:
             - containerPort: 8888
               name: http
             - containerPort: 8887
               name: grpc
           livenessProbe:
-            exec:
-              command:
-                - wget
-                - -q
-                - -S
-                - -O
-                - '-'
-                - http://localhost:8888/apis/v1beta1/healthz
+            httpGet:
+              path: /apis/v1beta1/healthz
+              port: http
+              {{ if .PodToPodTLS }}
+              scheme: HTTPS
+              {{ end }}
             initialDelaySeconds: 3
             periodSeconds: 5
             timeoutSeconds: 2
           readinessProbe:
-            exec:
-              command:
-                - wget
-                - -q
-                - -S
-                - -O
-                - '-'
-                - http://localhost:8888/apis/v1beta1/healthz
+            httpGet:
+              path: /apis/v1beta1/healthz
+              port: http
+              {{ if .PodToPodTLS }}
+              scheme: HTTPS
+              {{ end }}
             initialDelaySeconds: 3
             periodSeconds: 5
             timeoutSeconds: 2
@@ -233,6 +237,10 @@ spec:
             - name: server-config
               mountPath: /config/config.json
               subPath: {{ .APIServer.CustomServerConfig.Key }}
+          {{ if .PodToPodTLS }}
+            - mountPath: /etc/tls/private
+              name: proxy-tls
+          {{ end }}
           {{ if or .APIServer.EnableSamplePipeline .CustomCABundle }}
             {{ if .APIServer.EnableSamplePipeline }}
             - name: sample-config
@@ -252,7 +260,14 @@ spec:
             - --https-address=:8443
             - --provider=openshift
             - --openshift-service-account={{.APIServerDefaultResourceName}}
+            {{ if .PodToPodTLS }}
+            # because we use certs signed by openshift, these certs are not valid for
+            # localhost, thus we have to use the service name
+            - --upstream=https://{{.APIServerServiceDNSName}}:8888
+            - --upstream-ca=/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt
+            {{ else }}
             - --upstream=http://localhost:8888
+            {{ end }}
             - --tls-cert=/etc/tls/private/tls.crt
             - --tls-key=/etc/tls/private/tls.key
             - --cookie-secret=SECRET

diff --git a/config/internal/mlpipelines-ui/deployment.yaml.tmpl b/config/internal/mlpipelines-ui/deployment.yaml.tmpl
@@ -45,9 +45,15 @@ spec:
             - name: ARGO_ARCHIVE_LOGS
               value: "true"
             - name: ML_PIPELINE_SERVICE_HOST
-              value: ds-pipeline-{{.Name}}
+              value: {{.APIServerServiceDNSName}}
             - name: ML_PIPELINE_SERVICE_PORT
               value: '8888'
+            {{ if .PodToPodTLS }}
+            - name: ML_PIPELINE_SERVICE_SCHEME
+              value: 'https'
+            - name: NODE_EXTRA_CA_CERTS
+              value: '/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt'
+            {{ end }}
             - name: METADATA_ENVOY_SERVICE_SERVICE_HOST
               value: ds-pipeline-md-{{.Name}}
             - name: METADATA_ENVOY_SERVICE_SERVICE_PORT

diff --git a/config/internal/persistence-agent/deployment.yaml.tmpl b/config/internal/persistence-agent/deployment.yaml.tmpl
@@ -40,6 +40,10 @@ spec:
               {{ else }}
               value: PipelineRun
               {{ end }}
+            {{ if .PodToPodTLS }}
+            - name: SSL_CERT_DIR
+              value: "/etc/pki/tls/certs:/var/run/secrets/kubernetes.io/serviceaccount/"
+            {{ end }}
           image: "{{.PersistenceAgent.Image}}"
           imagePullPolicy: IfNotPresent
           name: ds-pipeline-persistenceagent
@@ -48,7 +52,10 @@ spec:
             - "--logtostderr=true"
             - "--ttlSecondsAfterWorkflowFinish=86400"
             - "--numWorker={{.PersistenceAgent.NumWorkers}}"
-            - "--mlPipelineAPIServerName={{.APIServerServiceName}}"
+            - "--mlPipelineAPIServerName={{.APIServerServiceDNSName}}"
+            {{ if .PodToPodTLS }}
+            - "--mlPipelineServiceTLSEnabled=true"
+            {{ end }}
             - "--namespace={{.Namespace}}"
             - "--mlPipelineServiceHttpPort=8888"
             - "--mlPipelineServiceGRPCPort=8887"

diff --git a/controllers/dspipeline_params.go b/controllers/dspipeline_params.go
@@ -86,6 +86,10 @@ type DSPAParams struct {
 	// pipeline pods
 	CustomCABundle *dspa.CABundle
 	DSPONamespace  string
+	// Use to enable tls communication between component pods.
+	PodToPodTLS bool
+
+	APIServerServiceDNSName string
 }
 
 type DBConnection struct {
@@ -578,6 +582,7 @@ func (p *DSPAParams) ExtractParams(ctx context.Context, dsp *dspa.DataSciencePip
 	p.APIServer = dsp.Spec.APIServer.DeepCopy()
 	p.APIServerDefaultResourceName = apiServerDefaultResourceNamePrefix + dsp.Name
 	p.APIServerServiceName = fmt.Sprintf("%s-%s", config.DSPServicePrefix, p.Name)
+	p.APIServerServiceDNSName = fmt.Sprintf("%s.%s.svc.cluster.local", p.APIServerServiceName, p.Namespace)
 	p.ScheduledWorkflow = dsp.Spec.ScheduledWorkflow.DeepCopy()
 	p.ScheduledWorkflowDefaultResourceName = scheduledWorkflowDefaultResourceNamePrefix + dsp.Name
 	p.PersistenceAgent = dsp.Spec.PersistenceAgent.DeepCopy()
@@ -589,8 +594,19 @@ func (p *DSPAParams) ExtractParams(ctx context.Context, dsp *dspa.DataSciencePip
 	p.MLMD = dsp.Spec.MLMD.DeepCopy()
 	p.CustomCABundleRootMountPath = config.CustomCABundleRootMountPath
 	p.PiplinesCABundleMountPath = config.GetCABundleFileMountPath()
+	p.PodToPodTLS = false
 	dspTrustedCAConfigMapKey := config.CustomDSPTrustedCAConfigMapKey
 
+	// PodToPodTLS is only used in v2 dsp
+	if p.UsingV2Pipelines(dsp) {
+		// by default it's enabled when omitted
+		if dsp.Spec.PodToPodTLS == nil {
+			p.PodToPodTLS = true
+		} else {
+			p.PodToPodTLS = *dsp.Spec.PodToPodTLS
+		}
+	}
+
 	log := loggr.WithValues("namespace", p.Namespace).WithValues("dspa_name", p.Name)
 
 	if p.APIServer != nil {

diff --git a/controllers/testdata/declarative/case_2/expected/created/mlpipelines-ui_deployment.yaml b/controllers/testdata/declarative/case_2/expected/created/mlpipelines-ui_deployment.yaml
@@ -46,7 +46,7 @@ spec:
             - name: ARGO_ARCHIVE_LOGS
               value: "true"
             - name: ML_PIPELINE_SERVICE_HOST
-              value: ds-pipeline-testdsp2
+              value: ds-pipeline-testdsp2.default.svc.cluster.local
             - name: ML_PIPELINE_SERVICE_PORT
               value: '8888'
             - name: METADATA_ENVOY_SERVICE_SERVICE_HOST

diff --git a/controllers/testdata/declarative/case_4/expected/created/mlpipelines-ui_deployment.yaml b/controllers/testdata/declarative/case_4/expected/created/mlpipelines-ui_deployment.yaml
@@ -46,7 +46,7 @@ spec:
             - name: ARGO_ARCHIVE_LOGS
               value: "true"
             - name: ML_PIPELINE_SERVICE_HOST
-              value: ds-pipeline-testdsp4
+              value: ds-pipeline-testdsp4.default.svc.cluster.local
             - name: ML_PIPELINE_SERVICE_PORT
               value: '8888'
             - name: METADATA_ENVOY_SERVICE_SERVICE_HOST

diff --git a/controllers/testdata/declarative/case_5/expected/created/mlpipelines-ui_deployment.yaml b/controllers/testdata/declarative/case_5/expected/created/mlpipelines-ui_deployment.yaml
@@ -46,7 +46,7 @@ spec:
             - name: ARGO_ARCHIVE_LOGS
               value: "true"
             - name: ML_PIPELINE_SERVICE_HOST
-              value: ds-pipeline-testdsp5
+              value: ds-pipeline-testdsp5.default.svc.cluster.local
             - name: ML_PIPELINE_SERVICE_PORT
               value: '8888'
             - name: METADATA_ENVOY_SERVICE_SERVICE_HOST

diff --git a/controllers/testdata/declarative/case_6/expected/created/apiserver_deployment.yaml b/controllers/testdata/declarative/case_6/expected/created/apiserver_deployment.yaml
@@ -93,6 +93,8 @@ spec:
               value: "8887"
             - name: SIGNED_URL_EXPIRY_TIME_SECONDS
               value: "20"
+            - name: ML_PIPELINE_TLS_ENABLED
+              value: "true"
             - name: EXECUTIONTYPE
               value: Workflow
             - name: DB_DRIVER_NAME
@@ -117,6 +119,8 @@ spec:
           args:
             - --config=/config
             - -logtostderr=true
+            - --tlsCertPath=/etc/tls/private/tls.crt
+            - --tlsCertKeyPath=/etc/tls/private/tls.key
           ports:
             - containerPort: 8888
               name: http
@@ -125,29 +129,15 @@ spec:
               name: grpc
               protocol: TCP
           livenessProbe:
-            exec:
-              command:
-                - wget
-                - -q
-                - -S
-                - -O
-                - '-'
-                - http://localhost:8888/apis/v1beta1/healthz
-            initialDelaySeconds: 3
-            periodSeconds: 5
-            timeoutSeconds: 2
+            httpGet:
+              path: /apis/v1beta1/healthz
+              port: http
+              scheme: HTTPS
           readinessProbe:
-            exec:
-              command:
-                - wget
-                - -q
-                - -S
-                - -O
-                - '-'
-                - http://localhost:8888/apis/v1beta1/healthz
-            initialDelaySeconds: 3
-            periodSeconds: 5
-            timeoutSeconds: 2
+            httpGet:
+              path: /apis/v1beta1/healthz
+              port: http
+              scheme: HTTPS
           resources:
             requests:
               cpu: 250m
@@ -159,14 +149,17 @@ spec:
             - name: server-config
               mountPath: /config/config.json
               subPath: config.json
+            - mountPath: /etc/tls/private
+              name: proxy-tls
             - name: ca-bundle
               mountPath: /dspa/custom-certs
         - name: oauth-proxy
           args:
             - --https-address=:8443
             - --provider=openshift
             - --openshift-service-account=ds-pipeline-testdsp6
-            - --upstream=http://localhost:8888
+            - --upstream=https://ds-pipeline-testdsp6.default.svc.cluster.local:8888
+            - --upstream-ca=/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt
             - --tls-cert=/etc/tls/private/tls.crt
             - --tls-key=/etc/tls/private/tls.key
             - --cookie-secret=SECRET

diff --git a/controllers/testdata/declarative/case_7/expected/created/apiserver_deployment.yaml b/controllers/testdata/declarative/case_7/expected/created/apiserver_deployment.yaml
@@ -83,6 +83,8 @@ spec:
               value: ds-pipeline-testdsp7.default.svc.cluster.local
             - name: ML_PIPELINE_SERVICE_PORT_GRPC
               value: "8887"
+            - name: ML_PIPELINE_TLS_ENABLED
+              value: "true"
             - name: SIGNED_URL_EXPIRY_TIME_SECONDS
               value: "15"
             - name: EXECUTIONTYPE
@@ -110,6 +112,8 @@ spec:
             - --config=/config
             - -logtostderr=true
             - --sampleconfig=/config/sample_config.json
+            - --tlsCertPath=/etc/tls/private/tls.crt
+            - --tlsCertKeyPath=/etc/tls/private/tls.key
           ports:
             - containerPort: 8888
               name: http
@@ -118,29 +122,15 @@ spec:
               name: grpc
               protocol: TCP
           livenessProbe:
-            exec:
-              command:
-                - wget
-                - -q
-                - -S
-                - -O
-                - '-'
-                - http://localhost:8888/apis/v1beta1/healthz
-            initialDelaySeconds: 3
-            periodSeconds: 5
-            timeoutSeconds: 2
+            httpGet:
+              path: /apis/v1beta1/healthz
+              port: http
+              scheme: HTTPS
           readinessProbe:
-            exec:
-              command:
-                - wget
-                - -q
-                - -S
-                - -O
-                - '-'
-                - http://localhost:8888/apis/v1beta1/healthz
-            initialDelaySeconds: 3
-            periodSeconds: 5
-            timeoutSeconds: 2
+            httpGet:
+              path: /apis/v1beta1/healthz
+              port: http
+              scheme: HTTPS
           resources:
             requests:
               cpu: 1231m
@@ -152,6 +142,8 @@ spec:
             - name: server-config
               mountPath: /config/config.json
               subPath: config.json
+            - mountPath: /etc/tls/private
+              name: proxy-tls
             - mountPath: /config/sample_config.json
               name: sample-config
               subPath: sample_config.json
@@ -162,7 +154,8 @@ spec:
             - --https-address=:8443
             - --provider=openshift
             - --openshift-service-account=ds-pipeline-testdsp7
-            - --upstream=http://localhost:8888
+            - --upstream=https://ds-pipeline-testdsp7.default.svc.cluster.local:8888
+            - --upstream-ca=/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt
             - --tls-cert=/etc/tls/private/tls.crt
             - --tls-key=/etc/tls/private/tls.key
             - --cookie-secret=SECRET

diff --git a/controllers/testdata/declarative/case_7/expected/created/mlpipelines-ui_deployment.yaml b/controllers/testdata/declarative/case_7/expected/created/mlpipelines-ui_deployment.yaml
@@ -46,9 +46,13 @@ spec:
             - name: ARGO_ARCHIVE_LOGS
               value: "true"
             - name: ML_PIPELINE_SERVICE_HOST
-              value: ds-pipeline-testdsp7
+              value: ds-pipeline-testdsp7.default.svc.cluster.local
             - name: ML_PIPELINE_SERVICE_PORT
               value: '8888'
+            - name: ML_PIPELINE_SERVICE_SCHEME
+              value: 'https'
+            - name: NODE_EXTRA_CA_CERTS
+              value: '/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt'
             - name: METADATA_ENVOY_SERVICE_SERVICE_HOST
               value: ds-pipeline-md-testdsp7
             - name: METADATA_ENVOY_SERVICE_SERVICE_PORT