From a290b860b37c3582342d5b2ede24c86cf45c16c4 Mon Sep 17 00:00:00 2001 From: Humair Khan Date: Tue, 21 May 2024 11:16:24 -0400 Subject: [PATCH] add apiserver tls support * add openshift ingress cabundle to pa/apiserver/ui * add ui tls enabled kfp server client Signed-off-by: Humair Khan --- api/v1alpha1/dspipeline_types.go | 6 + api/v1alpha1/zz_generated.deepcopy.go | 5 + ...b.io_datasciencepipelinesapplications.yaml | 6 + .../apiserver/default/deployment.yaml.tmpl | 47 +++-- .../mlpipelines-ui/deployment.yaml.tmpl | 8 +- .../persistence-agent/deployment.yaml.tmpl | 9 +- controllers/dspipeline_params.go | 16 ++ .../created/mlpipelines-ui_deployment.yaml | 2 +- .../created/mlpipelines-ui_deployment.yaml | 2 +- .../created/mlpipelines-ui_deployment.yaml | 2 +- .../created/apiserver_deployment.yaml | 39 ++-- .../created/apiserver_deployment.yaml | 39 ++-- .../created/mlpipelines-ui_deployment.yaml | 6 +- .../created/persistence-agent_deployment.yaml | 2 + .../created/apiserver_deployment.yaml | 39 ++-- .../testdata/declarative/case_9/config.yaml | 14 ++ .../declarative/case_9/deploy/00_cr.yaml | 22 ++ .../created/apiserver_deployment.yaml | 194 ++++++++++++++++++ .../created/mlpipelines-ui_deployment.yaml | 171 +++++++++++++++ .../created/persistence-agent_deployment.yaml | 89 ++++++++ tests/resources/dspa-external-lite.yaml | 1 + tests/resources/dspa-lite.yaml | 1 + 22 files changed, 629 insertions(+), 91 deletions(-) create mode 100644 controllers/testdata/declarative/case_9/config.yaml create mode 100644 controllers/testdata/declarative/case_9/deploy/00_cr.yaml create mode 100644 controllers/testdata/declarative/case_9/expected/created/apiserver_deployment.yaml create mode 100644 controllers/testdata/declarative/case_9/expected/created/mlpipelines-ui_deployment.yaml create mode 100644 controllers/testdata/declarative/case_9/expected/created/persistence-agent_deployment.yaml diff --git a/api/v1alpha1/dspipeline_types.go b/api/v1alpha1/dspipeline_types.go index a3dd1596..72af4a3f 100644 --- a/api/v1alpha1/dspipeline_types.go +++ b/api/v1alpha1/dspipeline_types.go @@ -44,6 +44,12 @@ type DSPASpec struct { // +kubebuilder:validation:Optional // +kubebuilder:default:="v1" DSPVersion string `json:"dspVersion,omitempty"` + + // PodToPodTLS Set to "true" or "false" to enable or disable TLS communication between DSPA components (pods). Defaults to "true" to enable TLS between all pods. Only supported in DSP V2 on OpenShift. + // +kubebuilder:default:=true + // +kubebuilder:validation:Optional + PodToPodTLS *bool `json:"podToPodTLS"` + // WorkflowController is an argo-specific component that manages a DSPA's Workflow objects and handles the orchestration of them with the central Argo server // +kubebuilder:validation:Optional *WorkflowController `json:"workflowController,omitempty"` diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 83c0261b..29464a33 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -119,6 +119,11 @@ func (in *DSPASpec) DeepCopyInto(out *DSPASpec) { *out = new(MLMD) (*in).DeepCopyInto(*out) } + if in.PodToPodTLS != nil { + in, out := &in.PodToPodTLS, &out.PodToPodTLS + *out = new(bool) + **out = **in + } if in.WorkflowController != nil { in, out := &in.WorkflowController, &out.WorkflowController *out = new(WorkflowController) diff --git a/config/crd/bases/datasciencepipelinesapplications.opendatahub.io_datasciencepipelinesapplications.yaml b/config/crd/bases/datasciencepipelinesapplications.opendatahub.io_datasciencepipelinesapplications.yaml index 517a1090..d28b446b 100644 --- a/config/crd/bases/datasciencepipelinesapplications.opendatahub.io_datasciencepipelinesapplications.yaml +++ b/config/crd/bases/datasciencepipelinesapplications.opendatahub.io_datasciencepipelinesapplications.yaml @@ -754,6 +754,12 @@ spec: type: object type: object type: object + podToPodTLS: + default: true + description: PodToPodTLS Set to "true" or "false" to enable or disable + TLS communication between DSPA components (pods). Defaults to "true" + to enable TLS between all pods. Only supported in DSP V2 on OpenShift. + type: boolean scheduledWorkflow: default: deploy: true diff --git a/config/internal/apiserver/default/deployment.yaml.tmpl b/config/internal/apiserver/default/deployment.yaml.tmpl index 73a609e5..4ec655ec 100644 --- a/config/internal/apiserver/default/deployment.yaml.tmpl +++ b/config/internal/apiserver/default/deployment.yaml.tmpl @@ -118,6 +118,10 @@ spec: value: "8887" - name: SIGNED_URL_EXPIRY_TIME_SECONDS value: "{{.APIServer.ArtifactSignedURLExpirySeconds}}" + {{ if .PodToPodTLS }} + - name: ML_PIPELINE_TLS_ENABLED + value: "true" + {{ end }} {{ if (eq .DSPVersion "v2") }} ## Argo-Specific Env Vars ## - name: EXECUTIONTYPE @@ -181,32 +185,32 @@ spec: {{ if .APIServer.EnableSamplePipeline }} - --sampleconfig=/config/sample_config.json {{ end }} + {{ if .PodToPodTLS }} + - --tlsCertPath=/etc/tls/private/tls.crt + - --tlsCertKeyPath=/etc/tls/private/tls.key + {{ end }} ports: - containerPort: 8888 name: http - containerPort: 8887 name: grpc livenessProbe: - exec: - command: - - wget - - -q - - -S - - -O - - '-' - - http://localhost:8888/apis/v1beta1/healthz + httpGet: + path: /apis/v1beta1/healthz + port: http + {{ if .PodToPodTLS }} + scheme: HTTPS + {{ end }} initialDelaySeconds: 3 periodSeconds: 5 timeoutSeconds: 2 readinessProbe: - exec: - command: - - wget - - -q - - -S - - -O - - '-' - - http://localhost:8888/apis/v1beta1/healthz + httpGet: + path: /apis/v1beta1/healthz + port: http + {{ if .PodToPodTLS }} + scheme: HTTPS + {{ end }} initialDelaySeconds: 3 periodSeconds: 5 timeoutSeconds: 2 @@ -233,6 +237,10 @@ spec: - name: server-config mountPath: /config/config.json subPath: {{ .APIServer.CustomServerConfig.Key }} + {{ if .PodToPodTLS }} + - mountPath: /etc/tls/private + name: proxy-tls + {{ end }} {{ if or .APIServer.EnableSamplePipeline .CustomCABundle }} {{ if .APIServer.EnableSamplePipeline }} - name: sample-config @@ -252,7 +260,14 @@ spec: - --https-address=:8443 - --provider=openshift - --openshift-service-account={{.APIServerDefaultResourceName}} + {{ if .PodToPodTLS }} + # because we use certs signed by openshift, these certs are not valid for + # localhost, thus we have to use the service name + - --upstream=https://{{.APIServerServiceDNSName}}:8888 + - --upstream-ca=/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt + {{ else }} - --upstream=http://localhost:8888 + {{ end }} - --tls-cert=/etc/tls/private/tls.crt - --tls-key=/etc/tls/private/tls.key - --cookie-secret=SECRET diff --git a/config/internal/mlpipelines-ui/deployment.yaml.tmpl b/config/internal/mlpipelines-ui/deployment.yaml.tmpl index 81336d1b..3fd21c3b 100644 --- a/config/internal/mlpipelines-ui/deployment.yaml.tmpl +++ b/config/internal/mlpipelines-ui/deployment.yaml.tmpl @@ -45,9 +45,15 @@ spec: - name: ARGO_ARCHIVE_LOGS value: "true" - name: ML_PIPELINE_SERVICE_HOST - value: ds-pipeline-{{.Name}} + value: {{.APIServerServiceDNSName}} - name: ML_PIPELINE_SERVICE_PORT value: '8888' + {{ if .PodToPodTLS }} + - name: ML_PIPELINE_SERVICE_SCHEME + value: 'https' + - name: NODE_EXTRA_CA_CERTS + value: '/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt' + {{ end }} - name: METADATA_ENVOY_SERVICE_SERVICE_HOST value: ds-pipeline-md-{{.Name}} - name: METADATA_ENVOY_SERVICE_SERVICE_PORT diff --git a/config/internal/persistence-agent/deployment.yaml.tmpl b/config/internal/persistence-agent/deployment.yaml.tmpl index 83180996..705d5620 100644 --- a/config/internal/persistence-agent/deployment.yaml.tmpl +++ b/config/internal/persistence-agent/deployment.yaml.tmpl @@ -40,6 +40,10 @@ spec: {{ else }} value: PipelineRun {{ end }} + {{ if .PodToPodTLS }} + - name: SSL_CERT_DIR + value: "/etc/pki/tls/certs:/var/run/secrets/kubernetes.io/serviceaccount/" + {{ end }} image: "{{.PersistenceAgent.Image}}" imagePullPolicy: IfNotPresent name: ds-pipeline-persistenceagent @@ -48,7 +52,10 @@ spec: - "--logtostderr=true" - "--ttlSecondsAfterWorkflowFinish=86400" - "--numWorker={{.PersistenceAgent.NumWorkers}}" - - "--mlPipelineAPIServerName={{.APIServerServiceName}}" + - "--mlPipelineAPIServerName={{.APIServerServiceDNSName}}" + {{ if .PodToPodTLS }} + - "--mlPipelineServiceTLSEnabled=true" + {{ end }} - "--namespace={{.Namespace}}" - "--mlPipelineServiceHttpPort=8888" - "--mlPipelineServiceGRPCPort=8887" diff --git a/controllers/dspipeline_params.go b/controllers/dspipeline_params.go index f1ee5f57..711f32f7 100644 --- a/controllers/dspipeline_params.go +++ b/controllers/dspipeline_params.go @@ -86,6 +86,10 @@ type DSPAParams struct { // pipeline pods CustomCABundle *dspa.CABundle DSPONamespace string + // Use to enable tls communication between component pods. + PodToPodTLS bool + + APIServerServiceDNSName string } type DBConnection struct { @@ -578,6 +582,7 @@ func (p *DSPAParams) ExtractParams(ctx context.Context, dsp *dspa.DataSciencePip p.APIServer = dsp.Spec.APIServer.DeepCopy() p.APIServerDefaultResourceName = apiServerDefaultResourceNamePrefix + dsp.Name p.APIServerServiceName = fmt.Sprintf("%s-%s", config.DSPServicePrefix, p.Name) + p.APIServerServiceDNSName = fmt.Sprintf("%s.%s.svc.cluster.local", p.APIServerServiceName, p.Namespace) p.ScheduledWorkflow = dsp.Spec.ScheduledWorkflow.DeepCopy() p.ScheduledWorkflowDefaultResourceName = scheduledWorkflowDefaultResourceNamePrefix + dsp.Name p.PersistenceAgent = dsp.Spec.PersistenceAgent.DeepCopy() @@ -589,8 +594,19 @@ func (p *DSPAParams) ExtractParams(ctx context.Context, dsp *dspa.DataSciencePip p.MLMD = dsp.Spec.MLMD.DeepCopy() p.CustomCABundleRootMountPath = config.CustomCABundleRootMountPath p.PiplinesCABundleMountPath = config.GetCABundleFileMountPath() + p.PodToPodTLS = false dspTrustedCAConfigMapKey := config.CustomDSPTrustedCAConfigMapKey + // PodToPodTLS is only used in v2 dsp + if p.UsingV2Pipelines(dsp) { + // by default it's enabled when omitted + if dsp.Spec.PodToPodTLS == nil { + p.PodToPodTLS = true + } else { + p.PodToPodTLS = *dsp.Spec.PodToPodTLS + } + } + log := loggr.WithValues("namespace", p.Namespace).WithValues("dspa_name", p.Name) if p.APIServer != nil { diff --git a/controllers/testdata/declarative/case_2/expected/created/mlpipelines-ui_deployment.yaml b/controllers/testdata/declarative/case_2/expected/created/mlpipelines-ui_deployment.yaml index 6ea29e2c..444851cc 100644 --- a/controllers/testdata/declarative/case_2/expected/created/mlpipelines-ui_deployment.yaml +++ b/controllers/testdata/declarative/case_2/expected/created/mlpipelines-ui_deployment.yaml @@ -46,7 +46,7 @@ spec: - name: ARGO_ARCHIVE_LOGS value: "true" - name: ML_PIPELINE_SERVICE_HOST - value: ds-pipeline-testdsp2 + value: ds-pipeline-testdsp2.default.svc.cluster.local - name: ML_PIPELINE_SERVICE_PORT value: '8888' - name: METADATA_ENVOY_SERVICE_SERVICE_HOST diff --git a/controllers/testdata/declarative/case_4/expected/created/mlpipelines-ui_deployment.yaml b/controllers/testdata/declarative/case_4/expected/created/mlpipelines-ui_deployment.yaml index 2a90d3e0..1b759701 100644 --- a/controllers/testdata/declarative/case_4/expected/created/mlpipelines-ui_deployment.yaml +++ b/controllers/testdata/declarative/case_4/expected/created/mlpipelines-ui_deployment.yaml @@ -46,7 +46,7 @@ spec: - name: ARGO_ARCHIVE_LOGS value: "true" - name: ML_PIPELINE_SERVICE_HOST - value: ds-pipeline-testdsp4 + value: ds-pipeline-testdsp4.default.svc.cluster.local - name: ML_PIPELINE_SERVICE_PORT value: '8888' - name: METADATA_ENVOY_SERVICE_SERVICE_HOST diff --git a/controllers/testdata/declarative/case_5/expected/created/mlpipelines-ui_deployment.yaml b/controllers/testdata/declarative/case_5/expected/created/mlpipelines-ui_deployment.yaml index 30809c05..abec9309 100644 --- a/controllers/testdata/declarative/case_5/expected/created/mlpipelines-ui_deployment.yaml +++ b/controllers/testdata/declarative/case_5/expected/created/mlpipelines-ui_deployment.yaml @@ -46,7 +46,7 @@ spec: - name: ARGO_ARCHIVE_LOGS value: "true" - name: ML_PIPELINE_SERVICE_HOST - value: ds-pipeline-testdsp5 + value: ds-pipeline-testdsp5.default.svc.cluster.local - name: ML_PIPELINE_SERVICE_PORT value: '8888' - name: METADATA_ENVOY_SERVICE_SERVICE_HOST diff --git a/controllers/testdata/declarative/case_6/expected/created/apiserver_deployment.yaml b/controllers/testdata/declarative/case_6/expected/created/apiserver_deployment.yaml index 67ca1d06..03828eb2 100644 --- a/controllers/testdata/declarative/case_6/expected/created/apiserver_deployment.yaml +++ b/controllers/testdata/declarative/case_6/expected/created/apiserver_deployment.yaml @@ -93,6 +93,8 @@ spec: value: "8887" - name: SIGNED_URL_EXPIRY_TIME_SECONDS value: "20" + - name: ML_PIPELINE_TLS_ENABLED + value: "true" - name: EXECUTIONTYPE value: Workflow - name: DB_DRIVER_NAME @@ -117,6 +119,8 @@ spec: args: - --config=/config - -logtostderr=true + - --tlsCertPath=/etc/tls/private/tls.crt + - --tlsCertKeyPath=/etc/tls/private/tls.key ports: - containerPort: 8888 name: http @@ -125,29 +129,15 @@ spec: name: grpc protocol: TCP livenessProbe: - exec: - command: - - wget - - -q - - -S - - -O - - '-' - - http://localhost:8888/apis/v1beta1/healthz - initialDelaySeconds: 3 - periodSeconds: 5 - timeoutSeconds: 2 + httpGet: + path: /apis/v1beta1/healthz + port: http + scheme: HTTPS readinessProbe: - exec: - command: - - wget - - -q - - -S - - -O - - '-' - - http://localhost:8888/apis/v1beta1/healthz - initialDelaySeconds: 3 - periodSeconds: 5 - timeoutSeconds: 2 + httpGet: + path: /apis/v1beta1/healthz + port: http + scheme: HTTPS resources: requests: cpu: 250m @@ -159,6 +149,8 @@ spec: - name: server-config mountPath: /config/config.json subPath: config.json + - mountPath: /etc/tls/private + name: proxy-tls - name: ca-bundle mountPath: /dspa/custom-certs - name: oauth-proxy @@ -166,7 +158,8 @@ spec: - --https-address=:8443 - --provider=openshift - --openshift-service-account=ds-pipeline-testdsp6 - - --upstream=http://localhost:8888 + - --upstream=https://ds-pipeline-testdsp6.default.svc.cluster.local:8888 + - --upstream-ca=/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt - --tls-cert=/etc/tls/private/tls.crt - --tls-key=/etc/tls/private/tls.key - --cookie-secret=SECRET diff --git a/controllers/testdata/declarative/case_7/expected/created/apiserver_deployment.yaml b/controllers/testdata/declarative/case_7/expected/created/apiserver_deployment.yaml index b3f252d9..06ee27d6 100644 --- a/controllers/testdata/declarative/case_7/expected/created/apiserver_deployment.yaml +++ b/controllers/testdata/declarative/case_7/expected/created/apiserver_deployment.yaml @@ -83,6 +83,8 @@ spec: value: ds-pipeline-testdsp7.default.svc.cluster.local - name: ML_PIPELINE_SERVICE_PORT_GRPC value: "8887" + - name: ML_PIPELINE_TLS_ENABLED + value: "true" - name: SIGNED_URL_EXPIRY_TIME_SECONDS value: "15" - name: EXECUTIONTYPE @@ -110,6 +112,8 @@ spec: - --config=/config - -logtostderr=true - --sampleconfig=/config/sample_config.json + - --tlsCertPath=/etc/tls/private/tls.crt + - --tlsCertKeyPath=/etc/tls/private/tls.key ports: - containerPort: 8888 name: http @@ -118,29 +122,15 @@ spec: name: grpc protocol: TCP livenessProbe: - exec: - command: - - wget - - -q - - -S - - -O - - '-' - - http://localhost:8888/apis/v1beta1/healthz - initialDelaySeconds: 3 - periodSeconds: 5 - timeoutSeconds: 2 + httpGet: + path: /apis/v1beta1/healthz + port: http + scheme: HTTPS readinessProbe: - exec: - command: - - wget - - -q - - -S - - -O - - '-' - - http://localhost:8888/apis/v1beta1/healthz - initialDelaySeconds: 3 - periodSeconds: 5 - timeoutSeconds: 2 + httpGet: + path: /apis/v1beta1/healthz + port: http + scheme: HTTPS resources: requests: cpu: 1231m @@ -152,6 +142,8 @@ spec: - name: server-config mountPath: /config/config.json subPath: config.json + - mountPath: /etc/tls/private + name: proxy-tls - mountPath: /config/sample_config.json name: sample-config subPath: sample_config.json @@ -162,7 +154,8 @@ spec: - --https-address=:8443 - --provider=openshift - --openshift-service-account=ds-pipeline-testdsp7 - - --upstream=http://localhost:8888 + - --upstream=https://ds-pipeline-testdsp7.default.svc.cluster.local:8888 + - --upstream-ca=/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt - --tls-cert=/etc/tls/private/tls.crt - --tls-key=/etc/tls/private/tls.key - --cookie-secret=SECRET diff --git a/controllers/testdata/declarative/case_7/expected/created/mlpipelines-ui_deployment.yaml b/controllers/testdata/declarative/case_7/expected/created/mlpipelines-ui_deployment.yaml index eac54ebc..56028396 100644 --- a/controllers/testdata/declarative/case_7/expected/created/mlpipelines-ui_deployment.yaml +++ b/controllers/testdata/declarative/case_7/expected/created/mlpipelines-ui_deployment.yaml @@ -46,9 +46,13 @@ spec: - name: ARGO_ARCHIVE_LOGS value: "true" - name: ML_PIPELINE_SERVICE_HOST - value: ds-pipeline-testdsp7 + value: ds-pipeline-testdsp7.default.svc.cluster.local - name: ML_PIPELINE_SERVICE_PORT value: '8888' + - name: ML_PIPELINE_SERVICE_SCHEME + value: 'https' + - name: NODE_EXTRA_CA_CERTS + value: '/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt' - name: METADATA_ENVOY_SERVICE_SERVICE_HOST value: ds-pipeline-md-testdsp7 - name: METADATA_ENVOY_SERVICE_SERVICE_PORT diff --git a/controllers/testdata/declarative/case_7/expected/created/persistence-agent_deployment.yaml b/controllers/testdata/declarative/case_7/expected/created/persistence-agent_deployment.yaml index abcb70d3..a1995284 100644 --- a/controllers/testdata/declarative/case_7/expected/created/persistence-agent_deployment.yaml +++ b/controllers/testdata/declarative/case_7/expected/created/persistence-agent_deployment.yaml @@ -36,6 +36,8 @@ spec: value: "" - name: EXECUTIONTYPE value: Workflow + - name: SSL_CERT_DIR + value: "/etc/pki/tls/certs:/var/run/secrets/kubernetes.io/serviceaccount/" image: persistenceagent:test7 imagePullPolicy: IfNotPresent name: ds-pipeline-persistenceagent diff --git a/controllers/testdata/declarative/case_8/expected/created/apiserver_deployment.yaml b/controllers/testdata/declarative/case_8/expected/created/apiserver_deployment.yaml index ffe08b4f..836eed7a 100644 --- a/controllers/testdata/declarative/case_8/expected/created/apiserver_deployment.yaml +++ b/controllers/testdata/declarative/case_8/expected/created/apiserver_deployment.yaml @@ -91,6 +91,8 @@ spec: value: ds-pipeline-testdsp8.default.svc.cluster.local - name: ML_PIPELINE_SERVICE_PORT_GRPC value: "8887" + - name: ML_PIPELINE_TLS_ENABLED + value: "true" - name: SIGNED_URL_EXPIRY_TIME_SECONDS value: "15" - name: EXECUTIONTYPE @@ -117,6 +119,8 @@ spec: args: - --config=/config - -logtostderr=true + - --tlsCertPath=/etc/tls/private/tls.crt + - --tlsCertKeyPath=/etc/tls/private/tls.key ports: - containerPort: 8888 name: http @@ -125,29 +129,15 @@ spec: name: grpc protocol: TCP livenessProbe: - exec: - command: - - wget - - -q - - -S - - -O - - '-' - - http://localhost:8888/apis/v1beta1/healthz - initialDelaySeconds: 3 - periodSeconds: 5 - timeoutSeconds: 2 + httpGet: + path: /apis/v1beta1/healthz + port: http + scheme: HTTPS readinessProbe: - exec: - command: - - wget - - -q - - -S - - -O - - '-' - - http://localhost:8888/apis/v1beta1/healthz - initialDelaySeconds: 3 - periodSeconds: 5 - timeoutSeconds: 2 + httpGet: + path: /apis/v1beta1/healthz + port: http + scheme: HTTPS resources: requests: cpu: 250m @@ -159,6 +149,8 @@ spec: - name: server-config mountPath: /config/config.json subPath: config.json + - mountPath: /etc/tls/private + name: proxy-tls - name: ca-bundle mountPath: /dsp-custom-certs - name: oauth-proxy @@ -166,7 +158,8 @@ spec: - --https-address=:8443 - --provider=openshift - --openshift-service-account=ds-pipeline-testdsp8 - - --upstream=http://localhost:8888 + - --upstream=https://ds-pipeline-testdsp8.default.svc.cluster.local:8888 + - --upstream-ca=/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt - --tls-cert=/etc/tls/private/tls.crt - --tls-key=/etc/tls/private/tls.key - --cookie-secret=SECRET diff --git a/controllers/testdata/declarative/case_9/config.yaml b/controllers/testdata/declarative/case_9/config.yaml new file mode 100644 index 00000000..dbcd4d0d --- /dev/null +++ b/controllers/testdata/declarative/case_9/config.yaml @@ -0,0 +1,14 @@ +Images: + MlPipelineUI: frontend:test9 + MariaDB: mariadb:test9 + Minio: minio:test9 + OAuthProxy: oauth-proxy:test9 +ImagesV2: + Argo: + ApiServer: api-server:test9 + ArgoLauncherImage: argolauncherimage:test9 + ArgoDriverImage: argodriverimage:test9 + PersistentAgent: persistenceagent:test9 +DSPO: + ApiServer: + IncludeOwnerReference: false diff --git a/controllers/testdata/declarative/case_9/deploy/00_cr.yaml b/controllers/testdata/declarative/case_9/deploy/00_cr.yaml new file mode 100644 index 00000000..612fb3dc --- /dev/null +++ b/controllers/testdata/declarative/case_9/deploy/00_cr.yaml @@ -0,0 +1,22 @@ +# Test: +# podToPodTLS = false, should disable any tls configs for apiserver, pa, and kfp ui +apiVersion: datasciencepipelinesapplications.opendatahub.io/v1alpha1 +kind: DataSciencePipelinesApplication +metadata: + name: testdsp9 +spec: + dspVersion: v2 + podToPodTLS: false + objectStorage: + minio: + image: minio:test9 + database: + mariaDB: + deploy: true + mlpipelineUI: + deploy: true + image: frontend:test9 + apiServer: + deploy: true + enableOauth: true + enableSamplePipeline: false diff --git a/controllers/testdata/declarative/case_9/expected/created/apiserver_deployment.yaml b/controllers/testdata/declarative/case_9/expected/created/apiserver_deployment.yaml new file mode 100644 index 00000000..ba534e8e --- /dev/null +++ b/controllers/testdata/declarative/case_9/expected/created/apiserver_deployment.yaml @@ -0,0 +1,194 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ds-pipeline-testdsp9 + namespace: default + labels: + app: ds-pipeline-testdsp9 + component: data-science-pipelines + dspa: testdsp9 +spec: + selector: + matchLabels: + app: ds-pipeline-testdsp9 + component: data-science-pipelines + dspa: testdsp9 + template: + metadata: + labels: + app: ds-pipeline-testdsp9 + component: data-science-pipelines + dspa: testdsp9 + spec: + containers: + - env: + - name: POD_NAMESPACE + value: "default" + - name: DBCONFIG_USER + value: "mlpipeline" + - name: DBCONFIG_PASSWORD + valueFrom: + secretKeyRef: + key: "password" + name: "ds-pipeline-db-testdsp9" + - name: DBCONFIG_DBNAME + value: "mlpipeline" + - name: DBCONFIG_HOST + value: "mariadb-testdsp9.default.svc.cluster.local" + - name: DBCONFIG_PORT + value: "3306" + - name: AUTO_UPDATE_PIPELINE_DEFAULT_VERSION + value: "true" + - name: DBCONFIG_CONMAXLIFETIMESEC + value: "120" + - name: ML_PIPELINE_VISUALIZATIONSERVER_SERVICE_HOST + value: "ds-pipeline-visualizationserver" + - name: ML_PIPELINE_VISUALIZATIONSERVER_SERVICE_PORT + value: "8888" + - name: OBJECTSTORECONFIG_CREDENTIALSSECRET + value: "ds-pipeline-s3-testdsp9" + - name: OBJECTSTORECONFIG_CREDENTIALSACCESSKEYKEY + value: "accesskey" + - name: OBJECTSTORECONFIG_CREDENTIALSSECRETKEYKEY + value: "secretkey" + - name: DEFAULTPIPELINERUNNERSERVICEACCOUNT + value: "pipeline-runner-testdsp9" + - name: OBJECTSTORECONFIG_BUCKETNAME + value: "mlpipeline" + - name: OBJECTSTORECONFIG_ACCESSKEY + valueFrom: + secretKeyRef: + key: "accesskey" + name: "ds-pipeline-s3-testdsp9" + - name: OBJECTSTORECONFIG_SECRETACCESSKEY + valueFrom: + secretKeyRef: + key: "secretkey" + name: "ds-pipeline-s3-testdsp9" + - name: OBJECTSTORECONFIG_SECURE + value: "false" + - name: MINIO_SERVICE_SERVICE_HOST + value: "minio-testdsp9.default.svc.cluster.local" + - name: MINIO_SERVICE_SERVICE_PORT + value: "9000" + - name: V2_LAUNCHER_IMAGE + value: "argolauncherimage:test9" + - name: V2_DRIVER_IMAGE + value: "argodriverimage:test9" + - name: METADATA_GRPC_SERVICE_SERVICE_HOST + value: "ds-pipeline-metadata-grpc-testdsp9.default.svc.cluster.local" + - name: METADATA_GRPC_SERVICE_SERVICE_PORT + value: "8080" + - name: ML_PIPELINE_SERVICE_HOST + value: ds-pipeline-testdsp9.default.svc.cluster.local + - name: ML_PIPELINE_SERVICE_PORT_GRPC + value: "8887" + - name: EXECUTIONTYPE + value: Workflow + - name: DB_DRIVER_NAME + value: mysql + - name: DBCONFIG_MYSQLCONFIG_USER + value: mlpipeline + - name: DBCONFIG_MYSQLCONFIG_PASSWORD + valueFrom: + secretKeyRef: + key: "password" + name: "ds-pipeline-db-testdsp9" + - name: DBCONFIG_MYSQLCONFIG_DBNAME + value: "mlpipeline" + - name: DBCONFIG_MYSQLCONFIG_HOST + value: "mariadb-testdsp9.default.svc.cluster.local" + - name: DBCONFIG_MYSQLCONFIG_PORT + value: "3306" + image: api-server:test9 + imagePullPolicy: Always + name: ds-pipeline-api-server + command: ['/bin/apiserver'] + args: + - --config=/config + - -logtostderr=true + ports: + - containerPort: 8888 + name: http + protocol: TCP + - containerPort: 8887 + name: grpc + protocol: TCP + livenessProbe: + httpGet: + path: /apis/v1beta1/healthz + port: http + scheme: HTTPS + readinessProbe: + httpGet: + path: /apis/v1beta1/healthz + port: http + scheme: HTTPS + resources: + requests: + cpu: 250m + memory: 500Mi + limits: + cpu: 500m + memory: 1Gi + volumeMounts: + - name: server-config + mountPath: /config/config.json + subPath: config.json + - name: oauth-proxy + args: + - --https-address=:8443 + - --provider=openshift + - --openshift-service-account=ds-pipeline-testdsp9 + - --upstream=http://localhost:8888 + - --tls-cert=/etc/tls/private/tls.crt + - --tls-key=/etc/tls/private/tls.key + - --cookie-secret=SECRET + - '--openshift-delegate-urls={"/": {"group":"route.openshift.io","resource":"routes","verb":"get","name":"ds-pipeline-testdsp9","namespace":"default"}}' + - '--openshift-sar={"namespace":"default","resource":"routes","resourceName":"ds-pipeline-testdsp9","verb":"get","resourceAPIGroup":"route.openshift.io"}' + - --skip-auth-regex='(^/metrics|^/apis/v1beta1/healthz)' + image: oauth-proxy:test9 + ports: + - containerPort: 8443 + name: oauth + protocol: TCP + livenessProbe: + httpGet: + path: /oauth/healthz + port: oauth + scheme: HTTPS + initialDelaySeconds: 30 + timeoutSeconds: 1 + periodSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /oauth/healthz + port: oauth + scheme: HTTPS + initialDelaySeconds: 5 + timeoutSeconds: 1 + periodSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 100m + memory: 256Mi + volumeMounts: + - mountPath: /etc/tls/private + name: proxy-tls + volumes: + - name: proxy-tls + secret: + secretName: ds-pipelines-proxy-tls-testdsp9 + defaultMode: 420 + - name: server-config + configMap: + name: ds-pipeline-server-config-testdsp9 + defaultMode: 420 + serviceAccountName: ds-pipeline-testdsp9 diff --git a/controllers/testdata/declarative/case_9/expected/created/mlpipelines-ui_deployment.yaml b/controllers/testdata/declarative/case_9/expected/created/mlpipelines-ui_deployment.yaml new file mode 100644 index 00000000..a34edeb0 --- /dev/null +++ b/controllers/testdata/declarative/case_9/expected/created/mlpipelines-ui_deployment.yaml @@ -0,0 +1,171 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ds-pipeline-ui-testdsp9 + namespace: default + labels: + app: ds-pipeline-ui-testdsp9 + component: data-science-pipelines + dspa: testdsp9 +spec: + selector: + matchLabels: + app: ds-pipeline-ui-testdsp9 + component: data-science-pipelines + dspa: testdsp9 + template: + metadata: + annotations: + cluster-autoscaler.kubernetes.io/safe-to-evict: "true" + labels: + app: ds-pipeline-ui-testdsp9 + component: data-science-pipelines + dspa: testdsp9 + spec: + containers: + - env: + - name: VIEWER_TENSORBOARD_POD_TEMPLATE_SPEC_PATH + value: /etc/config/viewer-pod-template.json + - name: MINIO_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: MINIO_ACCESS_KEY + valueFrom: + secretKeyRef: + key: "accesskey" + name: "ds-pipeline-s3-testdsp9" + - name: MINIO_SECRET_KEY + valueFrom: + secretKeyRef: + key: "secretkey" + name: "ds-pipeline-s3-testdsp9" + - name: ALLOW_CUSTOM_VISUALIZATIONS + value: "true" + - name: ARGO_ARCHIVE_LOGS + value: "true" + - name: ML_PIPELINE_SERVICE_HOST + value: ds-pipeline-testdsp9.default.svc.cluster.local + - name: ML_PIPELINE_SERVICE_PORT + value: '8888' + - name: METADATA_ENVOY_SERVICE_SERVICE_HOST + value: ds-pipeline-md-testdsp9 + - name: METADATA_ENVOY_SERVICE_SERVICE_PORT + value: "9090" + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + key: "accesskey" + name: "ds-pipeline-s3-testdsp9" + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + key: "secretkey" + name: "ds-pipeline-s3-testdsp9" + - name: AWS_REGION + value: "minio" + - name: AWS_S3_ENDPOINT + value: "minio-testdsp9.default.svc.cluster.local" + - name: AWS_SSL + value: "false" + - name: DISABLE_GKE_METADATA + value: 'true' + image: frontend:test9 + imagePullPolicy: IfNotPresent + livenessProbe: + exec: + command: + - wget + - -q + - -S + - -O + - '-' + - http://localhost:3000/apis/v1beta1/healthz + initialDelaySeconds: 3 + periodSeconds: 5 + timeoutSeconds: 2 + name: ds-pipeline-ui + ports: + - containerPort: 3000 + protocol: TCP + readinessProbe: + exec: + command: + - wget + - -q + - -S + - -O + - '-' + - http://localhost:3000/apis/v1beta1/healthz + initialDelaySeconds: 3 + periodSeconds: 5 + timeoutSeconds: 2 + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 100m + memory: 256Mi + volumeMounts: + - mountPath: /etc/config + name: config-volume + readOnly: true + - name: oauth-proxy + args: + - --https-address=:8443 + - --provider=openshift + - --openshift-service-account=ds-pipeline-ui-testdsp9 + - --upstream=http://localhost:3000 + - --tls-cert=/etc/tls/private/tls.crt + - --tls-key=/etc/tls/private/tls.key + - --cookie-secret=SECRET + - '--openshift-delegate-urls={"/": {"group":"route.openshift.io","resource":"routes","verb":"get","name":"ds-pipeline-ui-testdsp9","namespace":"default"}}' + - '--openshift-sar={"namespace":"default","resource":"routes","resourceName":"ds-pipeline-ui-testdsp9","verb":"get","resourceAPIGroup":"route.openshift.io"}' + - --skip-auth-regex='(^/metrics|^/apis/v1beta1/healthz)' + image: oauth-proxy:test9 + ports: + - containerPort: 8443 + name: https + protocol: TCP + livenessProbe: + httpGet: + path: /oauth/healthz + port: 8443 + scheme: HTTPS + initialDelaySeconds: 30 + timeoutSeconds: 1 + periodSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /oauth/healthz + port: 8443 + scheme: HTTPS + initialDelaySeconds: 5 + timeoutSeconds: 1 + periodSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + resources: + limits: + cpu: 100m + memory: 256Mi + requests: + cpu: 100m + memory: 256Mi + volumeMounts: + - mountPath: /etc/tls/private + name: proxy-tls + serviceAccountName: ds-pipeline-ui-testdsp9 + volumes: + - configMap: + name: ds-pipeline-ui-configmap-testdsp9 + defaultMode: 420 + name: config-volume + - name: proxy-tls + secret: + secretName: ds-pipelines-ui-proxy-tls-testdsp9 + defaultMode: 420 diff --git a/controllers/testdata/declarative/case_9/expected/created/persistence-agent_deployment.yaml b/controllers/testdata/declarative/case_9/expected/created/persistence-agent_deployment.yaml new file mode 100644 index 00000000..46d622e3 --- /dev/null +++ b/controllers/testdata/declarative/case_9/expected/created/persistence-agent_deployment.yaml @@ -0,0 +1,89 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ds-pipeline-persistenceagent-testdsp9 + namespace: default + labels: + app: ds-pipeline-persistenceagent-testdsp9 + component: data-science-pipelines + dspa: testdsp9 +spec: + selector: + matchLabels: + app: ds-pipeline-persistenceagent-testdsp9 + component: data-science-pipelines + dspa: testdsp9 + template: + metadata: + annotations: + cluster-autoscaler.kubernetes.io/safe-to-evict: "true" + labels: + app: ds-pipeline-persistenceagent-testdsp9 + component: data-science-pipelines + dspa: testdsp9 + spec: + containers: + - env: + - name: NAMESPACE + value: "default" + - name: TTL_SECONDS_AFTER_WORKFLOW_FINISH + value: "86400" + - name: NUM_WORKERS + value: "2" + - name: KUBEFLOW_USERID_HEADER + value: kubeflow-userid + - name: KUBEFLOW_USERID_PREFIX + value: "" + - name: EXECUTIONTYPE + value: Workflow + image: persistenceagent:test9 + imagePullPolicy: IfNotPresent + name: ds-pipeline-persistenceagent + command: + - persistence_agent + - "--logtostderr=true" + - "--ttlSecondsAfterWorkflowFinish=86400" + - "--numWorker=5" + - "--mlPipelineAPIServerName=ds-pipeline-testdsp9" + - "--namespace=testdsp9" + - "--mlPipelineServiceHttpPort=8888" + - "--mlPipelineServiceGRPCPort=8887" + livenessProbe: + exec: + command: + - test + - -x + - persistence_agent + initialDelaySeconds: 30 + periodSeconds: 5 + timeoutSeconds: 2 + readinessProbe: + exec: + command: + - test + - -x + - persistence_agent + initialDelaySeconds: 3 + periodSeconds: 5 + timeoutSeconds: 2 + resources: + requests: + cpu: 120m + memory: 500Mi + limits: + cpu: 250m + memory: 1Gi + volumeMounts: + - mountPath: /var/run/secrets/kubeflow/tokens/persistenceagent-sa-token + name: persistenceagent-sa-token + subPath: ds-pipeline-persistenceagent-testdsp9-token + serviceAccountName: ds-pipeline-persistenceagent-testdsp9 + volumes: + - name: persistenceagent-sa-token + projected: + sources: + - serviceAccountToken: + audience: pipelines.kubeflow.org + expirationSeconds: 3600 + path: ds-pipeline-persistenceagent-testdsp9-token + defaultMode: 420 diff --git a/tests/resources/dspa-external-lite.yaml b/tests/resources/dspa-external-lite.yaml index a86e2a7d..e6ca8c71 100644 --- a/tests/resources/dspa-external-lite.yaml +++ b/tests/resources/dspa-external-lite.yaml @@ -4,6 +4,7 @@ metadata: name: dspa-ext spec: dspVersion: v2 + podToPodTLS: false apiServer: deploy: true enableOauth: false diff --git a/tests/resources/dspa-lite.yaml b/tests/resources/dspa-lite.yaml index 91faa429..225eef9d 100644 --- a/tests/resources/dspa-lite.yaml +++ b/tests/resources/dspa-lite.yaml @@ -4,6 +4,7 @@ metadata: name: test-dspa spec: dspVersion: v2 + podToPodTLS: false apiServer: deploy: true enableOauth: false