Merge pull request #539 from opendatahub-io/dspv2

DSPv2
opendatahub-io · Jan 18, 2024 · 2523906 · 2523906
2 parents a3ded39 + 9aaf91e
commit 2523906
Show file tree

Hide file tree

Showing 159 changed files with 6,510 additions and 1,407 deletions.
diff --git a/Makefile b/Makefile
@@ -52,6 +52,10 @@ IMG ?= quay.io/opendatahub/data-science-pipelines-operator:main
 ENVTEST_K8S_VERSION = 1.25.0
 # Namespace to deploy the operator
 OPERATOR_NS ?= opendatahub
+# Namespace to deploy v2 infrastructure
+V2INFRA_NS ?= openshift-pipelines
+# Namespace to deploy argo infrastructure
+ARGO_NS ?= argo
 
 # Integration Test ENVvars
 KUBECONFIGPATH ?= $(HOME)/.kube/config
@@ -177,6 +181,18 @@ undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/confi
 	cd config/overlays/make-deploy && $(KUSTOMIZE) edit set namespace ${OPERATOR_NS}
 	$(KUSTOMIZE) build config/overlays/make-deploy | kubectl delete --ignore-not-found=$(ignore-not-found) -f -
 
+.PHONY: argodeploy
+argodeploy: manifests kustomize
+	cd config/overlays/make-argodeploy \
+		&& $(KUSTOMIZE) edit set namespace ${ARGO_NS}
+	$(KUSTOMIZE) build config/overlays/make-argodeploy | kubectl apply -f -
+
+.PHONY: argoundeploy
+argoundeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
+	cd config/overlays/make-argodeploy \
+	    && $(KUSTOMIZE) edit set namespace ${ARGO_NS}
+	$(KUSTOMIZE) build config/overlays/make-argodeploy | kubectl delete --ignore-not-found=$(ignore-not-found) -f -
+
 .PHONY: undeploy-kind
 undeploy-kind: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
 	cd config/overlays/kind-tests \

diff --git a/api/v1alpha1/dspipeline_types.go b/api/v1alpha1/dspipeline_types.go
@@ -40,9 +40,17 @@ type DSPASpec struct {
 	// ObjectStorage specifies Object Store configurations, used for DS Pipelines artifact passing and storage. Specify either the your own External Storage (e.g. AWS S3), or use the default Minio deployment (unsupported, primarily for development, and testing) .
 	// +kubebuilder:validation:Required
 	*ObjectStorage `json:"objectStorage"`
+	*MLMD          `json:"mlmd,omitempty"`
 	// +kubebuilder:validation:Optional
 	// +kubebuilder:default:={deploy: false}
-	*MLMD `json:"mlmd"`
+	*CRDViewer `json:"crdviewer"`
+	// +kubebuilder:validation:Optional
+	// +kubebuilder:default:="v1"
+	DSPVersion string `json:"dspVersion,omitempty"`
+	// WorkflowController is an argo-specific component that manages a DSPA's Workflow objects and handles the orchestration of them with the central Argo server
+	// +kubebuilder:validation:Optional
+	// +kubebuilder:default:={deploy: false}
+	*WorkflowController `json:"workflowController,omitempty"`
 }
 
 type APIServer struct {
@@ -248,7 +256,7 @@ type Minio struct {
 }
 
 type MLMD struct {
-	// Enable DS Pipelines Operator management of MLMD. Setting Deploy to false disables operator reconciliation. Default: false
+	// Enable DS Pipelines Operator management of MLMD. Setting Deploy to false disables operator reconciliation. Default: true
 	// +kubebuilder:default:=false
 	// +kubebuilder:validation:Optional
 	Deploy  bool `json:"deploy"`
@@ -277,6 +285,20 @@ type Writer struct {
 	Image string `json:"image"`
 }
 
+type CRDViewer struct {
+	// +kubebuilder:default:=true
+	// +kubebuilder:validation:Optional
+	Deploy bool   `json:"deploy"`
+	Image  string `json:"image,omitempty"`
+}
+
+type WorkflowController struct {
+	// +kubebuilder:default:=true
+	// +kubebuilder:validation:Optional
+	Deploy bool   `json:"deploy"`
+	Image  string `json:"image,omitempty"`
+}
+
 // ResourceRequirements structures compute resource requirements.
 // Replaces ResourceRequirements from corev1 which also includes optional storage field.
 // We handle storage field separately, and should not include it as a subfield for Resources.
@@ -304,6 +326,7 @@ type ExternalStorage struct {
 
 type S3CredentialSecret struct {
 	// +kubebuilder:validation:Required
+	// Note: In V2 this value needs to be mlpipeline-minio-artifact
 	SecretName string `json:"secretName"`
 	// The "Keys" in the k8sSecret key/value pairs. Not to be confused with the values.
 	AccessKey string `json:"accessKey"`

diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
diff --git a/config/argo/clusterrole.argo-aggregate-to-admin.yaml b/config/argo/clusterrole.argo-aggregate-to-admin.yaml
@@ -0,0 +1,34 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    rbac.authorization.k8s.io/aggregate-to-admin: "true"
+  name: argo-aggregate-to-admin
+rules:
+- apiGroups:
+  - argoproj.io
+  resources:
+  - workflows
+  - workflows/finalizers
+  - workfloweventbindings
+  - workfloweventbindings/finalizers
+  - workflowtemplates
+  - workflowtemplates/finalizers
+  - cronworkflows
+  - cronworkflows/finalizers
+  - clusterworkflowtemplates
+  - clusterworkflowtemplates/finalizers
+  - workflowtasksets
+  - workflowtasksets/finalizers
+  - workflowtaskresults
+  - workflowtaskresults/finalizers
+  verbs:
+  - create
+  - delete
+  - deletecollection
+  - get
+  - list
+  - patch
+  - update
+  - watch
diff --git a/config/argo/clusterrole.argo-aggregate-to-edit.yaml b/config/argo/clusterrole.argo-aggregate-to-edit.yaml
@@ -0,0 +1,32 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    rbac.authorization.k8s.io/aggregate-to-edit: "true"
+  name: argo-aggregate-to-edit
+rules:
+- apiGroups:
+  - argoproj.io
+  resources:
+  - workflows
+  - workflows/finalizers
+  - workfloweventbindings
+  - workfloweventbindings/finalizers
+  - workflowtemplates
+  - workflowtemplates/finalizers
+  - cronworkflows
+  - cronworkflows/finalizers
+  - clusterworkflowtemplates
+  - clusterworkflowtemplates/finalizers
+  - workflowtaskresults
+  - workflowtaskresults/finalizers
+  verbs:
+  - create
+  - delete
+  - deletecollection
+  - get
+  - list
+  - patch
+  - update
+  - watch
diff --git a/config/argo/clusterrole.argo-aggregate-to-view.yaml b/config/argo/clusterrole.argo-aggregate-to-view.yaml
@@ -0,0 +1,27 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    rbac.authorization.k8s.io/aggregate-to-view: "true"
+  name: argo-aggregate-to-view
+rules:
+- apiGroups:
+  - argoproj.io
+  resources:
+  - workflows
+  - workflows/finalizers
+  - workfloweventbindings
+  - workfloweventbindings/finalizers
+  - workflowtemplates
+  - workflowtemplates/finalizers
+  - cronworkflows
+  - cronworkflows/finalizers
+  - clusterworkflowtemplates
+  - clusterworkflowtemplates/finalizers
+  - workflowtaskresults
+  - workflowtaskresults/finalizers
+  verbs:
+  - get
+  - list
+  - watch
diff --git a/config/argo/clusterrole.argo-cluster-role.yaml b/config/argo/clusterrole.argo-cluster-role.yaml
@@ -0,0 +1,106 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: argo-cluster-role
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  - pods/exec
+  verbs:
+  - create
+  - get
+  - list
+  - watch
+  - update
+  - patch
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - get
+  - watch
+  - list
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumeclaims
+  - persistentvolumeclaims/finalizers
+  verbs:
+  - create
+  - update
+  - delete
+  - get
+- apiGroups:
+  - argoproj.io
+  resources:
+  - workflows
+  - workflows/finalizers
+  - workflowtasksets
+  - workflowtasksets/finalizers
+  - workflowartifactgctasks
+  verbs:
+  - get
+  - list
+  - watch
+  - update
+  - patch
+  - delete
+  - create
+- apiGroups:
+  - argoproj.io
+  resources:
+  - workflowtemplates
+  - workflowtemplates/finalizers
+  - clusterworkflowtemplates
+  - clusterworkflowtemplates/finalizers
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - argoproj.io
+  resources:
+  - workflowtaskresults
+  verbs:
+  - list
+  - watch
+  - deletecollection
+- apiGroups:
+  - ""
+  resources:
+  - serviceaccounts
+  verbs:
+  - get
+  - list
+- apiGroups:
+  - argoproj.io
+  resources:
+  - cronworkflows
+  - cronworkflows/finalizers
+  verbs:
+  - get
+  - list
+  - watch
+  - update
+  - patch
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+- apiGroups:
+  - policy
+  resources:
+  - poddisruptionbudgets
+  verbs:
+  - create
+  - get
+  - delete