csi-wrapper: add example for AWS EBS CSI driver

and how to add the peer-pods CSI wrapper to it Fixes: #1363 Signed-off-by: Snir Sheriber <ssheribe@redhat.com>
confidential-containers · Sep 27, 2023 · dcb2cb7 · dcb2cb7
1 parent c87931a
commit dcb2cb7
Show file tree

Hide file tree

Showing 10 changed files with 521 additions and 0 deletions.
diff --git a/volumes/csi-wrapper/README.md b/volumes/csi-wrapper/README.md
@@ -11,3 +11,4 @@ CSI Wrapper for Peer Pod Storage
 
 * [Azure](examples/azure/README.md)
 * [IBM Cloud](examples/ibm/README.md)
+* [AWS](examples/aws/README.md)
diff --git a/volumes/csi-wrapper/examples/aws/README.md b/volumes/csi-wrapper/examples/aws/README.md
@@ -0,0 +1,76 @@
+# AWS EBS CSI Wrapper for Peer Pod Storage
+
+## Prerequisites
+
+* Running Kubernetes cluster (Version >= 1.20) on AWS
+
+* Peer-Pods is [deployed](../../../../aws/README.md)
+
+## AWS EBS CSI Driver Installation
+
+**NOTE:** the following is just a basic example, follow official [installation instructions](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/blob/master/docs/install.md) for advanced configuration.
+
+1. Create IAM Policy
+```
+aws iam create-policy \
+	--policy-name EBS_Policy \
+	--policy-document file://example-iam-policy.json
+```
+2.  Grant the driver IAM permissions:
+```
+kubectl create secret generic aws-secret \
+    --namespace kube-system \
+    --from-literal "key_id=${AWS_ACCESS_KEY_ID}" \
+    --from-literal "access_key=${AWS_SECRET_ACCESS_KEY}"
+```
+3. Deploy the driver:
+```
+kubectl apply -k "github.com/kubernetes-sigs/aws-ebs-csi-driver/deploy/kubernetes/overlays/stable/?ref=release-1.21"
+```
+4. Verify the pods are running:
+```
+kubectl get pods -n kube-system -l app.kubernetes.io/name=aws-ebs-csi-driver
+```
+
+## Apply the PeerPods CSI wrapper
+
+1. Create the PeerpodVolume CRD object
+```
+kubectl apply -f ../../crd/peerpodvolume.yaml
+```
+2. Apply RBAC roles to permit the wrapper to execute the required operations
+```
+kubectl apply -f rbac-ebs-csi-wrapper-runner.yaml
+kubectl apply -f rbac-ebs-csi-wrapper-podvm.yaml
+```
+3. Patch the EBS CSI Driver:
+```
+kubectl patch deploy ebs-csi-controller -n kube-system --patch-file patch-controller.yaml
+kubectl -n kube-system delete replicaset -l app=ebs-csi-controller
+kubectl patch ds ebs-csi-node -n kube-system --patch-file patch-node.yaml
+```
+4. Verify the pods are running (each pod should contain an additional container):
+```
+kubectl get pods -n kube-system -l app.kubernetes.io/name=aws-ebs-csi-driver
+```
+
+## Example Workload With Provisioned Volume
+
+This is based on the Dynamic Volume Provisioning [example](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/tree/master/examples/kubernetes/dynamic-provisioning)
+
+1. Deploy example pod on your cluster along with the StorageClass and PersistentVolumeClaim:
+```
+ kubectl apply -f dynamic-provisioning/
+```
+2. Validate the PersistentVolumeClaim is bound to your PersistentVolume:
+```
+kubectl get pvc ebs-claim
+```
+3. Once the pod is running you can validate some date (timestamps) has been written to the dynamically provisioned volume:
+```
+kubectl exec app -- cat /data/out.txt
+```
+4. Cleanup resources:
+```
+kubectl delete -f dynamic-provisioning/
+```
diff --git a/volumes/csi-wrapper/examples/aws/dynamic-provisioning/claim.yaml b/volumes/csi-wrapper/examples/aws/dynamic-provisioning/claim.yaml
@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: ebs-claim
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: ebs-sc
+  resources:
+    requests:
+      storage: 4Gi
diff --git a/volumes/csi-wrapper/examples/aws/dynamic-provisioning/pod.yaml b/volumes/csi-wrapper/examples/aws/dynamic-provisioning/pod.yaml
@@ -0,0 +1,89 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: app
+spec:
+  runtimeClassName: kata-remote
+  serviceAccountName: csi-ebs-podvm-sa
+  volumes:
+  - name: persistent-storage
+    persistentVolumeClaim:
+      claimName: ebs-claim
+  - name: kubelet-data-dir
+    hostPath:
+      path: /var/lib/kubelet
+      type: Directory
+  - emptyDir: {}
+    name: plugin-dir
+
+  containers:
+  - name: app
+    image: centos
+    command: ["/bin/sh"]
+    args: ["-c", "while true; do echo $(date -u) >> /data/out.txt; sleep 5; done"]
+    volumeMounts:
+    - name: persistent-storage
+      mountPath: /data
+
+  - name: csi-podvm-node-driver
+    env:
+      - name: KUBE_NODE_NAME
+        valueFrom:
+          fieldRef:
+            fieldPath: spec.nodeName
+      - name: CSI_NODE_NAME
+        valueFrom:
+          fieldRef:
+            fieldPath: spec.nodeName
+     # set when IMDS isn't accessible from the podvm
+     # - name: AWS_REGION
+     #   value: "us-east-1"
+
+    image: public.ecr.aws/ebs-csi-driver/aws-ebs-csi-driver:v1.21.0
+    imagePullPolicy: Always
+    securityContext:
+      privileged: true
+      runAsNonRoot: false
+      runAsUser: 0
+    ports:
+      - containerPort: 9808
+        name: healthz
+        protocol: TCP
+    volumeMounts:
+      - name: kubelet-data-dir
+        mountPath: /var/lib/kubelet
+        mountPropagation: Bidirectional
+      - mountPath: /tmp
+        name: plugin-dir
+
+  - name: csi-podvm-wrapper
+    env:
+      - name: BINARY
+        value: "csi-podvm-wrapper"
+      - name: POD_NAME
+        valueFrom:
+          fieldRef:
+            fieldPath: metadata.name
+      - name: POD_NAME_SPACE
+        valueFrom:
+          fieldRef:
+            fieldPath: metadata.namespace
+      - name: POD_UID
+        valueFrom:
+          fieldRef:
+            fieldPath: metadata.uid
+      - name: POD_NODE_NAME
+        valueFrom:
+          fieldRef:
+            fieldPath: spec.nodeName
+    image: quay.io/confidential-containers/csi-podvm-wrapper:latest
+    imagePullPolicy: Always
+    command: ["/usr/bin/csi-podvm-wrapper"] # TODO: using default entrypoint seems to fail with peer-pods
+    args:
+    - --v=2
+    - --endpoint=/tmp/csi-podvm-wrapper.sock
+    - --target-endpoint=/tmp/csi.sock
+    - --namespace=kube-system
+    volumeMounts:
+      - mountPath: /tmp
+        name: plugin-dir
diff --git a/volumes/csi-wrapper/examples/aws/dynamic-provisioning/storageclass.yaml b/volumes/csi-wrapper/examples/aws/dynamic-provisioning/storageclass.yaml
@@ -0,0 +1,6 @@
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: ebs-sc
+provisioner: ebs.csi.aws.com
+volumeBindingMode: WaitForFirstConsumer
diff --git a/volumes/csi-wrapper/examples/aws/example-iam-policy.json b/volumes/csi-wrapper/examples/aws/example-iam-policy.json
@@ -0,0 +1,133 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": [
+        "ec2:CreateSnapshot",
+        "ec2:AttachVolume",
+        "ec2:DetachVolume",
+        "ec2:ModifyVolume",
+        "ec2:DescribeAvailabilityZones",
+        "ec2:DescribeInstances",
+        "ec2:DescribeSnapshots",
+        "ec2:DescribeTags",
+        "ec2:DescribeVolumes",
+        "ec2:DescribeVolumesModifications"
+      ],
+      "Resource": "*"
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "ec2:CreateTags"
+      ],
+      "Resource": [
+        "arn:aws:ec2:*:*:volume/*",
+        "arn:aws:ec2:*:*:snapshot/*"
+      ],
+      "Condition": {
+        "StringEquals": {
+          "ec2:CreateAction": [
+            "CreateVolume",
+            "CreateSnapshot"
+          ]
+        }
+      }
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "ec2:DeleteTags"
+      ],
+      "Resource": [
+        "arn:aws:ec2:*:*:volume/*",
+        "arn:aws:ec2:*:*:snapshot/*"
+      ]
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "ec2:CreateVolume"
+      ],
+      "Resource": "*",
+      "Condition": {
+        "StringLike": {
+          "aws:RequestTag/ebs.csi.aws.com/cluster": "true"
+        }
+      }
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "ec2:CreateVolume"
+      ],
+      "Resource": "*",
+      "Condition": {
+        "StringLike": {
+          "aws:RequestTag/CSIVolumeName": "*"
+        }
+      }
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "ec2:DeleteVolume"
+      ],
+      "Resource": "*",
+      "Condition": {
+        "StringLike": {
+          "ec2:ResourceTag/ebs.csi.aws.com/cluster": "true"
+        }
+      }
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "ec2:DeleteVolume"
+      ],
+      "Resource": "*",
+      "Condition": {
+        "StringLike": {
+          "ec2:ResourceTag/CSIVolumeName": "*"
+        }
+      }
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "ec2:DeleteVolume"
+      ],
+      "Resource": "*",
+      "Condition": {
+        "StringLike": {
+          "ec2:ResourceTag/kubernetes.io/created-for/pvc/name": "*"
+        }
+      }
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "ec2:DeleteSnapshot"
+      ],
+      "Resource": "*",
+      "Condition": {
+        "StringLike": {
+          "ec2:ResourceTag/CSIVolumeSnapshotName": "*"
+        }
+      }
+    },
+    {
+      "Effect": "Allow",
+      "Action": [
+        "ec2:DeleteSnapshot"
+      ],
+      "Resource": "*",
+      "Condition": {
+        "StringLike": {
+          "ec2:ResourceTag/ebs.csi.aws.com/cluster": "true"
+        }
+      }
+    }
+  ]
+}
diff --git a/volumes/csi-wrapper/examples/aws/patch-controller.yaml b/volumes/csi-wrapper/examples/aws/patch-controller.yaml
@@ -0,0 +1,32 @@
+spec:
+  replicas: 1
+  template:
+    spec:
+      containers:
+        - name: csi-controller-wrapper
+          args:
+            - -v=5
+            - --endpoint=/var/lib/csi/sockets/pluginproxy/csi-controller-wrapper.sock
+            - --target-endpoint=/var/lib/csi/sockets/pluginproxy/csi.sock
+            - --namespace=kube-system
+          image: quay.io/confidential-containers/csi-controller-wrapper:latest
+          imagePullPolicy: IfNotPresent
+          volumeMounts:
+            - mountPath: /var/lib/csi/sockets/pluginproxy/
+              name: socket-dir
+
+        - name: csi-attacher
+          args:
+            - -v=2
+            - --csi-address=/var/lib/csi/sockets/pluginproxy/csi-controller-wrapper.sock
+            - --timeout=900s
+
+        - name: csi-provisioner
+          env:
+            - name: ADDRESS
+              value: /var/lib/csi/sockets/pluginproxy/csi-controller-wrapper.sock
+
+        - name: csi-resizer
+          env:
+            - name: ADDRESS
+              value: /var/lib/csi/sockets/pluginproxy/csi-controller-wrapper.sock