Opa e2e tests with cluster update

delivery.yaml

@@ -80,6 +80,8 @@ pipeline:
             value: "teapot-kubernetes-e2e-results"
           - name: ETCD_ENDPOINTS
             value: "https://etcd-server.etcd.teapot-e2e.zalan.do:2479"
+          - name: SKIPPER_OPA_ENABLED
+            value: "false"
           - name: HOSTED_ZONE
             valueFrom:
               configMapKeyRef:
@@ -155,6 +157,26 @@ pipeline:
               secretKeyRef:
                 name: kubernetes-e2e-config-secret
                 key: "OKTA_AUTH_ISSUER_URL"
+          - name: STYRA_TOKEN
+            valueFrom:
+              secretKeyRef:
+                name: kubernetes-e2e-config-secret
+                key: "STYRA_TOKEN"
+          - name: SKIPPER_OPA_BUCKET_ARN
+            valueFrom:
+              secretKeyRef:
+                name: kubernetes-e2e-config-secret
+                key: "SKIPPER_OPA_BUCKET_ARN"
+          - name: SKIPPER_OPA_OBSERVABILITY_URL
+            valueFrom:
+              secretKeyRef:
+                name: kubernetes-e2e-config-secret
+                key: "SKIPPER_OPA_OBSERVABILITY_URL"
+          - name: SKIPPER_OPA_BUNDLES_URL
+            valueFrom:
+              secretKeyRef:
+                name: kubernetes-e2e-config-secret
+                key: "SKIPPER_OPA_BUNDLES_URL"
           - name: CLUSTER_ADMIN_TOKEN
             valueFrom:
               secretKeyRef:
@@ -279,13 +301,204 @@ pipeline:
               cpu: 2
               memory: 1Gi
 
+- id: update-cluster-to-enable-opa
+  when:
+    event: pull_request
+  depends_on:
+    - e2e-tests
+    - e2e-load-test-result
+    - stackset-e2e-tests
+  type: process
+  desc: "Create cluster for OPA filter e2e tests"
+  target: stups-test
+  process: microservice_standard_test
+  config:
+    apply_manifests:
+      env: *apply_env
+    end2end_tests:
+      metadata:
+        name: e2e
+        labels:
+          application: kubernetes
+          component: e2e
+        annotations:
+          zalando.org/runtime-policy: require-on-demand
+      spec:
+        serviceAccountName: kubernetes-e2e
+        restartPolicy: Never
+        containers:
+          - name: e2e
+            image: "container-registry-test.zalando.net/teapot/kubernetes-on-aws-e2e:#{CDP_BUILD_VERSION}"
+            args:
+              - create-cluster
+            env: &opa_e2e_env
+              - name: CLUSTER_DOMAIN
+                value: teapot-e2e.zalan.do
+              - name: CLUSTER_DOMAIN_INTERNAL
+                value: ingress.cluster.local
+              - name: RESULT_BUCKET
+                value: "teapot-kubernetes-e2e-results"
+              - name: ETCD_ENDPOINTS
+                value: "https://etcd-server.etcd.teapot-e2e.zalan.do:2479"
+              - name: E2E_SKIP_CLUSTER_UPDATE
+                value: "true"
+              - name: SKIPPER_OPA_ENABLED
+                value: "true"
+              - name: FOCUS
+                value: "([Opa])"
+              - name: HOSTED_ZONE
+                valueFrom:
+                  configMapKeyRef:
+                    name: kubernetes-e2e-config
+                    key: "HOSTED_ZONE"
+              - name: REGION
+                valueFrom:
+                  configMapKeyRef:
+                    name: kubernetes-e2e-config
+                    key: "REGION"
+              - name: AWS_ACCOUNT
+                valueFrom:
+                  secretKeyRef:
+                    name: kubernetes-e2e-config-secret
+                    key: "AWS_ACCOUNT"
+              - name: ZMON_ROOT_ACCOUNT_ROLE
+                valueFrom:
+                  secretKeyRef:
+                    name: kubernetes-e2e-config-secret
+                    key: "ZMON_ROOT_ACCOUNT_ROLE"
+              - name: AUDITTRAIL_ROOT_ACCOUNT_ROLE
+                valueFrom:
+                  secretKeyRef:
+                    name: kubernetes-e2e-config-secret
+                    key: "AUDITTRAIL_ROOT_ACCOUNT_ROLE"
+              - name: SESSION_MANAGER_DESTINATION_ARN
+                valueFrom:
+                  secretKeyRef:
+                    name: kubernetes-e2e-config-secret
+                    key: "SESSION_MANAGER_DESTINATION_ARN"
+              - name: APISERVER_BUSINESS_PARTNER_IDS
+                valueFrom:
+                  secretKeyRef:
+                    name: kubernetes-e2e-config-secret
+                    key: "APISERVER_BUSINESS_PARTNER_IDS"
+              - name: LIGHTSTEP_TOKEN
+                valueFrom:
+                  secretKeyRef:
+                    name: kubernetes-e2e-config-secret
+                    key: "LIGHTSTEP_TOKEN"
+              - name: OWNER
+                valueFrom:
+                  secretKeyRef:
+                    name: kubernetes-e2e-config-secret
+                    key: "OWNER"
+              - name: VPC_ID
+                valueFrom:
+                  secretKeyRef:
+                    name: kubernetes-e2e-config-secret
+                    key: "VPC_ID"
+              - name: EFS_ID
+                valueFrom:
+                  secretKeyRef:
+                    name: kubernetes-e2e-config-secret
+                    key: "EFS_ID"
+              - name: ETCD_CLIENT_CA_CERT
+                valueFrom:
+                  secretKeyRef:
+                    name: kubernetes-e2e-config-secret
+                    key: "ETCD_CLIENT_CA_CERT"
+              - name: ETCD_CLIENT_CA_KEY
+                valueFrom:
+                  secretKeyRef:
+                    name: kubernetes-e2e-config-secret
+                    key: "ETCD_CLIENT_CA_KEY"
+              - name: ETCD_SCALYR_KEY
+                valueFrom:
+                  secretKeyRef:
+                    name: kubernetes-e2e-config-secret
+                    key: "ETCD_SCALYR_KEY"
+              - name: OKTA_AUTH_ISSUER_URL
+                valueFrom:
+                  secretKeyRef:
+                    name: kubernetes-e2e-config-secret
+                    key: "OKTA_AUTH_ISSUER_URL"
+              - name: STYRA_TOKEN
+                valueFrom:
+                  secretKeyRef:
+                    name: kubernetes-e2e-config-secret
+                    key: "STYRA_TOKEN"
+              - name: SKIPPER_OPA_BUCKET_ARN
+                valueFrom:
+                  secretKeyRef:
+                    name: kubernetes-e2e-config-secret
+                    key: "SKIPPER_OPA_BUCKET_ARN"
+              - name: SKIPPER_OPA_OBSERVABILITY_URL
+                valueFrom:
+                  secretKeyRef:
+                    name: kubernetes-e2e-config-secret
+                    key: "SKIPPER_OPA_OBSERVABILITY_URL"
+              - name: SKIPPER_OPA_BUNDLES_URL
+                valueFrom:
+                  secretKeyRef:
+                    name: kubernetes-e2e-config-secret
+                    key: "SKIPPER_OPA_BUNDLES_URL"
+              - name: CLUSTER_ADMIN_TOKEN
+                valueFrom:
+                  secretKeyRef:
+                    name: kubernetes-e2e-credentials
+                    key: "cluster-token-secret"
+            resources:
+              limits:
+                cpu: 500m
+                memory: 2Gi
+              requests:
+                cpu: 500m
+                memory: 2Gi
+
+- id: opa-e2e-tests
+  when:
+    event: pull_request
+  depends_on:
+    - update-cluster-to-enable-opa
+  type: process
+  desc: "OPA filter in Kubernetes e2e tests"
+  target: stups-test
+  process: microservice_standard_test
+  config:
+    apply_manifests:
+      env: *apply_env
+    end2end_tests:
+      metadata:
+        name: e2e
+        labels:
+          application: kubernetes
+          component: e2e
+        annotations:
+          zalando.org/runtime-policy: require-on-demand
+      spec:
+        serviceAccountName: kubernetes-e2e
+        restartPolicy: Never
+        containers:
+          - name: e2e
+            image: "container-registry-test.zalando.net/teapot/kubernetes-on-aws-e2e:#{CDP_BUILD_VERSION}"
+            args:
+              - e2e
+            env: *opa_e2e_env
+            resources:
+              limits:
+                cpu: 2
+                memory: 8Gi
+              requests:
+                cpu: 2
+                memory: 8Gi
+
 - id: decommission-cluster
   when:
     event: pull_request
   depends_on:
-  - e2e-tests
-  - stackset-e2e-tests
-  - e2e-load-test-result
+    - e2e-tests
+    - stackset-e2e-tests
+    - e2e-load-test-result
+    - opa-e2e-tests
   type: process
   desc: "Decommission cluster used for Kubernetes e2e tests"
   target: stups-test
@@ -305,18 +518,18 @@ pipeline:
         serviceAccountName: kubernetes-e2e
         restartPolicy: Never
         containers:
-        - name: e2e
-          image: "container-registry-test.zalando.net/teapot/kubernetes-on-aws-e2e:#{CDP_BUILD_VERSION}"
-          args:
-          - decommission-cluster
-          env: *e2e_env
-          resources:
-            limits:
-              cpu: 500m
-              memory: 1Gi
-            requests:
-              cpu: 500m
-              memory: 1Gi
+          - name: e2e
+            image: "container-registry-test.zalando.net/teapot/kubernetes-on-aws-e2e:#{CDP_BUILD_VERSION}"
+            args:
+              - decommission-cluster
+            env: *e2e_env
+            resources:
+              limits:
+                cpu: 500m
+                memory: 1Gi
+              requests:
+                cpu: 500m
+                memory: 1Gi
 
 - id: deploy-event
   type: deploy

test/e2e/apply/secret.yaml

@@ -19,3 +19,7 @@ data:
   ETCD_SCALYR_KEY: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwETRzvm1hGplyUn23FEXUVtAAAAnjCBmwYJKoZIhvcNAQcGoIGNMIGKAgEAMIGEBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDOfPJJJy60sDkZEIHgIBEIBXiANNciEqpcuZ3hFPCt6NkFtk0WBTSasDQHHbyuR8O+n5iM9k8/nUTLUrFlhba8blArq/ALE8vuKNdlS17q6PxGlvwJFFXQn/McohMpdyfnfQYKW8MPCu"
   OKTA_AUTH_ISSUER_URL: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwGmCMhSN2Er1sw2ofYnI44EAAAApDCBoQYJKoZIhvcNAQcGoIGTMIGQAgEAMIGKBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDO2IC+r/zcUzXoQEHAIBEIBdrFchwu9i7LpMbyDbslu/lBxvfyh+nCGK33jtcxT3RdxuTXWuSJhkX+gU4cgFXAI5LLnXh4M20jHUEEPU78MJWR47HLTPGPJcKQj5fOpPqpD3duuKIrZDRm5ba6AN"
   SESSION_MANAGER_DESTINATION_ARN: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwF7fOZ9i6BDvWdNEddR7LZOAAAArjCBqwYJKoZIhvcNAQcGoIGdMIGaAgEAMIGUBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDBJwU/Zns+mzOBgczQIBEIBn/86xpnVO2Apr5nG3waPEAGCFYDWdOXcaS7pFKdNIhpXaADtODQtEd874HcE0W2I3bjKr3d3ghJFdN8r0BZiSmTbgc0fn+5ZiBTyGBfzWP4BCzxjRMvURl/7MX8ygwL78hpSxyRypAQ=="
+  STYRA_TOKEN: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwEECuXXi+W3FFt7qLjWk/S6AAAAwDCBvQYJKoZIhvcNAQcGoIGvMIGsAgEAMIGmBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDGAuwqmeDmRyjVy91gIBEIB5u6jiCoj1vIwZJ/dJtdI/8cxG9y6RGjopd20Sh1+5TCoHKzPfyV97Whl6YFLRke6ixO+UBnA4KeNh5A/ykQ7yUIvg5b9WDH5tV8Gb+vWyvsd4sdULVfioeTS67e6S0ApSMd/CHCfZdsTwTi1iZ2spSkS0YWolGyY+9A=="
+  SKIPPER_OPA_BUCKET_ARN: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwGZdCVDLsCdProfzvZU7UAwAAAAlzCBlAYJKoZIhvcNAQcGoIGGMIGDAgEAMH4GCSqGSIb3DQEHATAeBglghkgBZQMEAS4wEQQMehOf7Uu444SWS6kbAgEQgFFPMaa0flwHLpxrkYjJMK4jXc0q4kX+KGrB5GFjKuUgOUPmQ+ME/aQduxwl2+xUilrKP50/NLXgMNHjeeHuZfoyiSgpGFBM4z8L0N6ggf2uE5U="
+  SKIPPER_OPA_OBSERVABILITY_URL: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwHl773AuNEvIpzaM6ycpDNSAAAAqzCBqAYJKoZIhvcNAQcGoIGaMIGXAgEAMIGRBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDGld6jpQ38gOzVcn0gIBEIBkTHbv3adeEfRntVTUQyyQkIhUnc0QXKtmtJEdvBoRzWiJIBKQUQuM1VBV0re3HkO8HSY59nkwyHEncBMkHJoI9rC2LJuWU20oCjPw9lbweih+6Sxo+nqkDrQd+mHp+uA9Om3KqA=="
+  SKIPPER_OPA_BUNDLES_URL: "deployment-secret:2:stups-test:AQICAHjXIrc66g/+P4X1Gl4MKcInWmwpFxivAqFGMI0fr9DvCwFnhaIRP4+3Y69xp1ycTI7qAAAAsTCBrgYJKoZIhvcNAQcGoIGgMIGdAgEAMIGXBgkqhkiG9w0BBwEwHgYJYIZIAWUDBAEuMBEEDF9gAl70l2g2kwfnJgIBEIBqP/DgIhIu5x5XNR1Ubqinz6r4ttQoHty8nXd6mxie2r6NxHskNOqkiSactUKhNIhboNlNsO4p4rKEkhglTeFZlEQvgEYNioWPw39xqICnUDPVr+Kp0Yrs/bzPLPV9wOlB917UiT7WJNybPg=="

test/e2e/cluster_config.sh

@@ -47,6 +47,11 @@ clusters:
     karpenter_pools_enabled: "true"
     okta_auth_client_id: "kubernetes.cluster.teapot-e2e"
     teapot_admission_controller_validate_pod_images_soft_fail_namespaces: "^kube-system$"
+    skipper_open_policy_agent_enabled: "${SKIPPER_OPA_ENABLED}"
+    skipper_open_policy_agent_styra_token: "${STYRA_TOKEN}"
+    skipper_open_policy_agent_bucket_arn: "${SKIPPER_OPA_BUCKET_ARN}"
+    skipper_open_policy_agent_observability_url: "${SKIPPER_OPA_OBSERVABILITY_URL}"
+    skipper_open_policy_agent_bundles_url: "${SKIPPER_OPA_BUNDLES_URL}"
   criticality_level: 1
   environment: e2e
   id: ${CLUSTER_ID}