SecureComms: E2e test SecureComms without KBS

Add support for e2e testing SecureComms without KBS

Signed-off-by: David Hadas <david.hadas@gmail.com>

.github/workflows/e2e_libvirt.yaml

@@ -23,7 +23,11 @@ on:
         description: Git ref to checkout the cloud-api-adaptor repository. Defaults to main.
         required: false
         type: string
-
+      e2e:
+        default: 'none'
+        description: SecureComms configuration. Defaults to none.
+        required: false
+        type: string
 env:
   CLOUD_PROVIDER: libvirt
   DEBIAN_FRONTEND: noninteractive
@@ -36,6 +40,11 @@ jobs:
   test:
     runs-on: az-ubuntu-2204
     steps:
+      - name: Test The Test
+        run: |
+          echo "This is a test - remove it!"
+          echo "secure_comms=\"${{ inputs.secure_comms }}\""
+
       - name: Checkout Code
         uses: actions/checkout@v4
         with:
@@ -84,7 +93,8 @@ jobs:
 
       - name: Config Libvirt
         run: |
-          ./libvirt/config_libvirt.sh
+          echo "secure_comms=\"${{ inputs.secure_comms }}\""
+          ./libvirt/config_libvirt.sh ${{ inputs.secure_comms }}
           echo "CAA_IMAGE=\"${{ inputs.caa_image }}\"" >> libvirt.properties
           # For debugging
           cat libvirt.properties

.github/workflows/e2e_run_all.yaml

@@ -99,6 +99,8 @@ jobs:
   prep_install:
     needs: [image]
     runs-on: ubuntu-latest
+    outputs:
+      securecomms: ${{ steps.securecomms.outputs.securecomms }}
     env:
       PROVIDERS: "libvirt"
     steps:
@@ -148,6 +150,11 @@ jobs:
           path: src/cloud-api-adaptor/install/
           retention-days: 7
 
+      - name: Define Secure Comms Test Configs
+        id: securecomms
+        run: |
+          ./src/cloud-api-adaptor/libvirt/config_libvirt.sh >> "$GITHUB_OUTPUT"
+  
   # Run libvirt e2e tests if pull request labeled 'test_e2e_libvirt'
   libvirt:
     name: libvirt
@@ -165,10 +172,12 @@ jobs:
           - generic
         arch:
           - amd64
+        secure_comms: ${{ fromJSON(needs.prep_install.outputs.securecomms) }}
     uses: ./.github/workflows/e2e_libvirt.yaml
     with:
       caa_image: ${{ inputs.registry }}/cloud-api-adaptor:${{ inputs.caa_image_tag }}-dev
       podvm_image: ${{ inputs.registry }}/podvm-${{ matrix.provider }}-${{ matrix.os }}-${{ matrix.arch }}:${{ inputs.podvm_image_tag }}
       install_directory_artifact: install_directory
       git_ref: ${{ inputs.git_ref }}
+      secure_comms: ${{ matrix.secure_comms }}
     secrets: inherit

src/cloud-api-adaptor/cmd/agent-protocol-forwarder/main.go

@@ -93,7 +93,9 @@ func (cfg *Config) Setup() (cmd.Starter, error) {
 		return nil, err
 	}
 
-	if secureComms {
+	if secureComms || cfg.daemonConfig.SecureComms {
+		var inbounds, outbounds []string
+
 		ppssh.Singleton()
 		host, port, err := net.SplitHostPort(cfg.listenAddr)
 		if err != nil {
@@ -103,15 +105,32 @@ func (cfg *Config) Setup() (cmd.Starter, error) {
 			logger.Printf("Address %s is changed to 127.0.0.1:%s since secure-comms is enabled.", cfg.listenAddr, port)
 			cfg.listenAddr = "127.0.0.1:" + port
 		}
-		inbounds := append([]string{"BOTH_PHASES:KBS:8080"}, strings.Split(secureCommsInbounds, ",")...)
-		outbounds := append([]string{"KUBERNETES_PHASE:KATAAGENT:" + port}, strings.Split(secureCommsOutbounds, ",")...)
 
 		// Create a Client that will approach the api-server-rest service at the podns
 		// To obtain secrets from KBS, we approach the api-server-rest service which then approaches the CDH asking for a secret resource
 		// the CDH than contact the KBS (possibly after approaching Attestation Agent for a token) and the KBS serves the requested key
 		// The communication between the CDH (and Attestation Agent) and the KBS is performed via an SSH tunnel named  "KBS"
 		apic := apic.NewApiClient(API_SERVER_REST_PORT, cfg.podNamespace)
-		services = append(services, ppssh.NewSshServer(inbounds, outbounds, ppssh.GetSecret(apic.GetKey), sshutil.SSHPORT))
+
+		ppSecrets := ppssh.NewPpSecrets(ppssh.GetSecret(apic.GetKey))
+
+		if secureComms {
+			// CoCo in production
+			ppSecrets.AddKey(ppssh.WN_PUBLIC_KEY)
+			ppSecrets.AddKey(ppssh.PP_PRIVATE_KEY)
+			inbounds = append([]string{"BOTH_PHASES:KBS:8080"}, strings.Split(secureCommsInbounds, ",")...)
+			outbounds = append([]string{"KUBERNETES_PHASE:KATAAGENT:" + port}, strings.Split(secureCommsOutbounds, ",")...)
+
+		} else {
+			// Never here under CoCo in production
+			// Set secureComms using daemonConfig for testing
+			ppSecrets.SetKey(ppssh.WN_PUBLIC_KEY, cfg.daemonConfig.WnPublicKey)
+			ppSecrets.SetKey(ppssh.PP_PRIVATE_KEY, cfg.daemonConfig.PpPrivateKey)
+			inbounds = append([]string{"BOTH_PHASES:KBS:8080"}, strings.Split(cfg.daemonConfig.SecureCommsInbounds, ",")...)
+			outbounds = append([]string{"KUBERNETES_PHASE:KATAAGENT:" + port}, strings.Split(cfg.daemonConfig.SecureCommsOutbounds, ",")...)
+		}
+
+		services = append(services, ppssh.NewSshServer(inbounds, outbounds, ppSecrets, sshutil.SSHPORT))
 	} else {
 		if !disableTLS {
 			cfg.tlsConfig = &tlsConfig

src/cloud-api-adaptor/cmd/cloud-api-adaptor/main.go

@@ -9,9 +9,11 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"strings"
 
 	"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/cmd"
 	"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/adaptor"
+	"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/adaptor/cloud"
 	"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/adaptor/proxy"
 	daemon "github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/forwarder"
 	"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/podnetwork/tunneler/vxlan"
@@ -28,7 +30,7 @@ const (
 )
 
 type daemonConfig struct {
-	serverConfig adaptor.ServerConfig
+	serverConfig cloud.ServerConfig
 	networkConfig
 }
 
@@ -86,12 +88,14 @@ func (cfg *daemonConfig) Setup() (cmd.Starter, error) {
 	}
 
 	var (
-		disableTLS           bool
-		tlsConfig            tlsutil.TLSConfig
-		secureComms          bool
-		secureCommsInbounds  string
-		secureCommsOutbounds string
-		secureCommsKbsAddr   string
+		disableTLS             bool
+		tlsConfig              tlsutil.TLSConfig
+		secureComms            bool
+		secureCommsInbounds    string
+		secureCommsOutbounds   string
+		secureCommsPpInbounds  string
+		secureCommsPpOutbounds string
+		secureCommsKbsAddr     string
 	)
 
 	cmd.Parse(programName, os.Args[1:], func(flags *flag.FlagSet) {
@@ -112,8 +116,10 @@ func (cfg *daemonConfig) Setup() (cmd.Starter, error) {
 		flags.BoolVar(&tlsConfig.SkipVerify, "tls-skip-verify", false, "Skip TLS certificate verification - use it only for testing")
 		flags.BoolVar(&disableTLS, "disable-tls", false, "Disable TLS encryption - use it only for testing")
 		flags.BoolVar(&secureComms, "secure-comms", false, "Use SSH to secure communication between cluster and peer pods")
-		flags.StringVar(&secureCommsInbounds, "secure-comms-inbounds", "", "Inbound tags for secure communication tunnels")
-		flags.StringVar(&secureCommsOutbounds, "secure-comms-outbounds", "", "Outbound tags for secure communication tunnels")
+		flags.StringVar(&secureCommsInbounds, "secure-comms-inbounds", "", "WN Inbound tags for secure communication tunnels")
+		flags.StringVar(&secureCommsOutbounds, "secure-comms-outbounds", "", "WN Outbound tags for secure communication tunnels")
+		flags.StringVar(&secureCommsPpInbounds, "secure-comms-pp-inbounds", "", "PP Inbound tags for secure communication tunnels")
+		flags.StringVar(&secureCommsPpOutbounds, "secure-comms-pp-outbounds", "", "PP Outbound tags for secure communication tunnels")
 		flags.StringVar(&secureCommsKbsAddr, "secure-comms-kbs", "kbs-service.trustee-operator-system:8080", "Address of a Trustee Service for Secure-Comms")
 		flags.DurationVar(&cfg.serverConfig.ProxyTimeout, "proxy-timeout", proxy.DefaultProxyTimeout, "Maximum timeout in minutes for establishing agent proxy connection")
 
@@ -137,10 +143,16 @@ func (cfg *daemonConfig) Setup() (cmd.Starter, error) {
 		if err != nil {
 			return nil, fmt.Errorf("secure comms failed to initialize KubeMgr: %w", err)
 		}
-
+		if strings.EqualFold(secureCommsKbsAddr, "false") {
+			secureCommsKbsAddr = ""
+			fmt.Printf("secureCommsKbsAddr was false\n")
+		}
+		fmt.Printf("secureCommsKbsAddr is %s\n", secureCommsKbsAddr)
 		cfg.serverConfig.SecureComms = true
 		cfg.serverConfig.SecureCommsInbounds = secureCommsInbounds
 		cfg.serverConfig.SecureCommsOutbounds = secureCommsOutbounds
+		cfg.serverConfig.SecureCommsPpInbounds = secureCommsPpInbounds
+		cfg.serverConfig.SecureCommsPpOutbounds = secureCommsPpOutbounds
 		cfg.serverConfig.SecureCommsKbsAddress = secureCommsKbsAddr
 	} else {
 		if !disableTLS {

src/cloud-api-adaptor/docs/SecureComms.md

@@ -27,7 +27,35 @@ Once the "Kubernetes Phase" SSH channel is established, Secure Comms connects th
 
 See [Secure Comms Architecture Slides](./SecureComms.pdf) for more details.
 
-## Setup
+## Setup for for testing without Trustee (and for non-CoCo peerpods)
+
+### Deploy CAA
+Use any of the option for installing CAA depending on the cloud driver used.
+
+
+### Activate Secure-Comms feature from CAA side
+Activate Secure-Comms from CAA side by changing the `SECURE_COMMS` parameter of the `peer-pods-cm` configMap in the `confidential-containers-system` namespace to `"true"`.  
+
+```sh
+kubectl -n confidential-containers-system  get cm peer-pods-cm  -o yaml | sed "s/SECURE_COMMS: \"false\"/SECURE_COMMS: \"true\"/"|kubectl apply -f -
+```
+
+You may also include additional Inbounds and Outbounds configurations to the Adaptor side using the `SECURE_COMMS_INBOUNDS` and `SECURE_COMMS_OUTBOUNDS` config points.
+You may also add Inbounds and Outbounds configurations to the Forwarder side using the `SECURE_COMMS_PP_INBOUNDS` and `SECURE_COMMS_PP_OUTBOUNDS` config points. [See more details regarding Inbounds and Outbounds below.](#adding-named-tunnels-to-the-ssh-channel)
+
+Use `kubectl edit cm peer-pods-cm -n confidential-containers-system` to make such changes in the configMap, for example:
+```sh
+apiVersion: v1
+data:
+    ...
+    SECURE_COMMS: "true"
+    SECURE_COMMS_OUTBOUNDS: "KUBERNETES_PHASE:mytunnel:149.81.64.62:7777"
+    SECURE_COMMS_PP_INBOUNDS: "KUBERNETES_PHASE:mytunnel:6666"
+    SECURE_COMMS_KBS_ADDR: "false"
+    ...
+```
+
+## Setup for CoCo with Trustee
 
 ### Deploy CAA
 Use any of the option for installing CAA depending on the cloud driver used.
@@ -53,8 +81,7 @@ kubectl get secret kbs-client -n trustee-operator-system -o json|jq --arg ns "co
 For a testing environment, you may need to change the policy of the KBS and AS using the KBS Client to allow all or fit your own policy. One way to do that is:
 
 ```sh
-kubectl -n trustee-operator-system exec deployment/trustee-deployment --container as -it -- /bin/bash
-        sed -i.bak 's/^default allow = false/default allow = true/' /opt/confidential-containers/attestation-service/opa/default.rego
+kubectl -n trustee-operator-system exec deployment/trustee-deployment --container as -it -- sed -i.bak 's/^default allow = false/default allow = true/' /opt/confidential-containers/attestation-service/opa/default.rego
 
 kubectl -n trustee-operator-system get cm resource-policy -o yaml | sed "s/default allow = false/default allow = true/"|kubectl apply -f -
 ```
@@ -66,18 +93,61 @@ Change the `src/cloud-api-adaptor/podvm/files/etc/systemd/system/agent-protocol-
 ExecStart=/usr/local/bin/agent-protocol-forwarder -pod-namespace /run/netns/podns -secure-comms -kata-agent-socket /run/kata-containers/agent.sock $TLS_OPTIONS $OPTIONS
 ```
 
-You may also include additional Inbounds and Outbounds configurations to the Forwarder using the `-secure-comms-inbounds` and `-secure-comms-outbounds` flags. See more details regarding Inbounds and Outbounds below.
+You may also include additional Inbounds and Outbounds configurations to the Forwarder using the `-secure-comms-inbounds` and `-secure-comms-outbounds` flags.  [See more details regarding Inbounds and Outbounds below.](#adding-named-tunnels-to-the-ssh-channel)
+
+For example:
+```sh
+ExecStart=/usr/local/bin/agent-protocol-forwarder -kata-agent-namespace /run/netns/podns -secure-comms -secure-comms-inbounds KUBERNETES_PHASE:mytunnel:6666 -kata-agent-socket /run/kata-containers/agent.sock $TLS_OPTIONS $OPTIONS
+```
 
 Once you changed `podvm/files/etc/systemd/system/agent-protocol-forwarder.service`, you will need to [rebuild the podvm](./../podvm/README.md).
 
 
 ### Activate CAA Secure-Comms feature
-Use `kubectl edit cm peer-pods-cm -n confidential-containers-system` to add to the `peer-pods-cm` config map at the `confidential-containers-system` namespace:
+Activate Secure-Comms of CAA by changing the `SECURE_COMMS` parameter of the `peer-pods-cm` configMap in the `confidential-containers-system` namespace to `"true"`.  
+
+```sh
+kubectl -n confidential-containers-system  get cm peer-pods-cm  -o yaml | sed "s/SECURE_COMMS: \"false\"/SECURE_COMMS: \"true\"/"|kubectl apply -f -
+```
+
+Set InitData to point KBC services to IP address 127.0.0.1 
+```sh
+cat <<EOF > /tmp/initdata.txt
+algorithm = "sha384"
+version = "0.1.0"
+
+[data]
+"aa.toml" = '''
+[token_configs]
+[token_configs.coco_as]
+url = 'http://127.0.0.1:8080'
+
+[token_configs.kbs]
+url = 'http://127.0.0.1:8080'
+'''
+"cdh.toml"  = '''
+socket = 'unix:///run/confidential-containers/cdh.sock'
+credentials = []
+
+[kbc]
+name = 'cc_kbc'
+url = 'http://127.0.0.1:8080'
+'''
+EOF
+export INITDATA=`base64 -w 0 /tmp/initdata.txt`
+kubectl -n confidential-containers-system  get cm peer-pods-cm  -o yaml | sed 's/^INITDATA: .*/INITDATA: '$INITDATA'/'|kubectl apply -f -
+
+```
+
+You may also include additional Inbounds and Outbounds configurations to the Adaptor using the `SECURE_COMMS_INBOUNDS` and `SECURE_COMMS_OUTBOUNDS` config points. [See more details regarding Inbounds and Outbounds below.](#adding-named-tunnels-to-the-ssh-channel)
+
+Use `kubectl edit cm peer-pods-cm -n confidential-containers-system` to make such changes in the configMap, for example:
 ```sh
 apiVersion: v1
 data:
     ...
     SECURE_COMMS: "true"
+    SECURE_COMMS_OUTBOUNDS: "KUBERNETES_PHASE:mytunnel:149.81.64.62:7777"
     ...
 ```
 
@@ -120,7 +190,7 @@ You may also set the KBS address using the `SECURE_COMMS_KBS_ADDR` config point.
 >
 
 
-### Adding named tunnels to the SSH channel
+## Adding named tunnels to the SSH channel
 Named tunnels can be added to the SSH channel. Adding a named tunnel requires adding an Inbound at one of the SSH channel peers and an Outbound at the other SSH channel peer. The Inbound and Outbound both carry the name of the tunnel being created.
 
             |---------Tunnel----------| 
@@ -158,5 +228,4 @@ Alternatively, the client and server can be separately executed in independent t
 
 - Add DeleteResource() support in KBS, KBC, api-server-rest, than cleanup resources added by Secure Comms to KBS whenever a Peer Pod fail to be created or when a Peer Pod is terminated.
 - Add support for running the vxlan tunnel traffic via a Secure Comms SSH tunnel
-- Add support for non-confidential Peer Pods which do not go via an Attestation Phase.
 - Add support for KBS identities allowing a Peer Pod to register its own identity in KBS and replace the current Secure Comms mechanism which delivers a private key to the Peer Pod via the KBS

src/cloud-api-adaptor/entrypoint.sh

@@ -24,6 +24,8 @@ optionals+=""
 [[ "${SECURE_COMMS}" == "true" ]] && optionals+="-secure-comms "
 [[ "${SECURE_COMMS_INBOUNDS}" ]] && optionals+="-secure-comms-inbounds ${SECURE_COMMS_INBOUNDS} "
 [[ "${SECURE_COMMS_OUTBOUNDS}" ]] && optionals+="-secure-comms-outbounds ${SECURE_COMMS_OUTBOUNDS} "
+[[ "${SECURE_COMMS_PP_INBOUNDS}" ]] && optionals+="-secure-comms-pp-inbounds ${SECURE_COMMS_PP_INBOUNDS} "
+[[ "${SECURE_COMMS_PP_OUTBOUNDS}" ]] && optionals+="-secure-comms-pp-outbounds ${SECURE_COMMS_PP_OUTBOUNDS} "
 [[ "${SECURE_COMMS_KBS_ADDR}" ]] && optionals+="-secure-comms-kbs ${SECURE_COMMS_KBS_ADDR} "
 [[ "${PEERPODS_LIMIT_PER_NODE}" ]] && optionals+="-peerpods-limit-per-node ${PEERPODS_LIMIT_PER_NODE} "
 

src/cloud-api-adaptor/libvirt/README.md

@@ -217,6 +217,7 @@ make TEST_PROVISION=no TEST_TEARDOWN=no TEST_PODVM_IMAGE=$PWD/podvm/podvm.qcow2
 * ``TEST_E2E_TIMEOUT`` - test timeout
 * ``DEPLOY_KBS`` - whether to deploy the key-broker-service, which is used to test the attestation flow
 * ``TEST_PROVISION_FILE`` - file specifying the libvirt connection and the ssh key file (created earlier by [config_libvirt.sh](config_libvirt.sh))
+* ``TEST_CAA_LOG`` - whether to log the CAA at teh end of the tests run
 
 # Delete Confidential Containers and cloud-api-adaptor from the cluster
 

src/cloud-api-adaptor/libvirt/config_libvirt.sh

@@ -92,6 +92,16 @@ installK8sclis() {
     fi
 }
 
+if [ $# -lt 1 ]
+then
+    echo 'securecomms=["none", "withoutKbs"]'
+    exit 0
+else
+    SECURE_COMMS=$1
+fi
+
+echo "SECURE_COMMS is ${SECURE_COMMS}"
+
 echo "Installing Go..."
 installGolang
 echo "Installing Libvirt..."
@@ -118,3 +128,21 @@ rm -f libvirt.properties
 echo "libvirt_uri=\"qemu+ssh://${USER}@${IP}/system?no_verify=1\"" >> libvirt.properties
 echo "libvirt_ssh_key_file=\"id_rsa\"" >> libvirt.properties
 echo "CLUSTER_NAME=\"peer-pods\"" >> libvirt.properties
+KBS_IMAGE=$(./hack/yq-shim.sh '.oci.kbs.registry' ./versions.yaml)
+KBS_IMAGE_TAG=$(./hack/yq-shim.sh '.oci.kbs.tag' ./versions.yaml)
+[ -z ${KBS_IMAGE} ] || echo "KBS_IMAGE=\"${KBS_IMAGE}\"" >> libvirt.properties
+[ -z ${KBS_IMAGE_TAG} ] || echo "KBS_IMAGE_TAG=\"${KBS_IMAGE_TAG}\"" >> libvirt.properties
+
+case $SECURE_COMMS in
+
+  "withoutKbs")
+    echo "processing withoutKbs"
+    echo "SECURE_COMMS=\"true\"" >> libvirt.properties
+    echo "SECURE_COMMS_KBS_ADDR=\"false\"" >> libvirt.properties
+    echo "INITDATA=\"\"" >> libvirt.properties
+    ;;
+
+  *)
+    echo "processing none"
+    ;;
+esac