From 914490c720d0991f8f45523688850f079535642c Mon Sep 17 00:00:00 2001 From: David Hadas Date: Sat, 28 Sep 2024 08:41:18 -0500 Subject: [PATCH] SecureComms: Activate PP SC using InitData Add apj.json to InitData apf.josn includes a secure-comms key used to activate secure comms (if not already activated using an agent-protocol-forwarder.service flag) Signed-off-by: David Hadas --- .../cmd/agent-protocol-forwarder/main.go | 19 ++++- src/cloud-api-adaptor/docs/SecureComms.md | 75 ++++++++++++++++--- .../pkg/adaptor/cloud/cloud.go | 2 +- .../pkg/forwarder/forwarder.go | 9 ++- .../pkg/userdata/provision.go | 4 +- 5 files changed, 90 insertions(+), 19 deletions(-) diff --git a/src/cloud-api-adaptor/cmd/agent-protocol-forwarder/main.go b/src/cloud-api-adaptor/cmd/agent-protocol-forwarder/main.go index 47178c7b8..6667cd97d 100644 --- a/src/cloud-api-adaptor/cmd/agent-protocol-forwarder/main.go +++ b/src/cloud-api-adaptor/cmd/agent-protocol-forwarder/main.go @@ -34,7 +34,9 @@ var logger = log.New(log.Writer(), "[forwarder] ", log.LstdFlags|log.Lmsgprefix) type Config struct { tlsConfig *tlsutil.TLSConfig daemonConfig daemon.Config - configPath string + apfConfig daemon.ApfConfig + daemonConfigPath string + apfConfigPath string listenAddr string kataAgentSocketPath string podNamespace string @@ -52,6 +54,8 @@ func load(path string, obj interface{}) error { return fmt.Errorf("failed to decode a Agent Protocol Forwarder config file file: %s: %w", path, err) } + logger.Printf("Succesfully loading config from %s\n", path) + return nil } @@ -68,7 +72,8 @@ func (cfg *Config) Setup() (cmd.Starter, error) { cmd.Parse(programName, os.Args, func(flags *flag.FlagSet) { flags.BoolVar(&showVersion, "version", false, "Show version") - flags.StringVar(&cfg.configPath, "config", daemon.DefaultConfigPath, "Path to a daemon config file") + flags.StringVar(&cfg.daemonConfigPath, "config", daemon.DefaultDaemonConfigPath, "Path to a daemon config file") + flags.StringVar(&cfg.apfConfigPath, "apf-config", daemon.DefaultAPFConfigPath, "Path to APF config file") flags.StringVar(&cfg.listenAddr, "listen", daemon.DefaultListenAddr, "Listen address") flags.StringVar(&cfg.kataAgentSocketPath, "kata-agent-socket", daemon.DefaultKataAgentSocketPath, "Path to a kata agent socket") flags.StringVar(&cfg.podNamespace, "pod-namespace", daemon.DefaultPodNamespace, "Path to the network namespace where the pod runs") @@ -89,10 +94,18 @@ func (cfg *Config) Setup() (cmd.Starter, error) { cmd.Exit(0) } - if err := load(cfg.configPath, &cfg.daemonConfig); err != nil { + if err := load(cfg.daemonConfigPath, &cfg.daemonConfig); err != nil { + return nil, err + } + + if err := load(cfg.apfConfigPath, &cfg.apfConfig); err != nil { return nil, err } + if cfg.apfConfig.SecureComms { + secureComms = true + } + if secureComms { ppssh.Singleton() host, port, err := net.SplitHostPort(cfg.listenAddr) diff --git a/src/cloud-api-adaptor/docs/SecureComms.md b/src/cloud-api-adaptor/docs/SecureComms.md index e9737dd8d..568370ca1 100644 --- a/src/cloud-api-adaptor/docs/SecureComms.md +++ b/src/cloud-api-adaptor/docs/SecureComms.md @@ -27,7 +27,7 @@ Once the "Kubernetes Phase" SSH channel is established, Secure Comms connects th See [Secure Comms Architecture Slides](./SecureComms.pdf) for more details. -## Setup +## Setup for CoCo with Trustee ### Deploy CAA Use any of the option for installing CAA depending on the cloud driver used. @@ -53,31 +53,62 @@ kubectl get secret kbs-client -n trustee-operator-system -o json|jq --arg ns "co For a testing environment, you may need to change the policy of the KBS and AS using the KBS Client to allow all or fit your own policy. One way to do that is: ```sh -kubectl -n trustee-operator-system exec deployment/trustee-deployment --container as -it -- /bin/bash - sed -i.bak 's/^default allow = false/default allow = true/' /opt/confidential-containers/attestation-service/opa/default.rego +kubectl -n trustee-operator-system exec deployment/trustee-deployment --container as -it -- sed -i.bak 's/^default allow = false/default allow = true/' /opt/confidential-containers/attestation-service/opa/default.rego kubectl -n trustee-operator-system get cm resource-policy -o yaml | sed "s/default allow = false/default allow = true/"|kubectl apply -f - ``` -### Build a podvm that enforces Secure-Comms +### Activate the Secure-Comms feature +Activate Secure-Comms at the CAA by changing the `SECURE_COMMS` parameter of the `peer-pods-cm` configMap in the `confidential-containers-system` namespace to `"true"`. -Change the `src/cloud-api-adaptor/podvm/files/etc/systemd/system/agent-protocol-forwarder.service` to include: ```sh -ExecStart=/usr/local/bin/agent-protocol-forwarder -pod-namespace /run/netns/podns -secure-comms -kata-agent-socket /run/kata-containers/agent.sock $TLS_OPTIONS $OPTIONS +kubectl -n confidential-containers-system get cm peer-pods-cm -o yaml | sed "s/SECURE_COMMS: \"false\"/SECURE_COMMS: \"true\"/"|kubectl apply -f - ``` -You may also include additional Inbounds and Outbounds configurations to the Forwarder using the `-secure-comms-inbounds` and `-secure-comms-outbounds` flags. See more details regarding Inbounds and Outbounds below. +Next, we need to signal the Peer Pod to also activate the Secure-Comms feature. This can be done using an `apf.json` file included in the InitData. InitData must be measured and attested together with the podvm measurement to be trusted. In use cases where the InitData is not trusted, [use a podvm image with Secure-Comms enabled by default as shown below.](#build-a-podvm-that-enforces-secure-comms-optional) -Once you changed `podvm/files/etc/systemd/system/agent-protocol-forwarder.service`, you will need to [rebuild the podvm](./../podvm/README.md). +Set InitData to point AA and KBC services to IP address 127.0.0.1 and activate Secure-Comms at the Peer Pod: +```sh +cat < /tmp/initdata.txt +algorithm = "sha384" +version = "0.1.0" + +[data] +"aa.toml" = ''' +[token_configs] +[token_configs.coco_as] +url = 'http://127.0.0.1:8080' -### Activate CAA Secure-Comms feature -Use `kubectl edit cm peer-pods-cm -n confidential-containers-system` to add to the `peer-pods-cm` config map at the `confidential-containers-system` namespace: +[token_configs.kbs] +url = 'http://127.0.0.1:8080' +''' +"apf.json" = ''' +{ + secure-comms: true +} +''' +"cdh.toml" = ''' +socket = 'unix:///run/confidential-containers/cdh.sock' +credentials = [] +[kbc] +name = 'cc_kbc' +url = 'http://127.0.0.1:8080' +''' +EOF +export INITDATA=`base64 -w 0 /tmp/initdata.txt` +kubectl -n confidential-containers-system get cm peer-pods-cm -o yaml | sed 's/^ INITDATA: .*/ INITDATA: '$INITDATA'/'|kubectl apply -f - + +``` +On the Worker Node side, you may include additional Inbounds and Outbounds configurations to the Adaptor using the `SECURE_COMMS_INBOUNDS` and `SECURE_COMMS_OUTBOUNDS` config points. [See more details regarding Inbounds and Outbounds below.](#adding-named-tunnels-to-the-ssh-channel) + +Use `kubectl edit cm peer-pods-cm -n confidential-containers-system` to make such changes in the configMap, for example: ```sh apiVersion: v1 data: ... SECURE_COMMS: "true" + SECURE_COMMS_OUTBOUNDS: "KUBERNETES_PHASE:mytunnel:149.81.64.62:7777" ... ``` @@ -120,7 +151,8 @@ You may also set the KBS address using the `SECURE_COMMS_KBS_ADDR` config point. > -### Adding named tunnels to the SSH channel + +## Adding named tunnels to the SSH channel Named tunnels can be added to the SSH channel. Adding a named tunnel requires adding an Inbound at one of the SSH channel peers and an Outbound at the other SSH channel peer. The Inbound and Outbound both carry the name of the tunnel being created. |---------Tunnel----------| @@ -129,9 +161,10 @@ Named tunnels can be added to the SSH channel. Adding a named tunnel requires ad Inbounds and Outbounds take the form of a comma separated inbound/outbound tags such that Inbounds are formed as "InboundTag1,InboundTag2,InboundTag3,..." and Outbounds are formed as "OutboundTag1,OutboundTag2,outboundTag3,..." -Each Inbound tag is structured as `Phase:Name:Port` where: +Each Inbound tag is structured as `Phase:Name:Namespace:Port` or `Phase:Name:Port` where: - Phase can be 'KUBERNETES_PHASE' to represent an outbound available during the Kubernetes phase, 'ATTESTATION_PHASE' to represent an outbound available during the Attestation phase, or 'BOTH_PHASES' to represent an outbound available during both phases. - Name is the name of the tunnel +- Namespace (if available) is a linux network namespace where the local service should be available. - Port is the local service port being opened to serve as ingress of the tunnel. Each outbound tag is structured as `Phase:Name:Host:Port` or `Phase:Name:Port` where: @@ -142,6 +175,24 @@ Each outbound tag is structured as `Phase:Name:Host:Port` or `Phase:Name:Port` w For example, an outbound tag such as `KUBERNETES_PHASE:ABC:myhost.com:1234` means that during the `Kubernetes phase`, an output of a tunnel named `ABC` is registered, such that information from a client connecting to ABC Inbound will be tunneled and forwarded to `myhost.com` port `1234`). +### Build a podvm that enforces Secure-Comms (Optional) + +This stage is optional, it can be used to activate Secure Comms in the podvm for cases in which InitData is not trusted. InitData must be measured and attested together with the podvm measurement to be trusted. The below procedure can be used in cases where such measurement is not used. In such cases, you may force the a podvm to activate Secure-Comms using a flag as described below. + +Change the `src/cloud-api-adaptor/podvm/files/etc/systemd/system/agent-protocol-forwarder.service` to include: +```sh +ExecStart=/usr/local/bin/agent-protocol-forwarder -pod-namespace /run/netns/podns -secure-comms -kata-agent-socket /run/kata-containers/agent.sock $TLS_OPTIONS $OPTIONS +``` + +You may also include additional Inbounds and Outbounds configurations to the Forwarder using the `-secure-comms-inbounds` and `-secure-comms-outbounds` flags. [See more details regarding Inbounds and Outbounds below.](#adding-named-tunnels-to-the-ssh-channel) + +For example: +```sh +ExecStart=/usr/local/bin/agent-protocol-forwarder -kata-agent-namespace /run/netns/podns -secure-comms -secure-comms-inbounds KUBERNETES_PHASE:mytunnel:6666 -kata-agent-socket /run/kata-containers/agent.sock $TLS_OPTIONS $OPTIONS +``` + +Once you changed `podvm/files/etc/systemd/system/agent-protocol-forwarder.service`, you will need to [rebuild the podvm](./../podvm/README.md). + ## Testing Testing securecomms as a standalone can be done by using: diff --git a/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go b/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go index 2ff72a7db..0b49c516e 100644 --- a/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go +++ b/src/cloud-api-adaptor/pkg/adaptor/cloud/cloud.go @@ -287,7 +287,7 @@ func (s *cloudService) CreateVM(ctx context.Context, req *pb.CreateVMRequest) (r Content: agentConfig, }, { - Path: forwarder.DefaultConfigPath, + Path: forwarder.DefaultDaemonConfigPath, Content: string(daemonJSON), }, }, diff --git a/src/cloud-api-adaptor/pkg/forwarder/forwarder.go b/src/cloud-api-adaptor/pkg/forwarder/forwarder.go index fd8a694d0..e737520b7 100644 --- a/src/cloud-api-adaptor/pkg/forwarder/forwarder.go +++ b/src/cloud-api-adaptor/pkg/forwarder/forwarder.go @@ -27,13 +27,15 @@ const ( DefaultListenHost = "0.0.0.0" DefaultListenPort = "15150" DefaultListenAddr = DefaultListenHost + ":" + DefaultListenPort - DefaultConfigPath = "/run/peerpod/daemon.json" + DefaultDaemonConfigPath = "/run/peerpod/daemon.json" + DefaultAPFConfigPath = "/run/peerpod/apf.json" DefaultPodNetworkSpecPath = "/run/peerpod/podnetwork.json" DefaultKataAgentSocketPath = "/run/kata-containers/agent.sock" DefaultPodNamespace = "/run/netns/podns" AgentURLPath = "/agent" ) +// Unmeasured Configuration via User Data type Config struct { PodNetwork *tunneler.Config `json:"pod-network"` PodNamespace string `json:"pod-namespace"` @@ -44,6 +46,11 @@ type Config struct { TLSClientCA string `json:"tls-client-ca,omitempty"` } +// Measured Configuration via InitData +type ApfConfig struct { + SecureComms bool `json:"secure-comms,omitempty"` +} + type Daemon interface { Start(ctx context.Context) error Shutdown() error diff --git a/src/cloud-api-adaptor/pkg/userdata/provision.go b/src/cloud-api-adaptor/pkg/userdata/provision.go index 0b90bdb28..d28474664 100644 --- a/src/cloud-api-adaptor/pkg/userdata/provision.go +++ b/src/cloud-api-adaptor/pkg/userdata/provision.go @@ -32,8 +32,8 @@ const ( ) var logger = log.New(log.Writer(), "[userdata/provision] ", log.LstdFlags|log.Lmsgprefix) -var WriteFilesList = []string{cloud.AAConfigPath, cloud.CDHConfigPath, agent.ConfigFilePath, forwarder.DefaultConfigPath, cloud.AuthFilePath, cloud.InitdataPath} -var InitdDataFilesList = []string{cloud.AAConfigPath, cloud.CDHConfigPath, PolicyPath} +var WriteFilesList = []string{cloud.AAConfigPath, cloud.CDHConfigPath, agent.ConfigFilePath, forwarder.DefaultDaemonConfigPath, cloud.AuthFilePath, cloud.InitdataPath} +var InitdDataFilesList = []string{cloud.AAConfigPath, cloud.CDHConfigPath, PolicyPath, forwarder.DefaultAPFConfigPath} type Config struct { fetchTimeout int