tests: Test key manager upgrade

.buildkite/code.pipeline.yml

@@ -390,3 +390,37 @@ steps:
       <<: *retry_agent_failure
     plugins:
       <<: *docker_plugin
+
+  ######################################################
+  # Key manager upgrade test - sgx1 (triggered manually)
+  ######################################################
+  - group: "Key manager upgrade test - sgx1"
+    depends_on:
+      - "build-go"
+      - "build-rust-runtime-loader"
+      - "build-rust-runtimes"
+    steps:
+      - block: "Confirm key manager upgrade test run"
+        prompt: "Run key manager upgrade test for this pull request"
+        blocked_state: passed
+
+      - label: Key manager upgrade test - sgx1
+        timeout_in_minutes: 20
+        command:
+          - .buildkite/scripts/download_e2e_test_artifacts.sh
+          - .buildkite/scripts/test_km_upgrade.sh --scenario e2e.runtime.keymanager-upgrade
+        artifact_paths:
+          - /tmp/e2e/**/*.log
+          - /tmp/e2e/**/genesis.json
+          - /tmp/e2e/**/runtime_genesis.json
+        env:
+          # Unsafe flags needed as the trust-root test rebuilds the enclave with embedded trust root data.
+          OASIS_UNSAFE_SKIP_AVR_VERIFY: "1"
+          OASIS_UNSAFE_ALLOW_DEBUG_ENCLAVES: "1"
+          TEST_BASE_DIR: /tmp
+        agents:
+          queue: sgx1
+        retry:
+          <<: *retry_agent_failure
+        plugins:
+          <<: *docker_plugin_sgx1

.buildkite/scripts/test_e2e.sh

@@ -49,7 +49,9 @@ ${test_runner_binary} \
     --basedir.no_cleanup \
     --e2e.node.binary ${node_binary} \
     --e2e/runtime.runtime.binary_dir.default ${WORKDIR}/target/default/debug \
+    --e2e/runtime.runtime.binary_dir.default.upgrade ${WORKDIR}/target/default/debug \
     --e2e/runtime.runtime.binary_dir.intel-sgx ${WORKDIR}/target/sgx/x86_64-fortanix-unknown-sgx/debug \
+    --e2e/runtime.runtime.binary_dir.intel-sgx.upgrade ${WORKDIR}/target/sgx/x86_64-fortanix-unknown-sgx/debug \
     --e2e/runtime.runtime.source_dir ${WORKDIR}/tests/runtimes \
     --e2e/runtime.runtime.target_dir ${WORKDIR}/target \
     --e2e/runtime.runtime.loader ${WORKDIR}/target/default/debug/oasis-core-runtime-loader \

.buildkite/scripts/test_km_upgrade.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+
+############################################################
+# This script tests the Oasis Core project key manager 
+# upgrades.
+#
+# Usage:
+# test_km_upgrade.sh
+############################################################
+
+# Helpful tips on writing build scripts:
+# https://buildkite.com/docs/pipelines/writing-build-scripts
+set -euxo pipefail
+
+# Working directory.
+WORKDIR=$PWD
+
+node_binary="${WORKDIR}/go/oasis-node/oasis-node"
+test_runner_binary="${WORKDIR}/go/oasis-test-runner/oasis-test-runner"
+
+ias_mock="true"
+set +x
+if [[ ${OASIS_IAS_APIKEY:-""} != "" ]]; then
+    set -x
+    ias_mock="false"
+fi
+set -x
+
+# Branch to test against.
+git_branch="stable/22.2.x"
+
+# Temporary directory for building the branch.
+DATADIR=${TEST_BASE_DIR:-"/tmp"}/oasis-km-upgrade/oasis-core
+
+# Remove old data.
+echo "Removing old data..."
+
+rm -rf "${DATADIR}"
+mkdir -p "${DATADIR}"
+
+# Download and build the branch.
+echo "Downloading and building oasis-core ${git_branch} branch"
+
+git clone https://github.com/oasisprotocol/oasis-core -b "${git_branch}" "${DATADIR}"
+pushd "${DATADIR}"
+    make build-runtimes
+popd
+
+# Run Oasis test runner.
+${test_runner_binary} \
+    ${BUILDKITE:+--basedir ${TEST_BASE_DIR:-$PWD}/e2e} \
+    --basedir.no_cleanup \
+    --e2e.node.binary ${node_binary} \
+    --e2e/runtime.runtime.binary_dir.default ${DATADIR}/target/default/debug \
+    --e2e/runtime.runtime.binary_dir.default.upgrade ${WORKDIR}/target/default/debug \
+    --e2e/runtime.runtime.binary_dir.intel-sgx ${DATADIR}/target/sgx/x86_64-fortanix-unknown-sgx/debug \
+    --e2e/runtime.runtime.binary_dir.intel-sgx.upgrade ${WORKDIR}/target/sgx/x86_64-fortanix-unknown-sgx/debug \
+    --e2e/runtime.runtime.source_dir ${WORKDIR}/tests/runtimes \
+    --e2e/runtime.runtime.target_dir ${WORKDIR}/target \
+    --e2e/runtime.runtime.loader ${WORKDIR}/target/default/debug/oasis-core-runtime-loader \
+    --e2e/runtime.tee_hardware ${OASIS_TEE_HARDWARE:-""} \
+    --e2e/runtime.ias.mock=${ias_mock} \
+    --remote-signer.binary ${WORKDIR}/go/oasis-remote-signer/oasis-remote-signer \
+    --plugin-signer.name example \
+    --plugin-signer.binary ${WORKDIR}/go/oasis-test-runner/scenario/pluginsigner/example_signer_plugin/example_signer_plugin \
+    --log.level debug \
+    "$@"

go/oasis-test-runner/scenario/e2e/runtime/keymanager_upgrade.go

@@ -45,7 +45,7 @@ func (sc *kmUpgradeImpl) Fixture() (*oasis.NetworkFixture, error) {
 	}
 
 	// Load the upgraded keymanager binary.
-	newKmBinaries := sc.resolveRuntimeBinaries("simple-keymanager-upgrade")
+	newKmBinaries := sc.ResolveRuntimeUpgradeBinaries("simple-keymanager-upgrade")
 	// Setup the upgraded runtime.
 	kmRuntimeFix := f.Runtimes[0]
 	if kmRuntimeFix.Kind != registry.KindKeyManager {

go/oasis-test-runner/scenario/e2e/runtime/runtime.go

@@ -28,14 +28,16 @@ import (
 )
 
 const (
-	cfgRuntimeBinaryDirDefault  = "runtime.binary_dir.default"
-	cfgRuntimeBinaryDirIntelSGX = "runtime.binary_dir.intel-sgx"
-	cfgRuntimeSourceDir         = "runtime.source_dir"
-	cfgRuntimeTargetDir         = "runtime.target_dir"
-	cfgRuntimeLoader            = "runtime.loader"
-	cfgTEEHardware              = "tee_hardware"
-	cfgIasMock                  = "ias.mock"
-	cfgEpochInterval            = "epoch.interval"
+	cfgRuntimeBinaryDirDefault         = "runtime.binary_dir.default"
+	cfgRuntimeBinaryDirDefaultUpgrade  = "runtime.binary_dir.default.upgrade"
+	cfgRuntimeBinaryDirIntelSGX        = "runtime.binary_dir.intel-sgx"
+	cfgRuntimeBinaryDirIntelSGXUpgrade = "runtime.binary_dir.intel-sgx.upgrade"
+	cfgRuntimeSourceDir                = "runtime.source_dir"
+	cfgRuntimeTargetDir                = "runtime.target_dir"
+	cfgRuntimeLoader                   = "runtime.loader"
+	cfgTEEHardware                     = "tee_hardware"
+	cfgIasMock                         = "ias.mock"
+	cfgEpochInterval                   = "epoch.interval"
 )
 
 var (
@@ -125,7 +127,9 @@ func NewScenario(name string, testClient TestClient) *Scenario {
 		testClient: testClient,
 	}
 	sc.Flags.String(cfgRuntimeBinaryDirDefault, "", "(no-TEE) path to the runtime binaries directory")
+	sc.Flags.String(cfgRuntimeBinaryDirDefaultUpgrade, "", "(no-TEE) path to the runtime upgrade binaries directory")
 	sc.Flags.String(cfgRuntimeBinaryDirIntelSGX, "", "(Intel SGX) path to the runtime binaries directory")
+	sc.Flags.String(cfgRuntimeBinaryDirIntelSGXUpgrade, "", "(Intel SGX) path to the runtime upgrade binaries directory")
 	sc.Flags.String(cfgRuntimeSourceDir, "", "path to the runtime source base dir")
 	sc.Flags.String(cfgRuntimeTargetDir, "", "path to the Cargo target dir (should be a parent of the runtime binary dir)")
 	sc.Flags.String(cfgRuntimeLoader, "oasis-core-runtime-loader", "path to the runtime loader")
@@ -200,7 +204,7 @@ func (sc *Scenario) Fixture() (*oasis.NetworkFixture, error) {
 				GovernanceModel: registry.GovernanceEntity,
 				Deployments: []oasis.DeploymentCfg{
 					{
-						Binaries: sc.resolveRuntimeBinaries(keyManagerBinary),
+						Binaries: sc.ResolveRuntimeBinaries(keyManagerBinary),
 					},
 				},
 			},
@@ -243,7 +247,7 @@ func (sc *Scenario) Fixture() (*oasis.NetworkFixture, error) {
 				GovernanceModel: registry.GovernanceEntity,
 				Deployments: []oasis.DeploymentCfg{
 					{
-						Binaries: sc.resolveRuntimeBinaries(runtimeBinary),
+						Binaries: sc.ResolveRuntimeBinaries(runtimeBinary),
 					},
 				},
 			},
@@ -303,26 +307,58 @@ func (sc *Scenario) getTEEHardware() (node.TEEHardware, error) {
 	return tee, nil
 }
 
-func (sc *Scenario) resolveRuntimeBinaries(baseRuntimeBinary string) map[node.TEEHardware]string {
+// ResolveRuntimeBinaries returns the paths to the runtime binaries.
+func (sc *Scenario) ResolveRuntimeBinaries(runtimeBinary string) map[node.TEEHardware]string {
+	return sc.resolveRuntimeBinaries(runtimeBinary, false)
+}
+
+// ResolveRuntimeUpgradeBinaries returns the paths to the runtime upgrade binaries.
+func (sc *Scenario) ResolveRuntimeUpgradeBinaries(runtimeBinary string) map[node.TEEHardware]string {
+	return sc.resolveRuntimeBinaries(runtimeBinary, true)
+}
+
+// resolveRuntimeBinaries returns the paths to the runtime (upgrade) binaries.
+func (sc *Scenario) resolveRuntimeBinaries(runtimeBinary string, upgrade bool) map[node.TEEHardware]string {
 	binaries := make(map[node.TEEHardware]string)
 	for _, tee := range []node.TEEHardware{
 		node.TEEHardwareInvalid,
 		node.TEEHardwareIntelSGX,
 	} {
-		binaries[tee] = sc.resolveRuntimeBinary(baseRuntimeBinary, tee)
+		binaries[tee] = sc.resolveRuntimeBinary(runtimeBinary, tee, upgrade)
 	}
 	return binaries
 }
 
-func (sc *Scenario) resolveRuntimeBinary(runtimeBinary string, tee node.TEEHardware) string {
+// ResolveRuntimeBinary returns the path to the runtime binary.
+func (sc *Scenario) ResolveRuntimeBinary(runtimeBinary string, tee node.TEEHardware) string {
+	return sc.resolveRuntimeBinary(runtimeBinary, tee, false)
+}
+
+// ResolveRuntimeUpgradeBinary returns the path to the runtime upgrade binary.
+func (sc *Scenario) ResolveRuntimeUpgradeBinary(runtimeBinary string, tee node.TEEHardware) string {
+	return sc.resolveRuntimeBinary(runtimeBinary, tee, true)
+}
+
+// resolveRuntimeBinary returns the path to the runtime (upgrade) binary.
+func (sc *Scenario) resolveRuntimeBinary(runtimeBinary string, tee node.TEEHardware, upgrade bool) string {
 	var runtimeExt, path string
 	switch tee {
 	case node.TEEHardwareInvalid:
 		runtimeExt = ""
-		path, _ = sc.Flags.GetString(cfgRuntimeBinaryDirDefault)
+		switch upgrade {
+		case true:
+			path, _ = sc.Flags.GetString(cfgRuntimeBinaryDirDefaultUpgrade)
+		case false:
+			path, _ = sc.Flags.GetString(cfgRuntimeBinaryDirDefault)
+		}
 	case node.TEEHardwareIntelSGX:
 		runtimeExt = ".sgxs"
-		path, _ = sc.Flags.GetString(cfgRuntimeBinaryDirIntelSGX)
+		switch upgrade {
+		case true:
+			path, _ = sc.Flags.GetString(cfgRuntimeBinaryDirIntelSGXUpgrade)
+		case false:
+			path, _ = sc.Flags.GetString(cfgRuntimeBinaryDirIntelSGX)
+		}
 	}
 
 	return filepath.Join(path, runtimeBinary+runtimeExt)

go/oasis-test-runner/scenario/e2e/runtime/runtime_upgrade.go

@@ -57,7 +57,7 @@ func (sc *runtimeUpgradeImpl) Fixture() (*oasis.NetworkFixture, error) {
 	}
 
 	// Load the upgraded runtime binary.
-	newRuntimeBinaries := sc.resolveRuntimeBinaries("simple-keyvalue-upgrade")
+	newRuntimeBinaries := sc.ResolveRuntimeUpgradeBinaries("simple-keyvalue-upgrade")
 
 	// Setup the upgraded runtime (first is keymanager, others should be generic compute).
 	runtimeFix := f.Runtimes[computeIndex]