tests/upgrade: Test encryption/decryption in upgrade test

.buildkite/code.pipeline.yml

@@ -382,6 +382,7 @@ steps:
       - label: E2E upgrade tests - sgx1
         timeout_in_minutes: 60
         command:
+          - trap 'buildkite-agent artifact upload "/tmp/oasis-{post,pre}-upgrade/e2e/**/*.log;/tmp/oasis-{post,pre}-upgrade/e2e/**/genesis.json;/tmp/oasis-{post,pre}-upgrade/e2e/**/runtime_genesis.json"' EXIT
           - .buildkite/scripts/download_e2e_test_artifacts.sh
           - .buildkite/scripts/test_upgrade.sh
         env:

.changelog/5399.feature.md

@@ -0,0 +1 @@
+tests/upgrade: Test encryption/decryption in upgrade test

tests/upgrade/post/go.mod

@@ -15,7 +15,7 @@ replace (
 	golang.org/x/crypto/ed25519 => github.com/oasisprotocol/curve25519-voi/primitives/ed25519 v0.0.0-20210505121811-294cf0fbfb43
 )
 
-require github.com/oasisprotocol/oasis-core/go v0.2202.1-0.20231010115211-e67f164f7842
+require github.com/oasisprotocol/oasis-core/go v0.2300.1-0.20231012072316-cf039bf94a0e
 
 require (
 	github.com/a8m/envsubst v1.4.2 // indirect

tests/upgrade/post/go.sum

@@ -510,10 +510,8 @@ github.com/oasisprotocol/cometbft v0.37.2-oasis1 h1:7yr/5f0vV1bQLKoZHFhs3mcHay3n
 github.com/oasisprotocol/cometbft v0.37.2-oasis1/go.mod h1:w7GZmT0jJ/i9Uz9ZWtzE3chuNZ3phUboSrFzPZIoYts=
 github.com/oasisprotocol/curve25519-voi v0.0.0-20230110094441-db37f07504ce h1:/pEpMk55wH0X+E5zedGEMOdLuWmV8P4+4W3+LZaM6kg=
 github.com/oasisprotocol/curve25519-voi v0.0.0-20230110094441-db37f07504ce/go.mod h1:hVoHR2EVESiICEMbg137etN/Lx+lSrHPTD39Z/uE+2s=
-github.com/oasisprotocol/oasis-core/go v0.2202.1-0.20231007081532-553e006f90d6 h1:nI4czq/c7ZapUqSJuE/LflHyWLu/LnJN81nOBRmOgLQ=
-github.com/oasisprotocol/oasis-core/go v0.2202.1-0.20231007081532-553e006f90d6/go.mod h1:lWZtDYY7ziW20hVzqnSt8UQlUkXNHisG3bS6Q3eTsqQ=
-github.com/oasisprotocol/oasis-core/go v0.2202.1-0.20231010115211-e67f164f7842 h1:2MXcN7dTHnHKjCDTVjP0011vtkYU1hQxqL4JmdQAU3Q=
-github.com/oasisprotocol/oasis-core/go v0.2202.1-0.20231010115211-e67f164f7842/go.mod h1:lWZtDYY7ziW20hVzqnSt8UQlUkXNHisG3bS6Q3eTsqQ=
+github.com/oasisprotocol/oasis-core/go v0.2300.1-0.20231012072316-cf039bf94a0e h1:M0/NXwj72rsdGnqeY6ZtYBjyFbFRtmR7Ccmmke1CdgI=
+github.com/oasisprotocol/oasis-core/go v0.2300.1-0.20231012072316-cf039bf94a0e/go.mod h1:lWZtDYY7ziW20hVzqnSt8UQlUkXNHisG3bS6Q3eTsqQ=
 github.com/oasisprotocol/safeopen v0.0.0-20200528085122-e01cfdfc7661 h1:MB73kGMtuMGS+6VDoU/mitzff7Cu+aZo9ta5wabuxVA=
 github.com/oasisprotocol/safeopen v0.0.0-20200528085122-e01cfdfc7661/go.mod h1:SwBxaVibf6Sr2IZ6M3WnUue0yp8dPLAo1riQRNQ60+g=
 github.com/oklog/run v1.0.0 h1:Ru7dDtJNOyC66gQ5dQmaCa0qIsAUFY3sFpK1Xk8igrw=

tests/upgrade/post/scenario/e2e/runtime/secure_upgrade.go

@@ -7,7 +7,9 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"time"
 
+	consensus "github.com/oasisprotocol/oasis-core/go/consensus/api"
 	"github.com/oasisprotocol/oasis-core/go/oasis-test-runner/env"
 	"github.com/oasisprotocol/oasis-core/go/oasis-test-runner/oasis"
 	"github.com/oasisprotocol/oasis-core/go/oasis-test-runner/oasis/cli"
@@ -23,35 +25,58 @@ var (
 	// consensus verification, and test the functionality of master secret rotations.
 	SecureUpgrade scenario.Scenario = newSecureUpgradeImpl()
 
-	secureUpgradePreTestScenario = runtime.NewTestClientScenario([]interface{}{
-		// Test that pre-upgrade key/value pairs are still in the database.
-		runtime.GetKeyValueTx{Key: "pre-key1", Response: "pre-value1", Encrypted: false},
-		runtime.GetKeyValueTx{Key: "pre-key2", Response: "pre-value2", Encrypted: true, Generation: 0},
-		runtime.GetKeyValueTx{Key: "pre-key1", Response: "", Encrypted: true, Generation: 0},
-		runtime.GetKeyValueTx{Key: "pre-key2", Response: "", Encrypted: false},
-		// Insert post-upgrade key/value pairs, one in plaintext and one encrypted.
-		runtime.InsertKeyValueTx{Key: "post-key1", Value: "post-value1", Response: "", Encrypted: false},
-		runtime.InsertKeyValueTx{Key: "post-key2", Value: "post-value2", Response: "", Encrypted: true, Generation: 0},
-		// Test that post-upgrade key/value pairs were correctly inserted into the database.
-		runtime.GetKeyValueTx{Key: "post-key1", Response: "post-value1", Encrypted: false},
-		runtime.GetKeyValueTx{Key: "post-key2", Response: "post-value2", Encrypted: true, Generation: 0},
-		// Test that post-upgrade key/value pairs are either in plaintext or encrypted.
-		runtime.GetKeyValueTx{Key: "post-key1", Response: "", Encrypted: true, Generation: 0},
-		runtime.GetKeyValueTx{Key: "post-key2", Response: "", Encrypted: false},
+	// verifyNodeUpgradeTestClientScenario tests that values inserted before node upgrade are still
+	// in the database.
+	verifyNodeUpgradeTestClientScenario = runtime.NewTestClientScenario([]interface{}{
+		runtime.GetKeyValueTx{Key: "node-key1", Response: "node-value1", Encrypted: false},
+		runtime.GetKeyValueTx{Key: "node-key2", Response: "node-value2", Encrypted: true, Generation: 0},
+		runtime.GetKeyValueTx{Key: "node-key1", Response: "", Encrypted: true, Generation: 0},
+		runtime.GetKeyValueTx{Key: "node-key2", Response: "", Encrypted: false},
 	})
 
-	secureUpgradePostTestScenario = runtime.NewTestClientScenario([]interface{}{
-		// Test that pre-upgrade key/value pairs are still in the database.
-		runtime.GetKeyValueTx{Key: "pre-key1", Response: "pre-value1", Encrypted: false},
-		runtime.GetKeyValueTx{Key: "pre-key2", Response: "pre-value2", Encrypted: true, Generation: 0},
-		runtime.GetKeyValueTx{Key: "pre-key1", Response: "", Encrypted: true, Generation: 0},
-		runtime.GetKeyValueTx{Key: "pre-key2", Response: "", Encrypted: false},
-		// Test that post-upgrade key/value pairs are still in the database.
-		runtime.GetKeyValueTx{Key: "post-key1", Response: "post-value1", Encrypted: false},
-		runtime.GetKeyValueTx{Key: "post-key2", Response: "post-value2", Encrypted: true, Generation: 0},
-		runtime.GetKeyValueTx{Key: "post-key1", Response: "", Encrypted: true, Generation: 0},
-		runtime.GetKeyValueTx{Key: "post-key2", Response: "", Encrypted: false},
-		// Test master secret generations.
+	// keyManagerUpgradeTestClientScenario inserts values before key manager upgrade.
+	keyManagerUpgradeTestClientScenario = runtime.NewTestClientScenario([]interface{}{
+		runtime.InsertKeyValueTx{Key: "km-key1", Value: "km-value1", Response: "", Encrypted: false},
+		runtime.InsertKeyValueTx{Key: "km-key2", Value: "km-value2", Response: "", Encrypted: true, Generation: 0},
+	})
+
+	// verifyKeyManagerUpgradeTestClientScenario tests that values inserted before key manager
+	// upgrade are still in the database.
+	verifyKeyManagerUpgradeTestClientScenario = runtime.NewTestClientScenario([]interface{}{
+		runtime.GetKeyValueTx{Key: "km-key1", Response: "km-value1", Encrypted: false},
+		runtime.GetKeyValueTx{Key: "km-key2", Response: "km-value2", Encrypted: true, Generation: 0},
+		runtime.GetKeyValueTx{Key: "km-key1", Response: "", Encrypted: true, Generation: 0},
+		runtime.GetKeyValueTx{Key: "km-key2", Response: "", Encrypted: false},
+	})
+
+	// runtimeUpgradeTestClientScenario inserts values before key-value runtime upgrade.
+	runtimeUpgradeTestClientScenario = runtime.NewTestClientScenario([]interface{}{
+		runtime.InsertKeyValueTx{Key: "rt-key1", Value: "rt-value1", Response: "", Encrypted: false},
+		runtime.InsertKeyValueTx{Key: "rt-key2", Value: "rt-value2", Response: "", Encrypted: true, Generation: 0},
+	})
+
+	// verifyRuntimeUpgradeTestClientScenario tests that values inserted before key-value runtime
+	// upgrade are still in the database.
+	verifyRuntimeUpgradeTestClientScenario = runtime.NewTestClientScenario([]interface{}{
+		runtime.GetKeyValueTx{Key: "rt-key1", Response: "rt-value1", Encrypted: false},
+		runtime.GetKeyValueTx{Key: "rt-key2", Response: "rt-value2", Encrypted: true, Generation: 0},
+		runtime.GetKeyValueTx{Key: "rt-key1", Response: "", Encrypted: true, Generation: 0},
+		runtime.GetKeyValueTx{Key: "rt-key2", Response: "", Encrypted: false},
+	})
+
+	// networkUpgradeTestClientScenario tests that everything works after the upgrade has been
+	// completed.
+	networkUpgradeTestClientScenario = runtime.NewTestClientScenario([]interface{}{
+		runtime.InsertKeyValueTx{Key: "network-key1", Value: "network-value1", Response: "", Encrypted: false},
+		runtime.InsertKeyValueTx{Key: "network-key2", Value: "network-value2", Response: "", Encrypted: true},
+		runtime.GetKeyValueTx{Key: "network-key1", Response: "network-value1", Encrypted: false},
+		runtime.GetKeyValueTx{Key: "network-key2", Response: "network-value2", Encrypted: true},
+		runtime.GetKeyValueTx{Key: "network-key1", Response: "", Encrypted: true},
+		runtime.GetKeyValueTx{Key: "network-key2", Response: "", Encrypted: false},
+	})
+
+	// masterSecretGenerationsTestClientScenario tests master secret generations.
+	masterSecretGenerationsTestClientScenario = runtime.NewTestClientScenario([]interface{}{
 		runtime.InsertKeyValueTx{Key: "msgn-key1", Value: "msgn-value1", Response: "", Encrypted: true, Generation: 0},
 		runtime.InsertKeyValueTx{Key: "msgn-key2", Value: "msgn-value2", Response: "", Encrypted: true, Generation: 1},
 		runtime.InsertKeyValueTx{Key: "msgn-key3", Value: "msgn-value3", Response: "", Encrypted: true, Generation: 2},
@@ -60,8 +85,9 @@ var (
 		runtime.GetKeyValueTx{Key: "msgn-key3", Response: "msgn-value3", Encrypted: true, Generation: 2},
 	})
 
-	invalidSecureUpgradeTestScenario = runtime.NewTestClientScenario([]interface{}{
-		// Test that key/value pair cannot be encrypted with a master secret from the future.
+	// futureMasterSecretGenerationsTestClientScenario tests that future master secrets are not
+	// available.
+	futureMasterSecretGenerationsTestClientScenario = runtime.NewTestClientScenario([]interface{}{
 		runtime.InsertKeyValueTx{Key: "msgn-key4", Value: "msgn-value4", Response: "", Encrypted: true, Generation: 1000},
 	})
 )
@@ -77,7 +103,7 @@ func newSecureUpgradeImpl() scenario.Scenario {
 	return &secureUpgradeImpl{
 		Scenario: *runtime.NewScenario(
 			"trust-root/secure-upgrade",
-			runtime.NewTestClient().WithSeed("post-seed").WithScenario(secureUpgradePreTestScenario),
+			runtime.NewTestClient().WithSeed("post-seed"),
 		),
 	}
 }
@@ -175,10 +201,20 @@ func (sc *secureUpgradeImpl) Run(ctx context.Context, childEnv *env.Env) error {
 		return fmt.Errorf("failed to refresh runtime bundle: %w", err)
 	}
 
-	// Start all workers and run the test client.
+	// Start all workers.
 	if err = sc.startClientAndComputeWorkers(ctx); err != nil {
 		return nil
 	}
+
+	encryptDecryptTestClientScenario := sc.newEncryptDecryptTestClientScenario(ctx, childEnv)
+
+	// Test after node upgrade.
+	sc.TestClient.WithScenario(runtime.JoinTestClientScenarios(
+		verifyNodeUpgradeTestClientScenario,
+		keyManagerUpgradeTestClientScenario,
+		verifyKeyManagerUpgradeTestClientScenario,
+		encryptDecryptTestClientScenario,
+	))
 	if err = sc.RunTestClientAndCheckLogs(ctx, childEnv); err != nil {
 		return err
 	}
@@ -208,38 +244,54 @@ func (sc *secureUpgradeImpl) Run(ctx context.Context, childEnv *env.Env) error {
 		return err
 	}
 
-	// Enable master secret rotations.
-	nonce, err = sc.TestEntityNonce(ctx)
-	if err != nil {
+	// Test after key manager upgrade.
+	sc.TestClient.WithScenario(runtime.JoinTestClientScenarios(
+		verifyKeyManagerUpgradeTestClientScenario,
+		runtimeUpgradeTestClientScenario,
+		verifyRuntimeUpgradeTestClientScenario,
+		encryptDecryptTestClientScenario,
+	))
+	if err = sc.RunTestClientAndCheckLogs(ctx, childEnv); err != nil {
 		return err
 	}
-	if err = sc.UpdateRotationInterval(ctx, childEnv, cli, 1, nonce); err != nil {
+
+	// Upgrade the compute runtime.
+	nonce, err = sc.TestEntityNonce(ctx)
+	if err != nil {
 		return err
 	}
-
-	// Wait until at least 3 secrets are generated.
-	if _, err = sc.WaitMasterSecret(ctx, 3); err != nil {
+	if err := sc.UpgradeComputeRuntime(ctx, childEnv, cli, sc.upgradedRuntimeIndex, nonce); err != nil {
 		return err
 	}
 
-	// Upgrade the compute runtime.
+	// Enable master secret rotations.
 	nonce, err = sc.TestEntityNonce(ctx)
 	if err != nil {
 		return err
 	}
-	if err := sc.UpgradeComputeRuntime(ctx, childEnv, cli, sc.upgradedRuntimeIndex, nonce); err != nil {
+	if err = sc.UpdateRotationInterval(ctx, childEnv, cli, 1, nonce); err != nil {
+		return err
+	}
+
+	// Wait until at least 3 secrets are generated.
+	if _, err = sc.WaitMasterSecret(ctx, 3); err != nil {
 		return err
 	}
 
-	// Run the test client again.
-	sc.TestClient.WithScenario(secureUpgradePostTestScenario)
+	// Test after key-value runtime upgrade.
+	sc.TestClient.WithScenario(runtime.JoinTestClientScenarios(
+		verifyRuntimeUpgradeTestClientScenario,
+		networkUpgradeTestClientScenario,
+		masterSecretGenerationsTestClientScenario,
+		encryptDecryptTestClientScenario,
+	))
 	if err = sc.RunTestClientAndCheckLogs(ctx, childEnv); err != nil {
 		return err
 	}
 
-	// Run the test client again, but expect it to fail this time as the requested master secret
-	// generation is not yet available.
-	sc.TestClient.WithScenario(invalidSecureUpgradeTestScenario)
+	// Test future master secrets, but expect the test to fail as the requested master secret
+	// generation should not be available.
+	sc.TestClient.WithScenario(futureMasterSecretGenerationsTestClientScenario)
 	err = sc.RunTestClientAndCheckLogs(ctx, childEnv)
 	switch {
 	case err == nil:
@@ -283,3 +335,24 @@ func (sc *secureUpgradeImpl) startClientAndComputeWorkers(ctx context.Context) e
 
 	return nil
 }
+
+func (sc *secureUpgradeImpl) newEncryptDecryptTestClientScenario(ctx context.Context, childEnv *env.Env) runtime.TestClientScenario {
+	return func(submit func(req interface{}) error) error {
+		// Fetch current epoch.
+		epoch, err := sc.Net.Controller().Beacon.GetEpoch(ctx, consensus.HeightLatest)
+		if err != nil {
+			return fmt.Errorf("failed to get current epoch: %w", err)
+		}
+
+		// Use unique values.
+		now := time.Now().UnixNano()
+		keyPairID := fmt.Sprintf("key-pair-id-%d", now)
+		message := fmt.Sprintf("message-%d", now)
+
+		return submit(runtime.EncryptDecryptTx{
+			Epoch:     epoch,
+			KeyPairID: keyPairID,
+			Message:   []byte(message),
+		})
+	}
+}

tests/upgrade/pre/scenario/e2e/runtime/secure_upgrade.go

@@ -21,16 +21,14 @@ var (
 	// consensus verification, and test the functionality of master secret rotations.
 	SecureUpgrade scenario.Scenario = newSecureUpgradeImpl()
 
-	secureUpgradeTestClientScenario = runtime.NewTestClientScenario([]interface{}{
-		// Insert pre-upgrade key/value pairs, one in plaintext and one encrypted.
-		runtime.InsertKeyValueTx{Key: "pre-key1", Value: "pre-value1", Response: "", Encrypted: false},
-		runtime.InsertKeyValueTx{Key: "pre-key2", Value: "pre-value2", Response: "", Encrypted: true},
-		// Test that pre-upgrade key/value pairs were correctly inserted into the database.
-		runtime.GetKeyValueTx{Key: "pre-key1", Response: "pre-value1", Encrypted: false},
-		runtime.GetKeyValueTx{Key: "pre-key2", Response: "pre-value2", Encrypted: true},
-		// Test that pre-upgrade key/value pairs are either in plaintext or encrypted.
-		runtime.GetKeyValueTx{Key: "pre-key1", Response: "", Encrypted: true},
-		runtime.GetKeyValueTx{Key: "pre-key2", Response: "", Encrypted: false},
+	// nodeUpgradeTestClientScenario tests that everything works before the upgrade starts.
+	nodeUpgradeTestClientScenario = runtime.NewTestClientScenario([]interface{}{
+		runtime.InsertKeyValueTx{Key: "node-key1", Value: "node-value1", Response: "", Encrypted: false},
+		runtime.InsertKeyValueTx{Key: "node-key2", Value: "node-value2", Response: "", Encrypted: true},
+		runtime.GetKeyValueTx{Key: "node-key1", Response: "node-value1", Encrypted: false},
+		runtime.GetKeyValueTx{Key: "node-key2", Response: "node-value2", Encrypted: true},
+		runtime.GetKeyValueTx{Key: "node-key1", Response: "", Encrypted: true},
+		runtime.GetKeyValueTx{Key: "node-key2", Response: "", Encrypted: false},
 	})
 )
 
@@ -42,7 +40,7 @@ func newSecureUpgradeImpl() scenario.Scenario {
 	return &secureUpgradeImpl{
 		TrustRootImpl: *runtime.NewTrustRootImpl(
 			"secure-upgrade",
-			runtime.NewKVTestClient().WithScenario(secureUpgradeTestClientScenario),
+			runtime.NewKVTestClient().WithScenario(nodeUpgradeTestClientScenario),
 		),
 	}
 }