From da106472b6caef4399903a2a1129e110d53276ca Mon Sep 17 00:00:00 2001 From: Thomas Bazin Date: Wed, 26 Jun 2024 09:48:38 +0200 Subject: [PATCH 1/3] Add a new implementation of SecretPersistence to store secrets into an Azure Key Vault. Aribyte secret's versions are not handled properly as it does not work in the same way into AKV. --- .../main/java/io/airbyte/config/Configs.java | 3 +- .../config-secrets/build.gradle.kts | 3 + .../AzureSecretManagerPersistence.java | 90 +++++++++++++++++++ 3 files changed, 95 insertions(+), 1 deletion(-) create mode 100644 airbyte-config/config-secrets/src/main/java/io/airbyte/config/secrets/persistence/AzureSecretManagerPersistence.java diff --git a/airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java b/airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java index eb0ffbe51d9..a6b21dcea80 100644 --- a/airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java +++ b/airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java @@ -192,7 +192,8 @@ enum SecretPersistenceType { TESTING_CONFIG_DB_TABLE, GOOGLE_SECRET_MANAGER, VAULT, - AWS_SECRET_MANAGER + AWS_SECRET_MANAGER, + AZURE_SECRET_MANAGER } /** diff --git a/airbyte-config/config-secrets/build.gradle.kts b/airbyte-config/config-secrets/build.gradle.kts index b8f5b8ea32d..08b7685ba3d 100644 --- a/airbyte-config/config-secrets/build.gradle.kts +++ b/airbyte-config/config-secrets/build.gradle.kts @@ -24,6 +24,9 @@ dependencies { api(libs.aws.java.sdk.sts) api(project(":airbyte-commons")) + implementation("com.azure:azure-security-keyvault-secrets:4.8.3") + implementation("com.azure:azure-identity:1.13.0") + /* * Marked as "implementation" to avoid leaking these dependencies to services * that only use the retrieval side of the secret infrastructure. The services diff --git a/airbyte-config/config-secrets/src/main/java/io/airbyte/config/secrets/persistence/AzureSecretManagerPersistence.java b/airbyte-config/config-secrets/src/main/java/io/airbyte/config/secrets/persistence/AzureSecretManagerPersistence.java new file mode 100644 index 00000000000..48b7a6eb290 --- /dev/null +++ b/airbyte-config/config-secrets/src/main/java/io/airbyte/config/secrets/persistence/AzureSecretManagerPersistence.java @@ -0,0 +1,90 @@ +package io.airbyte.config.secrets.persistence; + +import com.azure.core.exception.ResourceNotFoundException; +import com.azure.identity.DefaultAzureCredentialBuilder; +import com.azure.identity.IntelliJCredential; +import com.azure.identity.IntelliJCredentialBuilder; +import com.azure.security.keyvault.secrets.SecretClient; +import com.azure.security.keyvault.secrets.SecretClientBuilder; +import com.azure.security.keyvault.secrets.models.KeyVaultSecret; +import io.airbyte.config.secrets.SecretCoordinate; +import io.micronaut.context.annotation.Requires; +import io.micronaut.context.annotation.Value; +import jakarta.inject.Named; +import jakarta.inject.Singleton; + +import java.time.Instant; +import java.time.ZoneOffset; + +/* +https://learn.microsoft.com/en-us/java/api/overview/azure/security-keyvault-secrets-readme?view=azure-java-stable +https://learn.microsoft.com/en-us/java/api/overview/azure/identity-readme?view=azure-java-stable#defaultazurecredential + */ +@Singleton +@Requires(property = "airbyte.secret.persistence", pattern = "(?i)^azure_secret_manager$") +@Named("secretPersistence") +public class AzureSecretManagerPersistence implements SecretPersistence { + + @Value("${airbyte.secret.store.azure.key-vault-url}") + private String keyVaultUrl; + + // FIXME bazint create a client each time ? + private SecretClient secretClient; + + @Override + public void initialize() { + secretClient = new SecretClientBuilder() + .vaultUrl(keyVaultUrl) + // new ManagedIdentityCredentialBuilder().build() + .credential( + // new DefaultAzureCredentialBuilder().build() + new IntelliJCredentialBuilder().build() + ) + .buildClient(); + } + + @Override + public String read(SecretCoordinate coordinate) { + try { + return secretClient.getSecret(coordinate.getCoordinateBase()).getValue(); + } catch (ResourceNotFoundException e) { + return ""; + } + } + + @Override + public void write(SecretCoordinate coordinate, String payload) { + writeWithExpiry(coordinate, payload, null); + } + + @Override + public void writeWithExpiry(SecretCoordinate coordinate, String payload, Instant expiry) { + var secret = new KeyVaultSecret(coordinate.getCoordinateBase(), payload); + if (expiry != null) + secret.getProperties().setExpiresOn(expiry.atOffset(ZoneOffset.UTC)); + secretClient.setSecret(secret); + } + + @Override + public void delete(SecretCoordinate coordinate) { + // KeyVaultErrorException: Status code 409, + // "{"error":{"code":"Conflict","message":"Secret plop is currently in a deleted but recoverable state, and its name cannot be reused; + // in this state, the secret can only be recovered or purged.","innererror":{"code":"ObjectIsDeletedButRecoverable"}}}" + var poller = secretClient.beginDeleteSecret(coordinate.getCoordinateBase()); + poller.poll(); + poller.waitForCompletion(); + } + + @Override + public void disable(SecretCoordinate coordinate) { + } + + public static void main(String[] args) { + AzureSecretManagerPersistence persistence = new AzureSecretManagerPersistence(); + persistence.keyVaultUrl = "https://poc-a7h-kv.vault.azure.net/"; + persistence.initialize(); + persistence.write(new SecretCoordinate("plop", 1), "my secret !"); + // persistence.delete(new SecretCoordinate("plop", 1)); + } + +} From dea4d19e7db1bbc85cbc1af32e25c1a4a658caa9 Mon Sep 17 00:00:00 2001 From: Thomas Bazin Date: Wed, 26 Jun 2024 11:17:39 +0200 Subject: [PATCH 2/3] Create a Micronaut test to check code instead of main method. --- .../AzureSecretManagerPersistence.java | 68 ++++++++++++------- .../AzureSecretManagerPersistenceTest.java | 33 +++++++++ 2 files changed, 78 insertions(+), 23 deletions(-) create mode 100644 airbyte-config/config-secrets/src/test/java/io/airbyte/config/secrets/persistence/AzureSecretManagerPersistenceTest.java diff --git a/airbyte-config/config-secrets/src/main/java/io/airbyte/config/secrets/persistence/AzureSecretManagerPersistence.java b/airbyte-config/config-secrets/src/main/java/io/airbyte/config/secrets/persistence/AzureSecretManagerPersistence.java index 48b7a6eb290..048c786a89d 100644 --- a/airbyte-config/config-secrets/src/main/java/io/airbyte/config/secrets/persistence/AzureSecretManagerPersistence.java +++ b/airbyte-config/config-secrets/src/main/java/io/airbyte/config/secrets/persistence/AzureSecretManagerPersistence.java @@ -1,15 +1,16 @@ package io.airbyte.config.secrets.persistence; import com.azure.core.exception.ResourceNotFoundException; -import com.azure.identity.DefaultAzureCredentialBuilder; -import com.azure.identity.IntelliJCredential; import com.azure.identity.IntelliJCredentialBuilder; +import com.azure.identity.ManagedIdentityCredentialBuilder; import com.azure.security.keyvault.secrets.SecretClient; import com.azure.security.keyvault.secrets.SecretClientBuilder; import com.azure.security.keyvault.secrets.models.KeyVaultSecret; import io.airbyte.config.secrets.SecretCoordinate; import io.micronaut.context.annotation.Requires; import io.micronaut.context.annotation.Value; +import io.micronaut.context.env.Environment; +import jakarta.inject.Inject; import jakarta.inject.Named; import jakarta.inject.Singleton; @@ -25,28 +26,25 @@ @Named("secretPersistence") public class AzureSecretManagerPersistence implements SecretPersistence { - @Value("${airbyte.secret.store.azure.key-vault-url}") - private String keyVaultUrl; + private final String keyVaultUrl; - // FIXME bazint create a client each time ? - private SecretClient secretClient; + @Inject + private final AzureKeyVaultClientBuilder clientBuilder; + + public AzureSecretManagerPersistence(@Value("${airbyte.secret.store.azure.key-vault-url}") String keyVaultUrl, + AzureKeyVaultClientBuilder clientBuilder) { + this.keyVaultUrl = keyVaultUrl; + this.clientBuilder = clientBuilder; + } @Override public void initialize() { - secretClient = new SecretClientBuilder() - .vaultUrl(keyVaultUrl) - // new ManagedIdentityCredentialBuilder().build() - .credential( - // new DefaultAzureCredentialBuilder().build() - new IntelliJCredentialBuilder().build() - ) - .buildClient(); } @Override public String read(SecretCoordinate coordinate) { try { - return secretClient.getSecret(coordinate.getCoordinateBase()).getValue(); + return clientBuilder.build(keyVaultUrl).getSecret(coordinate.getCoordinateBase()).getValue(); } catch (ResourceNotFoundException e) { return ""; } @@ -62,7 +60,7 @@ public void writeWithExpiry(SecretCoordinate coordinate, String payload, Instant var secret = new KeyVaultSecret(coordinate.getCoordinateBase(), payload); if (expiry != null) secret.getProperties().setExpiresOn(expiry.atOffset(ZoneOffset.UTC)); - secretClient.setSecret(secret); + clientBuilder.build(keyVaultUrl).setSecret(secret); } @Override @@ -70,7 +68,7 @@ public void delete(SecretCoordinate coordinate) { // KeyVaultErrorException: Status code 409, // "{"error":{"code":"Conflict","message":"Secret plop is currently in a deleted but recoverable state, and its name cannot be reused; // in this state, the secret can only be recovered or purged.","innererror":{"code":"ObjectIsDeletedButRecoverable"}}}" - var poller = secretClient.beginDeleteSecret(coordinate.getCoordinateBase()); + var poller = clientBuilder.build(keyVaultUrl).beginDeleteSecret(coordinate.getCoordinateBase()); poller.poll(); poller.waitForCompletion(); } @@ -79,12 +77,36 @@ public void delete(SecretCoordinate coordinate) { public void disable(SecretCoordinate coordinate) { } - public static void main(String[] args) { - AzureSecretManagerPersistence persistence = new AzureSecretManagerPersistence(); - persistence.keyVaultUrl = "https://poc-a7h-kv.vault.azure.net/"; - persistence.initialize(); - persistence.write(new SecretCoordinate("plop", 1), "my secret !"); - // persistence.delete(new SecretCoordinate("plop", 1)); + public interface AzureKeyVaultClientBuilder { + SecretClient build(String keyVaultUrl); + } + + @Singleton + @Requires(notEnv = Environment.TEST) + public static class ClientBuilder implements AzureKeyVaultClientBuilder { + + @Override + public SecretClient build(String keyVaultUrl) { + return new SecretClientBuilder() + .vaultUrl(keyVaultUrl) + .credential(new ManagedIdentityCredentialBuilder().build()) + .buildClient(); + } + + } + + @Singleton + @Requires(env = Environment.TEST) + public static class TestClientBuilder implements AzureKeyVaultClientBuilder { + + @Override + public SecretClient build(String keyVaultUrl) { + return new SecretClientBuilder() + .vaultUrl(keyVaultUrl) + .credential(new IntelliJCredentialBuilder().build()) + .buildClient(); + } + } } diff --git a/airbyte-config/config-secrets/src/test/java/io/airbyte/config/secrets/persistence/AzureSecretManagerPersistenceTest.java b/airbyte-config/config-secrets/src/test/java/io/airbyte/config/secrets/persistence/AzureSecretManagerPersistenceTest.java new file mode 100644 index 00000000000..8804eccc9ad --- /dev/null +++ b/airbyte-config/config-secrets/src/test/java/io/airbyte/config/secrets/persistence/AzureSecretManagerPersistenceTest.java @@ -0,0 +1,33 @@ +package io.airbyte.config.secrets.persistence; + +import com.azure.identity.IntelliJCredentialBuilder; +import com.azure.security.keyvault.secrets.SecretClientBuilder; +import io.airbyte.config.secrets.SecretCoordinate; +import io.micronaut.context.annotation.Property; +import io.micronaut.context.annotation.Replaces; +import io.micronaut.test.extensions.junit5.annotation.MicronautTest; +import jakarta.inject.Inject; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +@MicronautTest +@Property(name = "airbyte.secret.persistence", value = "azure_secret_manager") +@Property(name = "airbyte.secret.store.azure.key-vault-url", value = "https://poc-a7h-kv.vault.azure.net/") +public class AzureSecretManagerPersistenceTest { + + @Inject + SecretPersistence persistence; + + @Test + public void testWrite() { + var coordinate = new SecretCoordinate("plop", 1); + var secret = persistence.read(coordinate); + assertTrue(secret.isEmpty()); + persistence.write(coordinate, "my secret !"); + secret = persistence.read(coordinate); + assertEquals(secret, "my secret !"); + } + +} From 8b01b6b277deede31efea7861e406211ee080a1e Mon Sep 17 00:00:00 2001 From: Thomas Bazin Date: Wed, 26 Jun 2024 11:19:00 +0200 Subject: [PATCH 3/3] Rename classes for consistency --- .../AzureSecretManagerPersistence.java | 112 ------------------ .../AzureSecretManagerPersistenceTest.java | 33 ------ 2 files changed, 145 deletions(-) delete mode 100644 airbyte-config/config-secrets/src/main/java/io/airbyte/config/secrets/persistence/AzureSecretManagerPersistence.java delete mode 100644 airbyte-config/config-secrets/src/test/java/io/airbyte/config/secrets/persistence/AzureSecretManagerPersistenceTest.java diff --git a/airbyte-config/config-secrets/src/main/java/io/airbyte/config/secrets/persistence/AzureSecretManagerPersistence.java b/airbyte-config/config-secrets/src/main/java/io/airbyte/config/secrets/persistence/AzureSecretManagerPersistence.java deleted file mode 100644 index 048c786a89d..00000000000 --- a/airbyte-config/config-secrets/src/main/java/io/airbyte/config/secrets/persistence/AzureSecretManagerPersistence.java +++ /dev/null @@ -1,112 +0,0 @@ -package io.airbyte.config.secrets.persistence; - -import com.azure.core.exception.ResourceNotFoundException; -import com.azure.identity.IntelliJCredentialBuilder; -import com.azure.identity.ManagedIdentityCredentialBuilder; -import com.azure.security.keyvault.secrets.SecretClient; -import com.azure.security.keyvault.secrets.SecretClientBuilder; -import com.azure.security.keyvault.secrets.models.KeyVaultSecret; -import io.airbyte.config.secrets.SecretCoordinate; -import io.micronaut.context.annotation.Requires; -import io.micronaut.context.annotation.Value; -import io.micronaut.context.env.Environment; -import jakarta.inject.Inject; -import jakarta.inject.Named; -import jakarta.inject.Singleton; - -import java.time.Instant; -import java.time.ZoneOffset; - -/* -https://learn.microsoft.com/en-us/java/api/overview/azure/security-keyvault-secrets-readme?view=azure-java-stable -https://learn.microsoft.com/en-us/java/api/overview/azure/identity-readme?view=azure-java-stable#defaultazurecredential - */ -@Singleton -@Requires(property = "airbyte.secret.persistence", pattern = "(?i)^azure_secret_manager$") -@Named("secretPersistence") -public class AzureSecretManagerPersistence implements SecretPersistence { - - private final String keyVaultUrl; - - @Inject - private final AzureKeyVaultClientBuilder clientBuilder; - - public AzureSecretManagerPersistence(@Value("${airbyte.secret.store.azure.key-vault-url}") String keyVaultUrl, - AzureKeyVaultClientBuilder clientBuilder) { - this.keyVaultUrl = keyVaultUrl; - this.clientBuilder = clientBuilder; - } - - @Override - public void initialize() { - } - - @Override - public String read(SecretCoordinate coordinate) { - try { - return clientBuilder.build(keyVaultUrl).getSecret(coordinate.getCoordinateBase()).getValue(); - } catch (ResourceNotFoundException e) { - return ""; - } - } - - @Override - public void write(SecretCoordinate coordinate, String payload) { - writeWithExpiry(coordinate, payload, null); - } - - @Override - public void writeWithExpiry(SecretCoordinate coordinate, String payload, Instant expiry) { - var secret = new KeyVaultSecret(coordinate.getCoordinateBase(), payload); - if (expiry != null) - secret.getProperties().setExpiresOn(expiry.atOffset(ZoneOffset.UTC)); - clientBuilder.build(keyVaultUrl).setSecret(secret); - } - - @Override - public void delete(SecretCoordinate coordinate) { - // KeyVaultErrorException: Status code 409, - // "{"error":{"code":"Conflict","message":"Secret plop is currently in a deleted but recoverable state, and its name cannot be reused; - // in this state, the secret can only be recovered or purged.","innererror":{"code":"ObjectIsDeletedButRecoverable"}}}" - var poller = clientBuilder.build(keyVaultUrl).beginDeleteSecret(coordinate.getCoordinateBase()); - poller.poll(); - poller.waitForCompletion(); - } - - @Override - public void disable(SecretCoordinate coordinate) { - } - - public interface AzureKeyVaultClientBuilder { - SecretClient build(String keyVaultUrl); - } - - @Singleton - @Requires(notEnv = Environment.TEST) - public static class ClientBuilder implements AzureKeyVaultClientBuilder { - - @Override - public SecretClient build(String keyVaultUrl) { - return new SecretClientBuilder() - .vaultUrl(keyVaultUrl) - .credential(new ManagedIdentityCredentialBuilder().build()) - .buildClient(); - } - - } - - @Singleton - @Requires(env = Environment.TEST) - public static class TestClientBuilder implements AzureKeyVaultClientBuilder { - - @Override - public SecretClient build(String keyVaultUrl) { - return new SecretClientBuilder() - .vaultUrl(keyVaultUrl) - .credential(new IntelliJCredentialBuilder().build()) - .buildClient(); - } - - } - -} diff --git a/airbyte-config/config-secrets/src/test/java/io/airbyte/config/secrets/persistence/AzureSecretManagerPersistenceTest.java b/airbyte-config/config-secrets/src/test/java/io/airbyte/config/secrets/persistence/AzureSecretManagerPersistenceTest.java deleted file mode 100644 index 8804eccc9ad..00000000000 --- a/airbyte-config/config-secrets/src/test/java/io/airbyte/config/secrets/persistence/AzureSecretManagerPersistenceTest.java +++ /dev/null @@ -1,33 +0,0 @@ -package io.airbyte.config.secrets.persistence; - -import com.azure.identity.IntelliJCredentialBuilder; -import com.azure.security.keyvault.secrets.SecretClientBuilder; -import io.airbyte.config.secrets.SecretCoordinate; -import io.micronaut.context.annotation.Property; -import io.micronaut.context.annotation.Replaces; -import io.micronaut.test.extensions.junit5.annotation.MicronautTest; -import jakarta.inject.Inject; -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - -@MicronautTest -@Property(name = "airbyte.secret.persistence", value = "azure_secret_manager") -@Property(name = "airbyte.secret.store.azure.key-vault-url", value = "https://poc-a7h-kv.vault.azure.net/") -public class AzureSecretManagerPersistenceTest { - - @Inject - SecretPersistence persistence; - - @Test - public void testWrite() { - var coordinate = new SecretCoordinate("plop", 1); - var secret = persistence.read(coordinate); - assertTrue(secret.isEmpty()); - persistence.write(coordinate, "my secret !"); - secret = persistence.read(coordinate); - assertEquals(secret, "my secret !"); - } - -}