Merge branch 'dev' into features/adx

docs/_reporting/hubs/template.md

@@ -101,6 +101,8 @@ Please ensure the following prerequisites are met before deploying this template
 | **dataExplorerFinalRetentionInMonths** | Int    | Optional. Number of months of data to retain in the Data Explorer \*_final_v\* tables.                                                                                                                                                                | 13                 |
 | **remoteHubStorageUri**                | String | Optional. Storage account to push data to for ingestion into a remote hub.                                                                                                                                                                            |                    |
 | **remoteHubStorageKey**                | String | Optional. Storage account key to use when pushing data to a remote hub.                                                                                                                                                                               |                    |
+| **enablePublicAccess**         | string | Optional. Disable public access to the datalake (storage firewall).                                                                                                                                                                                          | False               |
+| **virtualNetworkAddressPrefix**| string | Optional. IP Address range for the private vNet used by the toolkit. /27 recommended to avoid wasting IP's as the deployment will split it into 2 x /28 subnets.                                                                                                                                                                                           | '10.20.30.0/27'     |
 
 <br>
 
@@ -128,6 +130,7 @@ Resources use the following naming convention: `<hubName>-<purpose>-<unique-suff
       - `schemas/reservationrecommendations_2023-05-01_mca.json` – Reservation recommendations MCA schema definition version 2023-05-01 for parquet conversion.
       - `schemas/reservationtransactions_2023-05-01_ea.json` – Reservation transactions EA schema definition version 2023-05-01 for parquet conversion.
       - `schemas/reservationtransactions_2023-05-01_mca.json` – Reservation transactions MCA schema definition version 2023-05-01 for parquet conversion.
+- `<hubName>script<unique-suffix>` storage account (Data Lake Storage Gen2) for deployment scripts.
 - `<hubName>-engine-<unique-suffix>` Data Factory instance
   - Pipelines:
     - `config_InitializeHub` – Initializes (or updates) the FinOps hub instance after deployment.
@@ -147,6 +150,9 @@ Resources use the following naming convention: `<hubName>-<purpose>-<unique-suff
     - `config_MonthlySchedule` – Triggers the `config_RunExportJobs` pipeline monthly for the previous month's cost data.
     - `msexports_ManifestAdded` – Triggers the `msexports_ExecuteETL` pipeline when Cost Management exports complete.
     - `ingestion_DataFileAdded` – Triggers the `ingestion_ExecuteETL` pipeline when files are ingested.
+  - Managed Private Endpoints
+    - `<hubName>store<unique-suffix>` - Managed private endpoint for storage account.
+    - `<hubName>-vault-<unique-suffix>` - Managed private endpoint for Azure Key Vault.
 - `<hubName>-vault-<unique-suffix>` Key Vault instance
   - Secrets:
     - Data Factory system managed identity

docs/_resources/changelog.md

@@ -65,6 +65,12 @@ Legend:
 > 2. Auto-backfill – Backfill historical data from Microsoft Cost Management.
 > 3. Retention – Configure how long you want to keep Cost Management exports and normalized data in storage.
 > 4. ETL pipelile – Add support for parquet files created by Cost Management exports.
+> 5. Private endpoints support.
+>    - Added private endpoints for storage account & Keyvault.
+>    - Added managed virtual network & storage endpoint for Azure Data Factory Runtime.
+>    - All data processing now happens within a vNet.
+>    - Added param to disable external access to data lake
+>    - Added param to specify subnet range of vnet - minumum size = /27
 
 📊 Power BI reports
 {: .fs-5 .fw-500 .mt-4 mb-0 }

src/templates/finops-hub/main.bicep

@@ -129,6 +129,12 @@ param remoteHubStorageUri string = ''
 @secure()
 param remoteHubStorageKey string = ''
 
+@description('Optional. Enable public access to the data lake.  Default: true.')
+param enablePublicAccess bool = true
+
+@description('Optional. Address space for the workload. A /27 is required for the workload. Default: "10.20.30.0/27".')
+param virtualNetworkAddressPrefix string = '10.20.30.0/27'
+
 //==============================================================================
 // Resources
 //==============================================================================
@@ -152,6 +158,8 @@ module hub 'modules/hub.bicep' = {
     dataExplorerFinalRetentionInMonths: dataExplorerFinalRetentionInMonths
     remoteHubStorageUri: remoteHubStorageUri
     remoteHubStorageKey: remoteHubStorageKey
+    enablePublicAccess: enablePublicAccess
+    virtualNetworkAddressPrefix: virtualNetworkAddressPrefix
   }
 }
 

src/templates/finops-hub/modules/dataFactory.bicep

@@ -74,6 +74,8 @@ var datasetPropsDefault = {
 var safeExportContainerName = replace('${exportContainerName}', '-', '_')
 var safeIngestionContainerName = replace('${ingestionContainerName}', '-', '_')
 var safeConfigContainerName = replace('${configContainerName}', '-', '_')
+var managedVnetName = 'default'
+var managedIntegrationRuntimeName = 'AutoResolveIntegrationRuntime'
 
 // Separator used to separate ingestion ID from file name for ingested files
 var ingestionIdFileNameSeparator = '__'
@@ -133,6 +135,106 @@ module azuretimezones 'azuretimezones.bicep' = {
   }
 }
 
+resource managedVirtualNetwork 'Microsoft.DataFactory/factories/managedVirtualNetworks@2018-06-01' = {
+  name: managedVnetName
+  parent: dataFactory
+  properties: {}
+}
+
+resource managedIntegrationRuntime 'Microsoft.DataFactory/factories/integrationRuntimes@2018-06-01' = {
+  name: managedIntegrationRuntimeName
+  parent: dataFactory
+  properties: {
+    type: 'Managed'
+    managedVirtualNetwork: {
+      referenceName: managedVnetName
+      type: 'ManagedVirtualNetworkReference'
+    }
+    typeProperties: {
+      computeProperties: {
+        location: 'AutoResolve'
+      }
+    }
+  }
+  dependsOn: [
+    managedVirtualNetwork
+  ]
+}
+
+resource storageManagedPrivateEndpoint 'Microsoft.DataFactory/factories/managedVirtualNetworks/managedPrivateEndpoints@2018-06-01' = {
+  name: storageAccount.name
+  parent: managedVirtualNetwork
+  dependsOn: [
+    identityRoleAssignments
+  ]
+  properties: {
+    name: storageAccount.name
+    groupId: 'dfs'
+    privateLinkResourceId: storageAccount.id
+    fqdns: [
+      storageAccount.properties.primaryEndpoints.dfs
+    ]
+  }
+}
+
+module getStoragePrivateEndpointConnections 'storageEndpoints.bicep' = {
+  name: 'GetStoragePrivateEndpointConnections'
+  dependsOn: [
+    storageManagedPrivateEndpoint
+  ]
+  params: {
+    storageAccountName: storageAccount.name
+  }
+}
+
+module approveStoragePrivateEndpointConnections 'storageEndpoints.bicep' = {
+  name: 'ApproveStoragePrivateEndpointConnections'
+  dependsOn: [
+    getStoragePrivateEndpointConnections
+  ]
+  params: {
+    storageAccountName: storageAccount.name
+    privateEndpointConnections: getStoragePrivateEndpointConnections.outputs.privateEndpointConnections
+  }
+}
+
+resource keyVaultManagedPrivateEndpoint 'Microsoft.DataFactory/factories/managedVirtualNetworks/managedPrivateEndpoints@2018-06-01' = {
+  name: keyVault.name
+  parent: managedVirtualNetwork
+  dependsOn: [
+    identityRoleAssignments
+  ]
+  properties: {
+    name: keyVault.name
+    groupId: 'vault'
+    privateLinkResourceId: keyVault.id
+    fqdns: [
+      keyVault.properties.vaultUri
+    ]
+  }
+}
+
+module getKeyVaultPrivateEndpointConnections 'keyVaultEndpoints.bicep' = {
+  name: 'GetKeyVaultPrivateEndpointConnections'
+  dependsOn: [
+    keyVaultManagedPrivateEndpoint
+  ]
+  params: {
+    keyVaultName: keyVault.name
+  }
+}
+
+module approveKeyVaultPrivateEndpointConnections 'keyVaultEndpoints.bicep' = {
+  name: 'ApproveKeyVaultPrivateEndpointConnections'
+  dependsOn: [
+    getKeyVaultPrivateEndpointConnections
+  ]
+  params: {
+    keyVaultName: keyVault.name
+    privateEndpointConnections: getKeyVaultPrivateEndpointConnections.outputs.privateEndpointConnections
+  }
+}
+
 //------------------------------------------------------------------------------
 // Identities and RBAC
 //------------------------------------------------------------------------------
@@ -277,6 +379,10 @@ resource linkedService_storageAccount 'Microsoft.DataFactory/factories/linkedser
     typeProperties: {
       url: reference('Microsoft.Storage/storageAccounts/${storageAccount.name}', '2021-08-01').primaryEndpoints.dfs
     }
+    connectVia: {
+      referenceName: managedIntegrationRuntime.name
+      type: 'IntegrationRuntimeReference'
+    }
   }
 }
 

src/templates/finops-hub/modules/hub.bicep

@@ -127,6 +127,12 @@ param remoteHubStorageUri string = ''
 @secure()
 param remoteHubStorageKey string = ''
 
+@description('Optional. Address space for the workload. A /27 is required for the workload. Default: "10.20.30.0/27".')
+param virtualNetworkAddressPrefix string = '10.20.30.0/27'
+
+@description('Optional. Enable public access to the data lake. Default: true.')
+param enablePublicAccess bool = true
+
 @description('Optional. Enable telemetry to track anonymous module usage trends, monitor for bugs, and improve future releases.')
 param enableDefaultTelemetry bool = true
 
@@ -211,6 +217,19 @@ resource defaultTelemetry 'Microsoft.Resources/deployments@2022-09-01' = if (ena
   }
 }
 
+//------------------------------------------------------------------------------
+// Virtual network
+//------------------------------------------------------------------------------
+
+module vnet 'vnet.bicep' = {
+  name: 'vnet'
+  params: {
+    hubName: hubName
+    location: location
+    virtualNetworkAddressPrefix: virtualNetworkAddressPrefix
+  }
+}
+
 //------------------------------------------------------------------------------
 // ADLSv2 storage account for staging and archive
 //------------------------------------------------------------------------------
@@ -229,6 +248,10 @@ module storage 'storage.bicep' = {
     ingestionRetentionInMonths: ingestionRetentionInMonths
     rawRetentionInDays: dataExplorerRawRetentionInDays
     finalRetentionInMonths: dataExplorerFinalRetentionInMonths
+    virtualNetworkId: vnet.outputs.vNetId
+    privateEndpointSubnetId: vnet.outputs.finopsHubSubnetId
+    scriptSubnetId: vnet.outputs.scriptSubnetId
+    enablePublicAccess: enablePublicAccess
   }
 }
 
@@ -306,6 +329,8 @@ module keyVault 'keyVault.bicep' = {
     tags: resourceTags
     tagsByResource: tagsByResource
     storageAccountKey: remoteHubStorageKey
+    virtualNetworkId: vnet.outputs.vNetId
+    privateEndpointSubnetId: vnet.outputs.finopsHubSubnetId
     accessPolicies: [
       {
         objectId: dataFactory.identity.principalId

src/templates/finops-hub/modules/keyVault.bicep

@@ -34,6 +34,12 @@ param tags object = {}
 @description('Optional. Tags to apply to resources based on their resource type. Resource type specific tags will be merged with tags for all resources.')
 param tagsByResource object = {}
 
+@description('Required. Id of the virtual network for private endpoints.')
+param virtualNetworkId string
+
+@description('Required. Id of the subnet for private endpoints.')
+param privateEndpointSubnetId string
+
 //------------------------------------------------------------------------------
 // Variables
 //------------------------------------------------------------------------------
@@ -43,6 +49,7 @@ var keyVaultPrefix = '${replace(hubName, '_', '-')}-vault'
 var keyVaultSuffix = '-${uniqueSuffix}'
 var keyVaultName = replace('${take(keyVaultPrefix, 24 - length(keyVaultSuffix))}${keyVaultSuffix}', '--', '-')
 var keyVaultSecretName = '${toLower(hubName)}-storage-key'
+var keyVaultPrivateDnsZoneName = 'privatelink${replace(environment().suffixes.keyvaultDns, 'vault', 'vaultcore')}'
 
 var formattedAccessPolicies = [for accessPolicy in accessPolicies: {
   applicationId: contains(accessPolicy, 'applicationId') ? accessPolicy.applicationId : ''
@@ -74,6 +81,10 @@ resource keyVault 'Microsoft.KeyVault/vaults@2023-02-01' = {
       name: startsWith(location, 'china') ? 'standard' : sku
       family: 'A'
     }
+    networkAcls: {
+      bypass: 'AzureServices'
+      defaultAction: 'Deny'
+  }
   }
 }
 
@@ -98,6 +109,59 @@ resource keyVault_secret 'Microsoft.KeyVault/vaults/secrets@2023-02-01' = if (!e
   }
 }
 
+resource keyVaultPrivateDnsZone 'Microsoft.Network/privateDnsZones@2024-06-01' = {
+  name: keyVaultPrivateDnsZoneName
+  location: 'global'
+  properties: {}
+}
+
+resource keyVaultPrivateDnsZoneLink 'Microsoft.Network/privateDnsZones/virtualNetworkLinks@2024-06-01' = {
+  name: '${replace(keyVaultPrivateDnsZone.name, '.', '-')}-link'
+  location: 'global'
+  parent: keyVaultPrivateDnsZone
+  properties: {
+    virtualNetwork: {
+      id: virtualNetworkId
+    }
+    registrationEnabled: false
+  }
+}
+
+resource keyVaultEndpoint 'Microsoft.Network/privateEndpoints@2023-11-01' = {
+  name: '${keyVault.name}-ep'
+  location: location
+  properties: {
+    subnet: {
+      id: privateEndpointSubnetId
+    }
+    privateLinkServiceConnections: [
+      {
+        name: 'keyVaultLink'
+        properties: {
+          privateLinkServiceId: keyVault.id
+          groupIds: ['vault']
+        }
+      }
+    ]
+  }
+}
+
+resource keyVaultPrivateDnsZoneGroup 'Microsoft.Network/privateEndpoints/privateDnsZoneGroups@2023-11-01' = {
+  name: 'keyvault-endpoint-zone'
+  parent: keyVaultEndpoint
+  properties: {
+    privateDnsZoneConfigs: [
+      {
+        name: keyVaultPrivateDnsZone.name
+        properties: {
+          privateDnsZoneId: keyVaultPrivateDnsZone.id
+        }
+      }
+    ]
+  }
+}
+
+
 //==============================================================================
 // Outputs
 //==============================================================================

src/templates/finops-hub/modules/keyVaultEndpoints.bicep

@@ -0,0 +1,37 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+//==============================================================================
+// Parameters
+//==============================================================================
+
+@description('Optional. Array of private endpoint connections. Pending ones will be approved.')
+param privateEndpointConnections array = []
+
+@description('Required. Name of the KeyVault.')
+param keyVaultName string
+
+//==============================================================================
+// Resources
+//==============================================================================
+
+resource keyVault 'Microsoft.KeyVault/vaults@2023-07-01' existing = {
+  name: keyVaultName
+}
+
+resource privateEndpointConnection 'Microsoft.KeyVault/vaults/privateEndpointConnections@2023-07-01' =  [ for privateEndpointConnection in privateEndpointConnections : if (privateEndpointConnection.properties.privateLinkServiceConnectionState.status == 'Pending') {
+  name: last(array(split(privateEndpointConnection.id, '/')))
+  parent: keyVault
+  properties: {
+    privateLinkServiceConnectionState: {
+      status: 'Approved'
+      description: 'Approved-by-pipeline'
+    }
+  }
+}]
+
+//==============================================================================
+// Outputs
+//==============================================================================
+
+output privateEndpointConnections array = keyVault.properties.privateEndpointConnections