diff --git a/src/templates/finops-hub/createUiDefinition.json b/src/templates/finops-hub/createUiDefinition.json index 43bbd6938..e72ac4aa7 100644 --- a/src/templates/finops-hub/createUiDefinition.json +++ b/src/templates/finops-hub/createUiDefinition.json @@ -13,6 +13,9 @@ "Microsoft.KeyVault/vaults", "Microsoft.Kusto/clusters", "Microsoft.ManagedIdentity/userAssignedIdentities", + "Microsoft.Network/privateDnsZones", + "Microsoft.Network/privateDnsZones/virtualNetworkLinks", + "Microsoft.Network/privateEndpoints", "Microsoft.Resources/deploymentScripts", "Microsoft.Storage/storageAccounts" ] @@ -24,6 +27,9 @@ "Microsoft.KeyVault/vaults", "Microsoft.Kusto/clusters", "Microsoft.ManagedIdentity/userAssignedIdentities", + "Microsoft.Network/privateDnsZones", + "Microsoft.Network/privateDnsZones/virtualNetworkLinks", + "Microsoft.Network/privateEndpoints", "Microsoft.Resources/deploymentScripts", "Microsoft.Storage/storageAccounts" ], @@ -572,6 +578,9 @@ "Microsoft.KeyVault/vaults", "Microsoft.Kusto/clusters", "Microsoft.ManagedIdentity/userAssignedIdentities", + "Microsoft.Network/privateDnsZones", + "Microsoft.Network/privateDnsZones/virtualNetworkLinks", + "Microsoft.Network/privateEndpoints", "Microsoft.Resources/deploymentScripts", "Microsoft.Storage/storageAccounts" ] diff --git a/src/templates/finops-hub/modules/dataExplorer.bicep b/src/templates/finops-hub/modules/dataExplorer.bicep index f696408af..d6c0c82d9 100644 --- a/src/templates/finops-hub/modules/dataExplorer.bicep +++ b/src/templates/finops-hub/modules/dataExplorer.bicep @@ -108,11 +108,8 @@ param dataFactoryName string @description('Optional. Number of days of data to retain in the Data Explorer *_raw tables. Default: 0.') param rawRetentionInDays int = 0 -// @description('Required. Name of the storage account to use for data ingestion.') -// param storageAccountName string - -// @description('Required. Name of storage container to monitor for data ingestion.') -// param storageContainerName string +@description('Required. Name of the storage account to use for data ingestion.') +param storageAccountName string @description('Required. Resource ID of the virtual network for private endpoints.') param virtualNetworkId string @@ -132,6 +129,74 @@ var ftkVersion = contains(ftkver, '-') ? split(ftkver, '-')[0] : ftkver var ftkBranch = contains(ftkver, '-') ? split(ftkver, '-')[1] : '' var dataExplorerPrivateDnsZoneName = replace('privatelink.${location}.${replace(environment().suffixes.storage, 'core', 'kusto')}', '..', '.') +// Actual = Minimum(ClusterMaximumConcurrentOperations, Number of nodes in cluster * Maximum(1, Core count per node * CoreUtilizationCoefficient)) +var ingestionCapacity = { + 'Dev(No SLA)_Standard_E2a_v4': 1 + 'Dev(No SLA)_Standard_D11_v2': 1 + Standard_D11_v2: 2 + Standard_D12_v2: 4 + Standard_D13_v2: 8 + Standard_D14_v2: 16 + Standard_D16d_v5: 16 + Standard_D32d_v4: 32 + Standard_D32d_v5: 32 + 'Standard_DS13_v2+1TB_PS': 8 + 'Standard_DS13_v2+2TB_PS': 8 + 'Standard_DS14_v2+3TB_PS': 16 + 'Standard_DS14_v2+4TB_PS': 16 + Standard_E2a_v4: 2 + Standard_E2ads_v5: 2 + Standard_E2d_v4: 2 + Standard_E2d_v5: 2 + Standard_E4a_v4: 4 + Standard_E4ads_v5: 4 + Standard_E4d_v4: 4 + Standard_E4d_v5: 4 + Standard_E8a_v4: 8 + Standard_E8ads_v5: 8 + 'Standard_E8as_v4+1TB_PS': 8 + 'Standard_E8as_v4+2TB_PS': 8 + 'Standard_E8as_v5+1TB_PS': 8 + 'Standard_E8as_v5+2TB_PS': 8 + Standard_E8d_v4: 8 + Standard_E8d_v5: 8 + 'Standard_E8s_v4+1TB_PS': 8 + 'Standard_E8s_v4+2TB_PS': 8 + 'Standard_E8s_v5+1TB_PS': 8 + 'Standard_E8s_v5+2TB_PS': 8 + Standard_E16a_v4: 16 + Standard_E16ads_v5: 16 + 'Standard_E16as_v4+3TB_PS': 16 + 'Standard_E16as_v4+4TB_PS': 16 + 'Standard_E16as_v5+3TB_PS': 16 + 'Standard_E16as_v5+4TB_PS': 16 + Standard_E16d_v4: 16 + Standard_E16d_v5: 16 + 'Standard_E16s_v4+3TB_PS': 16 + 'Standard_E16s_v4+4TB_PS': 16 + 'Standard_E16s_v5+3TB_PS': 16 + 'Standard_E16s_v5+4TB_PS': 16 + Standard_E64i_v3: 64 + Standard_E80ids_v4: 80 + Standard_EC8ads_v5: 8 + 'Standard_EC8as_v5+1TB_PS': 8 + 'Standard_EC8as_v5+2TB_PS': 8 + Standard_EC16ads_v5: 16 + 'Standard_EC16as_v5+3TB_PS': 16 + 'Standard_EC16as_v5+4TB_PS': 16 + Standard_L4s: 4 + Standard_L8as_v3: 8 + Standard_L8s: 8 + Standard_L8s_v2: 8 + Standard_L8s_v3: 8 + Standard_L16as_v3: 16 + Standard_L16s: 16 + Standard_L16s_v2: 16 + Standard_L16s_v3: 16 + Standard_L32as_v3: 32 + Standard_L32s_v3: 32 +} + //============================================================================== // Resources //============================================================================== @@ -157,17 +222,9 @@ resource tablePrivateDnsZone 'Microsoft.Network/privateDnsZones@2024-06-01' exis name: 'privatelink.table.${environment().suffixes.storage}' } -// resource storage 'Microsoft.Storage/storageAccounts@2022-09-01' existing = { -// name: storageAccountName - -// resource blobServices 'blobServices' = { -// name: 'default' - -// resource landingContainer 'containers' = { -// name: storageContainerName -// } -// } -// } +resource storage 'Microsoft.Storage/storageAccounts@2022-09-01' existing = { + name: storageAccountName +} //------------------------------------------------------------------------------ // Cluster + databases @@ -238,7 +295,7 @@ resource cluster 'Microsoft.Kusto/clusters@2023-08-15' = { location: location kind: 'ReadWrite' dependsOn: [ - ingestionDb + ingestionDb::setupScript ] resource commonScript 'scripts' = { @@ -253,7 +310,6 @@ resource cluster 'Microsoft.Kusto/clusters@2023-08-15' = { resource setupScript 'scripts' = { name: 'SetupScript' dependsOn: [ - ingestionDb::setupScript hubDb::commonScript ] properties: { @@ -267,26 +323,27 @@ resource cluster 'Microsoft.Kusto/clusters@2023-08-15' = { } } -// // Authorize Kusto Cluster to read storage -// resource clusterStorageAccess 'Microsoft.Authorization/roleAssignments@2022-04-01' = { -// name: guid(cluster.name, storageContainerName, 'Storage Blob Data Contributor') -// scope: storage::blobServices -// properties: { -// description: 'Give "Storage Blob Data Contributor" to the cluster' -// principalId: cluster.identity.principalId -// // Required in case principal not ready when deploying the assignment -// principalType: 'ServicePrincipal' -// roleDefinitionId: subscriptionResourceId( -// 'Microsoft.Authorization/roleDefinitions', -// 'ba92f5b4-2d11-453d-a403-e96b0029c9fe' // Storage Blob Data Contributor -- https://learn.microsoft.com/azure/role-based-access-control/built-in-roles#storage -// ) -// } -// } +// Authorize Kusto Cluster to read storage +resource clusterStorageAccess 'Microsoft.Authorization/roleAssignments@2022-04-01' = { + name: guid(cluster.name, subscription().id, 'Storage Blob Data Contributor') + scope: storage + properties: { + description: 'Give "Storage Blob Data Contributor" to the cluster' + principalId: cluster.identity.principalId + // Required in case principal not ready when deploying the assignment + principalType: 'ServicePrincipal' + roleDefinitionId: subscriptionResourceId( + 'Microsoft.Authorization/roleDefinitions', + 'ba92f5b4-2d11-453d-a403-e96b0029c9fe' // Storage Blob Data Contributor -- https://learn.microsoft.com/azure/role-based-access-control/built-in-roles#storage + ) + } +} // DNS zone resource dataExplorerPrivateDnsZone 'Microsoft.Network/privateDnsZones@2024-06-01' = { name: dataExplorerPrivateDnsZoneName location: 'global' + tags: union(tags, contains(tagsByResource, 'Microsoft.Network/privateDnsZones') ? tagsByResource['Microsoft.Network/privateDnsZones'] : {}) properties: {} } @@ -295,6 +352,7 @@ resource dataExplorerPrivateDnsZoneLink 'Microsoft.Network/privateDnsZones/virtu name: '${replace(dataExplorerPrivateDnsZone.name, '.', '-')}-link' location: 'global' parent: dataExplorerPrivateDnsZone + tags: union(tags, contains(tagsByResource, 'Microsoft.Network/privateDnsZones/virtualNetworkLinks') ? tagsByResource['Microsoft.Network/privateDnsZones/virtualNetworkLinks'] : {}) properties: { virtualNetwork: { id: virtualNetworkId @@ -307,6 +365,7 @@ resource dataExplorerPrivateDnsZoneLink 'Microsoft.Network/privateDnsZones/virtu resource dataExplorerEndpoint 'Microsoft.Network/privateEndpoints@2023-11-01' = { name: '${cluster.name}-ep' location: location + tags: union(tags, contains(tagsByResource, 'Microsoft.Network/privateEndpoints') ? tagsByResource['Microsoft.Network/privateEndpoints'] : {}) properties: { subnet: { id: privateEndpointSubnetId @@ -364,6 +423,9 @@ resource dataExplorerPrivateDnsZoneGroup 'Microsoft.Network/privateEndpoints/pri @description('The resource ID of the cluster.') output clusterId string = cluster.id +@description('The ID of the cluster system assigned managed identity.') +output principalId string = cluster.identity.principalId + @description('The name of the cluster.') output clusterName string = cluster.name @@ -375,3 +437,6 @@ output ingestionDbName string = cluster::ingestionDb.name @description('The name of the database for queries.') output hubDbName string = cluster::hubDb.name + +@description('Max ingestion capacity of the cluster.') +output clusterIngestionCapacity int = ingestionCapacity[?clusterSku] ?? 1 diff --git a/src/templates/finops-hub/modules/dataFactory.bicep b/src/templates/finops-hub/modules/dataFactory.bicep index cd462882f..6c9dcecae 100644 --- a/src/templates/finops-hub/modules/dataFactory.bicep +++ b/src/templates/finops-hub/modules/dataFactory.bicep @@ -32,12 +32,18 @@ param dataExplorerName string = '' @description('Optional. Resource ID of the Azure Data Explorer cluster to use for advanced analytics, if applicable.') param dataExplorerId string = '' +@description('Optional. ID of the Azure Data Explorer cluster system assigned managed identity, if applicable.') +param dataExplorerPrincipalId string = '' + @description('Optional. URI of the Azure Data Explorer cluster to use for advanced analytics, if applicable.') param dataExplorerUri string = '' @description('Optional. Name of the Azure Data Explorer ingestion database. Default: "ingestion".') param dataExplorerIngestionDatabase string = 'Ingestion' +@description('Optional. Azure Data Explorer ingestion capacity. Increase for non-dev SKUs. Default: 1') +param dataExplorerIngestionCapacity int = 1 + @description('Optional. The location to use for the managed identity and deployment script to auto-start triggers. Default = (resource group location).') param location string = resourceGroup().location @@ -170,13 +176,13 @@ resource managedIntegrationRuntime 'Microsoft.DataFactory/factories/integrationR customProperties: [] } copyComputeScaleProperties: { - dataIntegrationUnit: 256 + dataIntegrationUnit: 16 timeToLive: 30 } pipelineExternalComputeScaleProperties: { timeToLive: 30 - numberOfPipelineNodes: 10 - numberOfExternalNodes: 10 + numberOfPipelineNodes: 1 + numberOfExternalNodes: 1 } } } @@ -853,7 +859,7 @@ resource trigger_IngestionManifestAdded 'Microsoft.DataFactory/factories/trigger typeProperties: { blobPathBeginsWith: '/${ingestionContainerName}/blobs/' blobPathEndsWith: 'manifest.json' - ignoreEmptyBlobs: false + ignoreEmptyBlobs: true scope: storageAccount.id events: [ 'Microsoft.Storage.BlobCreated' @@ -1164,10 +1170,44 @@ resource pipeline_InitializeHub 'Microsoft.DataFactory/factories/pipelines@2018- } ] ifTrueActivities: [ + { // Save ingestion policy in ADX + name: 'Set ingestion policy in ADX' + type: 'AzureDataExplorerCommand' + dependsOn: [] + policy: { + timeout: '0.12:00:00' + retry: 0 + retryIntervalInSeconds: 30 + secureOutput: false + secureInput: false + } + userProperties: [] + typeProperties: { + command: { + value: '.alter-merge database ${dataExplorerIngestionDatabase} policy managed_identity "[ { \'ObjectId\' : \'${dataExplorerPrincipalId}\', \'AllowedUsages\' : \'NativeIngestion\' }]"' + type: 'Expression' + } + commandTimeout: '00:20:00' + } + linkedServiceName: { + referenceName: linkedService_dataExplorer.name + type: 'LinkedServiceReference' + parameters: { + database: dataExplorerIngestionDatabase + } + } + } { // Save Hub Settings in ADX name: 'Save Hub Settings in ADX' type: 'AzureDataExplorerCommand' - dependsOn: [] + dependsOn: [ + { + activity: 'Set ingestion policy in ADX' + dependencyConditions: [ + 'Succeeded' + ] + } + ] policy: { timeout: '0.12:00:00' retry: 0 @@ -2638,7 +2678,7 @@ resource pipeline_ExecuteExportsETL 'Microsoft.DataFactory/factories/pipelines@2 typeProperties: { variableName: 'hasNoRows' value: { - value: '@or(equals(activity(\'Read Manifest\').output.firstRow.dataRowCount, null), equals(activity(\'Read Manifest\').output.firstRow.dataRowCount, 0))' + value: '@or(equals(activity(\'Read Manifest\').output.firstRow.blobCount, null), equals(activity(\'Read Manifest\').output.firstRow.blobCount, 0))' type: 'Expression' } } @@ -3202,6 +3242,7 @@ resource pipeline_ExecuteExportsETL 'Microsoft.DataFactory/factories/pipelines@2 value: '@if(variables(\'hasNoRows\'), json(\'[]\'), activity(\'Read Manifest\').output.firstRow.blobs)' type: 'Expression' } + batchCount: enablePublicAccess ? 30 : 4 // so we don't overload the managed runtime isSequential: false activities: [ { // Execute @@ -4007,7 +4048,6 @@ resource pipeline_ToDataExplorer 'Microsoft.DataFactory/factories/pipelines@2018 name: '${safeIngestionContainerName}_ETL_dataExplorer' parent: dataFactory properties: { - // concurrency: 8 // sanity check activities: [ { // Read Hub Config name: 'Read Hub Config' @@ -4199,15 +4239,15 @@ resource pipeline_ToDataExplorer 'Microsoft.DataFactory/factories/pipelines@2018 ] policy: { timeout: '0.12:00:00' - retry: 0 - retryIntervalInSeconds: 30 + retry: 3 + retryIntervalInSeconds: 120 secureOutput: false secureInput: false } userProperties: [] typeProperties: { command: { - value: '@concat(\'.ingest into table \', pipeline().parameters.table, \' ("${storageAccount.properties.primaryEndpoints.dfs}/${ingestionContainerName}/\', pipeline().parameters.folderPath, \'/\', pipeline().parameters.fileName, \'") with (format="parquet", ingestionMappingReference="\', pipeline().parameters.table, \'_mapping", tags="[\\"drop-by:\', pipeline().parameters.ingestionId, \'\\", \\"drop-by:\', pipeline().parameters.folderPath, \'/\', pipeline().parameters.originalFileName, \'\\", \\"drop-by:ftk-version-${ftkVersion}\\"]")\')' + value: '@concat(\'.ingest into table \', pipeline().parameters.table, \' ("abfss://${ingestionContainerName}@${storageAccount.name}.dfs.${environment().suffixes.storage}/\', pipeline().parameters.folderPath, \'/\', pipeline().parameters.fileName, \';managed_identity=system") with (format="parquet", ingestionMappingReference="\', pipeline().parameters.table, \'_mapping", tags="[\\"drop-by:\', pipeline().parameters.ingestionId, \'\\", \\"drop-by:\', pipeline().parameters.folderPath, \'/\', pipeline().parameters.originalFileName, \'\\", \\"drop-by:ftk-version-${ftkVersion}\\"]"); print Success = assert(iff(toscalar($command_results | project-keep HasErrors) == false, true, false), "Ingestion Failed")\')' type: 'Expression' } commandTimeout: '01:00:00' @@ -4450,6 +4490,7 @@ resource pipeline_ExecuteIngestionETL 'Microsoft.DataFactory/factories/pipelines name: '${safeIngestionContainerName}_ExecuteETL' parent: dataFactory properties: { + concurrency: 1 activities: [ { // Wait name: 'Wait' @@ -4593,6 +4634,7 @@ resource pipeline_ExecuteIngestionETL 'Microsoft.DataFactory/factories/pipelines ] userProperties: [] typeProperties: { + batchCount: dataExplorerIngestionCapacity // Concurrency limit items: { value: '@activity(\'Filter Out Folders\').output.Value' type: 'Expression' @@ -4612,7 +4654,7 @@ resource pipeline_ExecuteIngestionETL 'Microsoft.DataFactory/factories/pipelines referenceName: pipeline_ToDataExplorer.name type: 'PipelineReference' } - waitOnCompletion: false + waitOnCompletion: true parameters: { folderPath: { value: '@variables(\'containerFolderPath\')' diff --git a/src/templates/finops-hub/modules/hub.bicep b/src/templates/finops-hub/modules/hub.bicep index 49080cc9a..0e058f59a 100644 --- a/src/templates/finops-hub/modules/hub.bicep +++ b/src/templates/finops-hub/modules/hub.bicep @@ -167,6 +167,8 @@ var safeDataExplorerName = !deployDataExplorer ? '' : dataExplorer.outputs.clust var safeDataExplorerUri = !deployDataExplorer ? '' : dataExplorer.outputs.clusterUri var safeDataExplorerId = !deployDataExplorer ? '' : dataExplorer.outputs.clusterId var safeDataExplorerIngestionDb = !deployDataExplorer ? '' : dataExplorer.outputs.ingestionDbName +var safeDataExplorerIngestionCapacity = !deployDataExplorer ? 1 : dataExplorer.outputs.clusterIngestionCapacity +var safeDataExplorerPrincipalId = !deployDataExplorer ? '' : dataExplorer.outputs.principalId // var eventGridPrefix = '${replace(hubName, '_', '-')}-ns' // var eventGridSuffix = '-${uniqueSuffix}' @@ -232,6 +234,8 @@ module vnet 'vnet.bicep' = { hubName: hubName location: location virtualNetworkAddressPrefix: virtualNetworkAddressPrefix + tags: resourceTags + tagsByResource: tagsByResource } } @@ -279,6 +283,7 @@ module dataExplorer 'dataExplorer.bicep' = if (deployDataExplorer) { virtualNetworkId: vnet.outputs.vNetId privateEndpointSubnetId: vnet.outputs.dataExplorerSubnetId enablePublicAccess: enablePublicAccess + storageAccountName: storage.outputs.name } } @@ -314,7 +319,9 @@ module dataFactoryResources 'dataFactory.bicep' = { configContainerName: storage.outputs.configContainer ingestionContainerName: storage.outputs.ingestionContainer dataExplorerName: safeDataExplorerName + dataExplorerPrincipalId: safeDataExplorerPrincipalId dataExplorerIngestionDatabase: safeDataExplorerIngestionDb + dataExplorerIngestionCapacity: safeDataExplorerIngestionCapacity dataExplorerUri: safeDataExplorerUri dataExplorerId: safeDataExplorerId keyVaultName: keyVault.outputs.name diff --git a/src/templates/finops-hub/modules/keyVault.bicep b/src/templates/finops-hub/modules/keyVault.bicep index 37687505d..bf2d80db5 100644 --- a/src/templates/finops-hub/modules/keyVault.bicep +++ b/src/templates/finops-hub/modules/keyVault.bicep @@ -112,6 +112,7 @@ resource keyVault_secret 'Microsoft.KeyVault/vaults/secrets@2023-02-01' = if (!e resource keyVaultPrivateDnsZone 'Microsoft.Network/privateDnsZones@2024-06-01' = { name: keyVaultPrivateDnsZoneName location: 'global' + tags: union(tags, contains(tagsByResource, 'Microsoft.KeyVault/privateDnsZones') ? tagsByResource['Microsoft.KeyVault/privateDnsZones'] : {}) properties: {} } @@ -119,6 +120,7 @@ resource keyVaultPrivateDnsZoneLink 'Microsoft.Network/privateDnsZones/virtualNe name: '${replace(keyVaultPrivateDnsZone.name, '.', '-')}-link' location: 'global' parent: keyVaultPrivateDnsZone + tags: union(tags, contains(tagsByResource, 'Microsoft.Network/privateDnsZones/virtualNetworkLinks') ? tagsByResource['Microsoft.Network/privateDnsZones/virtualNetworkLinks'] : {}) properties: { virtualNetwork: { id: virtualNetworkId @@ -130,6 +132,7 @@ resource keyVaultPrivateDnsZoneLink 'Microsoft.Network/privateDnsZones/virtualNe resource keyVaultEndpoint 'Microsoft.Network/privateEndpoints@2023-11-01' = { name: '${keyVault.name}-ep' location: location + tags: union(tags, contains(tagsByResource, 'Microsoft.Network/privateEndpoints') ? tagsByResource['Microsoft.Network/privateEndpoints'] : {}) properties: { subnet: { id: privateEndpointSubnetId diff --git a/src/templates/finops-hub/modules/scripts/Common.kql b/src/templates/finops-hub/modules/scripts/Common.kql index 2770a2906..89c3de4d0 100644 --- a/src/templates/finops-hub/modules/scripts/Common.kql +++ b/src/templates/finops-hub/modules/scripts/Common.kql @@ -158,27 +158,3 @@ ifempty(val: dynamic, defaultVal: dynamic) { iff(isempty(val), defaultVal, val) } - -// parse_resourceid -.create-or-alter function -with (docstring = 'Parses an Azure resource ID to extract resource attributes like the name, type, resource group, and subaccount ID.', folder = 'Common') -parse_resourceid(resourceId: string) { - let ResourceId = tolower(resourceId); - // let ResourceId = tolower('/providers/Microsoft.BillingBenefits/savingsPlanOrders/2d2e284b-0638-427e-b8c6-1b874d4f17c8/sp/xxx'); - let SubAccountId = tostring(extract('/subscriptions/[^/]+', 1, ResourceId)); - let x_ResourceGroupName = tostring(extract('/resourcegroups/[^/]+', 1, ResourceId)); - let providerPath = iff(ResourceId !contains '/providers/', '', split(iff(ResourceId startswith '/subscriptions/', strcat('/providers/microsoft.resources/', ResourceId), ResourceId), '/providers/')[-1]); - let x_ResourceProvider = iff(isempty(providerPath), '', split(providerPath, '/')[0]); - let tmp_ResourceProviderPath = iff(isempty(providerPath), '', substring(providerPath, strlen(x_ResourceProvider) + 1)); - let segments = split(tmp_ResourceProviderPath, '/'); - let ResourceName = trim(@'/+', replace_string(strcat_array(array_iff( - dynamic([false, true, false, true, false, true, false, true, false, true, false, true, false, true, false, true, false, true]), - segments, dynamic([])), '/'), '//', '/')); - let x_ResourceTypePath = trim(@'/+', replace_string(strcat_array(array_iff( - dynamic([true, false, true, false, true, false, true, false, true, false, true, false, true, false, true, false, true, false]), - segments, dynamic([])), '/'), '//', '/')); - let xRT = iff(isempty(x_ResourceProvider) or isempty(x_ResourceTypePath), '', strcat(x_ResourceProvider, '/', x_ResourceTypePath)); - let types = split(toscalar(database('Ingestion').ResourceTypes | summarize types = make_set(strcat(x_ResourceType, '=', SingularDisplayName)) | extend types = strcat('||', strcat_array(types, '||')), '||'), strcat('||', xRT, '=')); - let ResourceType = iff(array_length(types) < 2, '', split(types[1], '||')[0]); - bag_pack('ResourceId', ResourceId, 'ResourceName', ResourceName, 'ResourceType', coalesce(ResourceType, xRT), 'SubAccountId', SubAccountId, 'x_ResourceGroupName', x_ResourceGroupName, 'x_ResourceProvider', x_ResourceProvider, 'x_ResourceType', xRT) -} diff --git a/src/templates/finops-hub/modules/scripts/IngestionSetup.kql b/src/templates/finops-hub/modules/scripts/IngestionSetup.kql index 1ea8311fb..8214d8969 100644 --- a/src/templates/finops-hub/modules/scripts/IngestionSetup.kql +++ b/src/templates/finops-hub/modules/scripts/IngestionSetup.kql @@ -18,7 +18,6 @@ // For allowed commands, see https://learn.microsoft.com/azure/data-explorer/database-script - //===| Settings |======================================================================================================= .create-merge table HubSettingsLog ( @@ -93,6 +92,32 @@ HubScopes() x_ServiceModel: string ) +//---------------------------------------------------------------------------------------------------------------------- + +// parse_resourceid +.create-or-alter function +with (docstring = 'Parses an Azure resource ID to extract resource attributes like the name, type, resource group, and subaccount ID.', folder = 'Common') +parse_resourceid(resourceId: string) { + let ResourceId = tolower(resourceId); + // let ResourceId = tolower('/providers/Microsoft.BillingBenefits/savingsPlanOrders/2d2e284b-0638-427e-b8c6-1b874d4f17c8/sp/xxx'); + let SubAccountId = tostring(extract('/subscriptions/[^/]+', 1, ResourceId)); + let x_ResourceGroupName = tostring(extract('/resourcegroups/[^/]+', 1, ResourceId)); + let providerPath = iff(ResourceId !contains '/providers/', '', split(iff(ResourceId startswith '/subscriptions/', strcat('/providers/microsoft.resources/', ResourceId), ResourceId), '/providers/')[-1]); + let x_ResourceProvider = iff(isempty(providerPath), '', split(providerPath, '/')[0]); + let tmp_ResourceProviderPath = iff(isempty(providerPath), '', substring(providerPath, strlen(x_ResourceProvider) + 1)); + let segments = split(tmp_ResourceProviderPath, '/'); + let ResourceName = trim(@'/+', replace_string(strcat_array(array_iff( + dynamic([false, true, false, true, false, true, false, true, false, true, false, true, false, true, false, true, false, true]), + segments, dynamic([])), '/'), '//', '/')); + let x_ResourceTypePath = trim(@'/+', replace_string(strcat_array(array_iff( + dynamic([true, false, true, false, true, false, true, false, true, false, true, false, true, false, true, false, true, false]), + segments, dynamic([])), '/'), '//', '/')); + let xRT = iff(isempty(x_ResourceProvider) or isempty(x_ResourceTypePath), '', strcat(x_ResourceProvider, '/', x_ResourceTypePath)); + let types = split(toscalar(database('Ingestion').ResourceTypes | summarize types = make_set(strcat(x_ResourceType, '=', SingularDisplayName)) | extend types = strcat('||', strcat_array(types, '||')), '||'), strcat('||', xRT, '=')); + let ResourceType = iff(array_length(types) < 2, '', split(types[1], '||')[0]); + bag_pack('ResourceId', ResourceId, 'ResourceName', ResourceName, 'ResourceType', coalesce(ResourceType, xRT), 'SubAccountId', SubAccountId, 'x_ResourceGroupName', x_ResourceGroupName, 'x_ResourceProvider', x_ResourceProvider, 'x_ResourceType', xRT) +} + //===| Prices |========================================================================================================= // NOTE: Must be before cost details. diff --git a/src/templates/finops-hub/modules/storage.bicep b/src/templates/finops-hub/modules/storage.bicep index 30d10f0ba..3d901542f 100644 --- a/src/templates/finops-hub/modules/storage.bicep +++ b/src/templates/finops-hub/modules/storage.bicep @@ -151,24 +151,28 @@ resource scriptStorageAccount 'Microsoft.Storage/storageAccounts@2022-09-01' = resource blobPrivateDnsZone 'Microsoft.Network/privateDnsZones@2024-06-01' = { name: 'privatelink.blob.${environment().suffixes.storage}' location: 'global' + tags: union(tags, contains(tagsByResource, 'Microsoft.Storage/privateDnsZones') ? tagsByResource['Microsoft.Storage/privateDnsZones'] : {}) properties: {} } resource dfsPrivateDnsZone 'Microsoft.Network/privateDnsZones@2024-06-01' = { name: 'privatelink.dfs.${environment().suffixes.storage}' location: 'global' + tags: union(tags, contains(tagsByResource, 'Microsoft.Storage/privateDnsZones') ? tagsByResource['Microsoft.Storage/privateDnsZones'] : {}) properties: {} } resource queuePrivateDnsZone 'Microsoft.Network/privateDnsZones@2024-06-01' = { name: 'privatelink.queue.${environment().suffixes.storage}' location: 'global' + tags: union(tags, contains(tagsByResource, 'Microsoft.Storage/privateDnsZones') ? tagsByResource['Microsoft.Storage/privateDnsZones'] : {}) properties: {} } resource tablePrivateDnsZone 'Microsoft.Network/privateDnsZones@2024-06-01' = { name: 'privatelink.table.${environment().suffixes.storage}' location: 'global' + tags: union(tags, contains(tagsByResource, 'Microsoft.Storage/privateDnsZones') ? tagsByResource['Microsoft.Storage/privateDnsZones'] : {}) properties: {} } @@ -176,6 +180,7 @@ resource blobPrivateDnsZoneLink 'Microsoft.Network/privateDnsZones/virtualNetwor parent: blobPrivateDnsZone name: '${replace(blobPrivateDnsZone.name, '.', '-')}-link' location: 'global' + tags: union(tags, contains(tagsByResource, 'Microsoft.Network/privateDnsZones/virtualNetworkLinks') ? tagsByResource['Microsoft.Network/privateDnsZones/virtualNetworkLinks'] : {}) properties: { registrationEnabled: false virtualNetwork: { @@ -188,6 +193,7 @@ resource dfsPrivateDnsZoneLink 'Microsoft.Network/privateDnsZones/virtualNetwork parent: dfsPrivateDnsZone name: '${replace(dfsPrivateDnsZone.name, '.', '-')}-link' location: 'global' + tags: union(tags, contains(tagsByResource, 'Microsoft.Network/privateDnsZones/virtualNetworkLinks') ? tagsByResource['Microsoft.Network/privateDnsZones/virtualNetworkLinks'] : {}) properties: { registrationEnabled: false virtualNetwork: { @@ -200,6 +206,7 @@ resource queuePrivateDnsZoneLink 'Microsoft.Network/privateDnsZones/virtualNetwo parent: queuePrivateDnsZone name: '${replace(queuePrivateDnsZone.name, '.', '-')}-link' location: 'global' + tags: union(tags, contains(tagsByResource, 'Microsoft.Network/privateDnsZones/virtualNetworkLinks') ? tagsByResource['Microsoft.Network/privateDnsZones/virtualNetworkLinks'] : {}) properties: { registrationEnabled: false virtualNetwork: { @@ -212,6 +219,7 @@ resource tablePrivateDnsZoneLink 'Microsoft.Network/privateDnsZones/virtualNetwo parent: tablePrivateDnsZone name: '${replace(tablePrivateDnsZone.name, '.', '-')}-link' location: 'global' + tags: union(tags, contains(tagsByResource, 'Microsoft.Network/privateDnsZones/virtualNetworkLinks') ? tagsByResource['Microsoft.Network/privateDnsZones/virtualNetworkLinks'] : {}) properties: { registrationEnabled: false virtualNetwork: { @@ -223,6 +231,7 @@ resource tablePrivateDnsZoneLink 'Microsoft.Network/privateDnsZones/virtualNetwo resource blobEndpoint 'Microsoft.Network/privateEndpoints@2023-11-01' = { name: '${storageAccount.name}-blob-ep' location: location + tags: union(tags, contains(tagsByResource, 'Microsoft.Network/privateEndpoints') ? tagsByResource['Microsoft.Network/privateEndpoints'] : {}) properties: { subnet: { id: privateEndpointSubnetId @@ -242,6 +251,7 @@ resource blobEndpoint 'Microsoft.Network/privateEndpoints@2023-11-01' = { resource scriptEndpoint 'Microsoft.Network/privateEndpoints@2023-11-01' = { name: '${scriptStorageAccount.name}-blob-ep' location: location + tags: union(tags, contains(tagsByResource, 'Microsoft.Network/privateEndpoints') ? tagsByResource['Microsoft.Network/privateEndpoints'] : {}) properties: { subnet: { id: privateEndpointSubnetId @@ -261,6 +271,7 @@ resource scriptEndpoint 'Microsoft.Network/privateEndpoints@2023-11-01' = { resource dfsEndpoint 'Microsoft.Network/privateEndpoints@2023-11-01' = { name: '${storageAccount.name}-dfs-ep' location: location + tags: union(tags, contains(tagsByResource, 'Microsoft.Network/privateEndpoints') ? tagsByResource['Microsoft.Network/privateEndpoints'] : {}) properties: { subnet: { id: privateEndpointSubnetId diff --git a/src/templates/finops-hub/modules/vnet.bicep b/src/templates/finops-hub/modules/vnet.bicep index f46345182..da70169ec 100644 --- a/src/templates/finops-hub/modules/vnet.bicep +++ b/src/templates/finops-hub/modules/vnet.bicep @@ -15,6 +15,12 @@ param virtualNetworkAddressPrefix string = '10.20.30.0/26' @description('Optional. Azure location where all resources should be created. See https://aka.ms/azureregions. Default: (resource group location).') param location string = resourceGroup().location +@description('Optional. Tags to apply to all resources.') +param tags object = {} + +@description('Optional. Tags to apply to resources based on their resource type. Resource type specific tags will be merged with tags for all resources.') +param tagsByResource object = {} + //------------------------------------------------------------------------------ // Variables //------------------------------------------------------------------------------ @@ -77,6 +83,7 @@ var subnets = [ resource nsg 'Microsoft.Network/networkSecurityGroups@2023-11-01' = { name: nsgName location: location + tags: union(tags, contains(tagsByResource, 'Microsoft.Storage/networkSecurityGroups') ? tagsByResource['Microsoft.Storage/networkSecurityGroups'] : {}) properties: { securityRules: [ { @@ -164,6 +171,7 @@ resource nsg 'Microsoft.Network/networkSecurityGroups@2023-11-01' = { resource vNet 'Microsoft.Network/virtualNetworks@2023-11-01' = { name: vNetName location: location + tags: union(tags, contains(tagsByResource, 'Microsoft.Storage/virtualNetworks') ? tagsByResource['Microsoft.Storage/virtualNetworks'] : {}) properties: { addressSpace: { addressPrefixes: [virtualNetworkAddressPrefix]