6.4.4

JulianHayward · Apr 17, 2024 · aa67339 · aa67339
1 parent d777afa
commit aa67339
Show file tree

Hide file tree

Showing 9 changed files with 2,803 additions and 2,825 deletions.
diff --git a/README.md b/README.md
@@ -87,24 +87,14 @@ As an alternative, you can use the [Azure Governance Visualizer accelerator](htt
 
 ## Release history
 
-__Changes__ (2024-Mar-19 / 6.4.3 Minor)
+__Changes__ (2024-Apr-17 / 6.4.4 Minor)
 
-* Support for `-DoAzureConsumptionPreviousMonth` - Azure Consumption data should be collected/reported for the previous month
-
-__Changes__ (2024-Mar-14 / 6.4.2 Minor)
-
-* optimize objects handling / best practices
-
-__Changes__ (2024-Feb-06 / 6.4.0 Minor)
-
-* change PowerShell parallel handling / batches
-* add addition JSON outputs  'definitions_tracking' and 'assignments_tracking' (JSON filenames have no displayName included; GUIDs only)
-* update ARM API-version for RBAC Role definitions. Using `2022-05-01-preview` instead of `2018-11-01-preview` consequently
-* fix *_roleDefinitions.csv - description partially missing
-* optimize array handling / best practices
-* optimize getting private endpoint capable resource types / in case resource provider 'microsoft.network' is not registered, try with next available subscription instead of throwing
-* use [AzAPICall](https://aka.ms/AzAPICall) PowerShell module version 1.2.0
-* documentation update - style guidance, links updates - kudos @ckittel
+* fix issue #230
+  * use [AzAPICall](https://aka.ms/AzAPICall) PowerShell module version 1.2.1
+* update [API reference](#api-reference) Microsoft.Security/pricings use API version 2024-01-01 (previous 2018-06-01)
+* add 'Mutate' to `ValidPolicyEffects`
+* location related tasks - use only physical locations (exclude logical)
+* optimize collection of Role definitions that are used in Policy definitions
 
 [Full release history](history.md)
 
@@ -614,7 +604,7 @@ Azure Governance Visualizer polls the following APIs
 | ARM      | 2022-05-01         | /subscriptions/`subscriptionId`/providers/Microsoft.Network/privateEndpoints                                                           |
 | ARM      | 2022-05-01         | /subscriptions/`subscriptionId`/providers/Microsoft.Network/virtualNetworks                                                            |
 | ARM      | 2020-06-01         | /subscriptions/`subscriptionId`/providers/Microsoft.Resources/tags/default                                                             |
-| ARM      | 2018-06-01         | /subscriptions/`subscriptionId`/providers/Microsoft.Security/pricings                                                                  |
+| ARM      | 2024-01-01         | /subscriptions/`subscriptionId`/providers/Microsoft.Security/pricings                                                                  |
 | ARM      | 2020-01-01         | /subscriptions/`subscriptionId`/providers/Microsoft.Security/securescores                                                              |
 | ARM      | 2020-01-01-preview | /subscriptions/`subscriptionId`/providers/Microsoft.Security/securityContacts                                                          |
 | ARM      | 2019-10-01         | /subscriptions/`subscriptionId`/providers                                                                                              |

diff --git a/history.md b/history.md
@@ -4,6 +4,15 @@
 
 ### Azure Governance Visualizer version 6
 
+__Changes__ (2024-Apr-17 / 6.4.4 Minor)
+
+* fix issue #230
+  * use [AzAPICall](https://aka.ms/AzAPICall) PowerShell module version 1.2.1
+* update [API reference](#api-reference) Microsoft.Security/pricings use API version 2024-01-01 (previous 2018-06-01)
+* add 'Mutate' to `ValidPolicyEffects`
+* location related tasks - use only physical locations (exclude logical)
+* optimize collection of Role definitions that are used in Policy definitions
+
 __Changes__ (2024-Mar-19 / 6.4.3 Minor)
 
 * Support for `-DoAzureConsumptionPreviousMonth` - Azure Consumption data should be collected/reported for the previous month

diff --git a/pwsh/AzGovVizParallel.ps1 b/pwsh/AzGovVizParallel.ps1
diff --git a/pwsh/dev/devAzGovVizParallel.ps1 b/pwsh/dev/devAzGovVizParallel.ps1
@@ -365,14 +365,14 @@ Param
     $Product = 'AzGovViz',
 
     [string]
-    $ProductVersion = '6.4.3',
+    $ProductVersion = '6.4.4',
 
     [string]
     $GithubRepository = 'aka.ms/AzGovViz',
 
     # <--- AzAPICall related parameters #consult the AzAPICall GitHub repository for details aka.ms/AzAPICall
     [string]
-    $AzAPICallVersion = '1.2.0',
+    $AzAPICallVersion = '1.2.1',
 
     [switch]
     $DebugAzAPICall,
@@ -627,7 +627,7 @@ Param
     $MSTenantIds = @('2f4a9838-26b7-47ee-be60-ccc1fdec5953', '33e01921-4d64-4f8c-a055-5bdaffd5e33d'),
 
     [array]
-    $ValidPolicyEffects = @('append', 'audit', 'auditIfNotExists', 'deny', 'denyAction', 'deployIfNotExists', 'modify', 'manual', 'disabled', 'EnforceRegoPolicy', 'enforceSetting')
+    $ValidPolicyEffects = @('append', 'audit', 'auditIfNotExists', 'deny', 'denyAction', 'deployIfNotExists', 'modify', 'manual', 'disabled', 'EnforceRegoPolicy', 'enforceSetting', 'Mutate')
 )
 
 $Error.clear()
@@ -648,6 +648,7 @@ if ($ManagementGroupId -match ' ') {
 }
 
 #region Functions
+. ".\$($ScriptPath)\functions\getPrivateEndpointCapableResourceTypes.ps1"
 . ".\$($ScriptPath)\functions\validateLeastPrivilegeForUser.ps1"
 . ".\$($ScriptPath)\functions\getPolicyRemediation.ps1"
 . ".\$($ScriptPath)\functions\getPolicyHash.ps1"
@@ -820,14 +821,14 @@ if (-not $ignoreARMLocation) {
 #EndRegion initAZAPICall
 
 #region required AzAPICall version
-if (-not ([System.Version]"$($azapicallConf['htParameters'].azAPICallModuleVersion)" -ge [System.Version]'1.1.84')) {
+if (-not ([System.Version]"$($azapicallConf['htParameters'].azAPICallModuleVersion)" -ge [System.Version]'1.2.1')) {
     Write-Host ''
     Write-Host 'Azure Governance Visualizer version '$ProductVersion' - AzAPICall PowerShell module version check failed -> https://aka.ms/AzAPICall; https://www.powershellgallery.com/packages/AzAPICall'
-    throw "This version of Azure Governance Visualizer '$ProductVersion' requires AzAPICall PowerShell module version '1.1.84' or greater"
+    throw "This version of Azure Governance Visualizer '$ProductVersion' requires AzAPICall PowerShell module version '1.2.1' or greater"
 }
 else {
     Write-Host ''
-    Write-Host "Azure Governance Visualizer version '$ProductVersion' - AzAPICall PowerShell module version requirement check succeeded: '1.1.84' or greater - current: '$($azapicallConf['htParameters'].azAPICallModuleVersion)' " -ForegroundColor Green
+    Write-Host "Azure Governance Visualizer version '$ProductVersion' - AzAPICall PowerShell module version requirement check succeeded: '1.2.1' or greater - current: '$($azapicallConf['htParameters'].azAPICallModuleVersion)' " -ForegroundColor Green
 }
 #endregion required AzAPICall version
 
@@ -1160,93 +1161,7 @@ if (-not $HierarchyMapOnly) {
         Write-Host "Getting Tenant Resource Providers duration: $((New-TimeSpan -Start $startGetRPs -End $endGetRPs).TotalMinutes) minutes ($((New-TimeSpan -Start $startGetRPs -End $endGetRPs).TotalSeconds) seconds)"
         #endregion Getting Tenant Resource Providers
 
-        #region Getting Available Private Endpoint Types
-        $startGetAvailablePrivateEndpointTypes = Get-Date
-        $privateEndpointAvailabilityCheckCompleted = $false
-        $subsToProcessForGettingPrivateEndpointTypes = [System.Collections.ArrayList]@()
-        $prioCounter = 0
-        foreach ($subscription in $subsToProcessInCustomDataCollection) {
-            $prioCounter++
-            if ($subscription.subscriptionId -eq $azAPICallConf['checkcontext'].Subscription.Id) {
-                $null = $subsToProcessForGettingPrivateEndpointTypes.Add([PSCustomObject]@{
-                        subscriptionInfo = $subscription
-                        prio             = 0
-                    })
-            }
-            else {
-                $null = $subsToProcessForGettingPrivateEndpointTypes.Add([PSCustomObject]@{
-                        subscriptionInfo = $subscription
-                        prio             = $prioCounter
-                    })
-            }
-        }
-
-        foreach ($subscription in $subsToProcessForGettingPrivateEndpointTypes | Sort-Object -Property prio) {
-
-            if ($privateEndpointAvailabilityCheckCompleted) {
-                continue
-            }
-
-            $subscriptionId = $subscription.subscriptionInfo.subscriptionId
-            $subscriptionName = $subscription.subscriptionInfo.subscriptionName
-
-            $currentTask = "Getting Locations for Subscription '$($subscriptionName)' ($($subscriptionId))"
-            Write-Host $currentTask
-            $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/subscriptions/$($subscriptionId)/locations?api-version=2020-01-01"
-            $method = 'GET'
-            $getLocations = AzAPICall -AzAPICallConfiguration $azAPICallConf -uri $uri -method $method -currentTask $currentTask
-            Write-Host " Returned $($getLocations.Count) locations"
-
-            Write-Host "Getting 'Available Private Endpoint Types' for Subscription '$($subscriptionName)' ($($subscriptionId)) for $($getLocations.Count) locations"
-
-            $batchSize = [math]::ceiling($getLocations.Count / $ThrottleLimit)
-            Write-Host "Optimal batch size: $($batchSize)"
-            $counterBatch = [PSCustomObject] @{ Value = 0 }
-            $getLocationsBatch = ($getLocations) | Group-Object -Property { [math]::Floor($counterBatch.Value++ / $batchSize) }
-            Write-Host "Processing data in $($getLocationsBatch.Count) batches"
-
-            $getLocationsBatch | ForEach-Object -Parallel {
-                $subscriptionId = $using:subscriptionId
-                $azAPICallConf = $using:azAPICallConf
-                $htAvailablePrivateEndpointTypes = $using:htAvailablePrivateEndpointTypes
-
-                foreach ($location in $_.Group) {
-                    $currentTask = "Getting 'Available Private Endpoint Types' for location $($location.name)"
-                    #Write-Host $currentTask
-                    $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/subscriptions/$($subscriptionId)/providers/Microsoft.Network/locations/$($location.name)/availablePrivateEndpointTypes?api-version=2022-07-01"
-                    $method = 'GET'
-                    $availablePrivateEndpointTypes = AzAPICall -AzAPICallConfiguration $azAPICallConf -uri $uri -method $method -currentTask $currentTask -skipOnErrorCode 400, 409
-                    Write-Host " Returned $($availablePrivateEndpointTypes.Count) 'Available Private Endpoint Types' for location $($location.name)"
-                    foreach ($availablePrivateEndpointType in $availablePrivateEndpointTypes) {
-                        if (-not $htAvailablePrivateEndpointTypes.(($availablePrivateEndpointType.resourceName).ToLower())) {
-                            $script:htAvailablePrivateEndpointTypes.(($availablePrivateEndpointType.resourceName).ToLower()) = @{}
-                        }
-                    }
-                }
-            } -ThrottleLimit $ThrottleLimit
-
-            if ($htAvailablePrivateEndpointTypes.Keys.Count -gt 0) {
-                #Write-Host " Created ht for $($htAvailablePrivateEndpointTypes.Keys.Count) 'Available Private Endpoint Types'"
-                $privateEndpointAvailabilityCheckCompleted = $true
-            }
-            else {
-                Write-Host " $($htAvailablePrivateEndpointTypes.Keys.Count) 'Available Private Endpoint Types' - likely the Resource Provider 'Microsoft.Network' is not registered - trying next available subscription"
-                $privateEndpointAvailabilityCheckCompleted = $false
-            }
-        }
-
-        if ($htAvailablePrivateEndpointTypes.Keys.Count -gt 0) {
-            Write-Host " Created ht for $($htAvailablePrivateEndpointTypes.Keys.Count) 'Available Private Endpoint Types'"
-        }
-        else {
-            $throwmsg = "$($htAvailablePrivateEndpointTypes.Keys.Count) 'Available Private Endpoint Types' - Checked for $($subsToProcessForGettingPrivateEndpointTypes.Count) Subscriptions with no success. Make sure that for at least one Subscription the Resource Provider 'Microsoft.Network' is registered. Once you registered the Resource Provider for Subscription 'subscriptionEnabled' it may be a good idea to use the parameter: -SubscriptionId4AzContext '<subscriptionId of subscriptionEnabled>'"
-            Write-Host $throwmsg -ForegroundColor DarkRed
-            Throw $throwmsg
-        }
-
-        $endGetAvailablePrivateEndpointTypes = Get-Date
-        Write-Host "Getting 'Available Private Endpoint Types' duration: $((New-TimeSpan -Start $startGetAvailablePrivateEndpointTypes -End $endGetAvailablePrivateEndpointTypes).TotalMinutes) minutes ($((New-TimeSpan -Start $startGetAvailablePrivateEndpointTypes -End $endGetAvailablePrivateEndpointTypes).TotalSeconds) seconds)"
-        #endregion Getting Available Private Endpoint Types
+        getPrivateEndpointCapableResourceTypes
     }
 
     Write-Host 'Collecting custom data'

diff --git a/pwsh/dev/functions/cacheBuiltIn.ps1 b/pwsh/dev/functions/cacheBuiltIn.ps1
@@ -79,15 +79,12 @@ function cacheBuiltIn {
                 if (-not [string]::IsNullOrWhiteSpace($builtinPolicyDefinition.properties.policyRule.then.details.roleDefinitionIds)) {
                     $script:htCacheDefinitionsPolicy.(($builtinPolicyDefinition.Id).ToLower()).RoleDefinitionIds = $builtinPolicyDefinition.properties.policyRule.then.details.roleDefinitionIds
                     foreach ($roledefinitionId in $builtinPolicyDefinition.properties.policyRule.then.details.roleDefinitionIds) {
-                        if (-not $htRoleDefinitionIdsUsedInPolicy.($roledefinitionId)) {
-                            $script:htRoleDefinitionIdsUsedInPolicy.($roledefinitionId) = @{
+                        if (-not $htRoleDefinitionIdsUsedInPolicy.(($roledefinitionId).ToLower())) {
+                            $script:htRoleDefinitionIdsUsedInPolicy.(($roledefinitionId).ToLower()) = @{
                                 UsedInPolicies = [System.Collections.ArrayList]@()
                             }
-                            #$null = $script:htRoleDefinitionIdsUsedInPolicy.($roledefinitionId).UsedInPolicies.Add($builtinPolicyDefinition.Id)
                         }
-                        #else {
-                        $null = $script:htRoleDefinitionIdsUsedInPolicy.($roledefinitionId).UsedInPolicies.Add($builtinPolicyDefinition.Id)
-                        #}
+                        $null = $script:htRoleDefinitionIdsUsedInPolicy.(($roledefinitionId).ToLower()).UsedInPolicies.Add($builtinPolicyDefinition.Id)
                     }
                 }
                 else {
@@ -170,15 +167,12 @@ function cacheBuiltIn {
                 if (-not [string]::IsNullOrWhiteSpace($staticPolicyDefinition.properties.policyRule.then.details.roleDefinitionIds)) {
                     $script:htCacheDefinitionsPolicy.(($staticPolicyDefinition.Id).ToLower()).RoleDefinitionIds = $staticPolicyDefinition.properties.policyRule.then.details.roleDefinitionIds
                     foreach ($roledefinitionId in $staticPolicyDefinition.properties.policyRule.then.details.roleDefinitionIds) {
-                        if (-not $htRoleDefinitionIdsUsedInPolicy.($roledefinitionId)) {
-                            $script:htRoleDefinitionIdsUsedInPolicy.($roledefinitionId) = @{
+                        if (-not $htRoleDefinitionIdsUsedInPolicy.(($roledefinitionId).ToLower())) {
+                            $script:htRoleDefinitionIdsUsedInPolicy.(($roledefinitionId).ToLower()) = @{
                                 UsedInPolicies = [System.Collections.ArrayList]@()
                             }
-                            #$null = $script:htRoleDefinitionIdsUsedInPolicy.($roledefinitionId).UsedInPolicies.Add($staticPolicyDefinition.Id)
                         }
-                        #else {
-                        $null = $script:htRoleDefinitionIdsUsedInPolicy.($roledefinitionId).UsedInPolicies.Add($staticPolicyDefinition.Id)
-                        #}
+                        $null = $script:htRoleDefinitionIdsUsedInPolicy.(($roledefinitionId).ToLower()).UsedInPolicies.Add($staticPolicyDefinition.Id)
                     }
                 }
                 else {

diff --git a/pwsh/dev/functions/dataCollection/dataCollectionFunctions.ps1 b/pwsh/dev/functions/dataCollection/dataCollectionFunctions.ps1
@@ -28,7 +28,7 @@ function dataCollectionDefenderPlans {
 
     $currentTask = "Getting Microsoft Defender for Cloud plans for Subscription: '$($scopeDisplayName)' ('$scopeId') [quotaId:'$SubscriptionQuotaId']"
     #https://learn.microsoft.com/rest/api/defenderforcloud/pricings
-    $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/subscriptions/$($scopeId)/providers/Microsoft.Security/pricings?api-version=2018-06-01"
+    $uri = "$($azAPICallConf['azAPIEndpointUrls'].ARM)/subscriptions/$($scopeId)/providers/Microsoft.Security/pricings?api-version=2024-01-01"
     $method = 'GET'
     $defenderPlansResult = AzAPICall -AzAPICallConfiguration $azAPICallConf -uri $uri -method $method -currentTask $currentTask -caller 'CustomDataCollection'
 
@@ -2194,15 +2194,12 @@ function dataCollectionPolicyDefinitions {
                 $script:htCacheDefinitionsPolicy.($hlpPolicyDefinitionId).RoleDefinitionIds = $scopePolicyDefinition.properties.policyRule.then.details.roleDefinitionIds
                 foreach ($roledefinitionId in $scopePolicyDefinition.properties.policyRule.then.details.roleDefinitionIds) {
                     if (-not [string]::IsNullOrEmpty($roledefinitionId)) {
-                        if (-not $htRoleDefinitionIdsUsedInPolicy.($roledefinitionId)) {
-                            $script:htRoleDefinitionIdsUsedInPolicy.($roledefinitionId) = @{
+                        if (-not $htRoleDefinitionIdsUsedInPolicy.(($roledefinitionId).ToLower())) {
+                            $script:htRoleDefinitionIdsUsedInPolicy.(($roledefinitionId).ToLower()) = @{
                                 UsedInPolicies = [System.Collections.ArrayList]@()
                             }
-                            #$null = $script:htRoleDefinitionIdsUsedInPolicy.($roledefinitionId).UsedInPolicies.Add($hlpPolicyDefinitionId)
                         }
-                        #else {
-                        $script:htRoleDefinitionIdsUsedInPolicy.($roledefinitionId).UsedInPolicies.Add($hlpPolicyDefinitionId)
-                        #}
+                        $script:htRoleDefinitionIdsUsedInPolicy.(($roledefinitionId).ToLower()).UsedInPolicies.Add($hlpPolicyDefinitionId)
                     }
                     else {
                         Write-Host "$currentTask $($hlpPolicyDefinitionId) Finding: empty roleDefinitionId in roledefinitionIds"