v6_major_20220425_1

JulianHayward · Apr 26, 2022 · a5da571 · a5da571
1 parent 4ca52e6
commit a5da571
Show file tree

Hide file tree

Showing 59 changed files with 47,156 additions and 21,239 deletions.
diff --git a/.azuredevops/pipelines/AzGovViz.pipeline.yml b/.azuredevops/pipelines/AzGovViz.pipeline.yml
@@ -35,7 +35,7 @@ variables:
 #      trigger:
 #        branches:
 #          include:
-#            - master #CHECK: branch 'master' is applicable? - delete me :)
+#            - master #CHECK branch 'master' is applicable? - delete me :)
 
 jobs:
 - job: AzGovViz

diff --git a/.azuredevops/pipelines/AzGovViz.variables.yml b/.azuredevops/pipelines/AzGovViz.variables.yml
@@ -221,10 +221,10 @@ variables:
     # Integer | Default = 5 | example: value: 10
     value:
 
-  # Will show memory usage //pending release
-  #- name: ShowMemoryUsage
-  #  # Switch | example: value: true
-  #  value:
+  # Will show memory usage
+  - name: ShowMemoryUsage
+    # Switch | example: value: true
+    value:
 
 # Dynamic Variables - Do Not Modify Anything Below this line!
   - name: ExcludedResourceTypesDiagnosticsCapable

diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json
@@ -1,13 +1,15 @@
 {
-	"name": "AzGovViz",
-	"dockerFile": "Dockerfile",
-	"settings": {
-		"terminal.integrated.defaultProfile.linux": "pwsh"
-	},
-	"extensions": [
-		"ms-vscode.powershell",
-		"analytic-signal.preview-html",
-		"bierner.markdown-mermaid"
-	],
-	"forwardPorts": []
-}
+    "name": "AzGovViz",
+    "dockerFile": "Dockerfile",
+    "settings": {
+        "terminal.integrated.defaultProfile.linux": "pwsh"
+    },
+    "extensions": [
+        "ms-vscode.powershell",
+        "analytic-signal.preview-html",
+        "bierner.markdown-mermaid",
+        "streetsidesoftware.code-spell-checker",
+        "yzhang.markdown-all-in-one"
+    ],
+    "forwardPorts": []
+}
diff --git a/.github/workflows/AzGovViz.yml b/.github/workflows/AzGovViz.yml
@@ -1,13 +1,16 @@
-# AzGovViz v6_major_20220116_2
+# AzGovViz v6_major_20220319_1
 name: AzGovViz
 
 env:
-  outputpath: wiki
-  ManagementGroupId: <managementGroupId>
+  OutputPath: wiki
+  ManagementGroupId: <Management Group Id>
+  ScriptDir: pwsh #example: 'my folder\pwsh' or 'my folder/pwsh'
+  ScriptPrereqFile: prerequisites.ps1
+  ScriptFile: AzGovVizParallel.ps1
 
 on:
   #schedule:
-  #  - cron:  '45 4,16 * * *'
+  #  - cron:  '30 4 * * *'
 
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
@@ -17,6 +20,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
       - name: Checkout
         uses: actions/checkout@v2
 
@@ -32,19 +36,13 @@ jobs:
           #   "clientId": "<GUID>",
           #   "clientSecret": "<GUID>"
           # }
-
-      - name: Check prerequisites
-        uses: azure/powershell@v1
-        with:
-          inlineScript: |
-            .\pwsh\prerequisites.ps1
-          azPSVersion: "latest"
 
       - name: Run AzGovViz
         uses: azure/powershell@v1
         with:
           inlineScript: |
-            .\pwsh\AzGovVizParallel.ps1 -ManagementGroupId ${env:ManagementGroupId} -outputpath ${env:outputpath}
+            . .\$($env:ScriptDir)\$($env:ScriptPrereqFile)
+            . .\$($env:ScriptDir)\$($env:ScriptFile) -ManagementGroupId ${env:ManagementGroupId} -ScriptPath ${env:ScriptDir} -OutputPath ${env:OutputPath}
           azPSVersion: "latest"
 
       - name: Push AzGovViz output to repository
@@ -54,4 +52,4 @@ jobs:
           git config pull.rebase false
           git add --all
           git commit -m "$GITHUB_WORKFLOW $GITHUB_JOB"
-          git push
+          git push
diff --git a/.github/workflows/AzGovViz_OIDC.yml b/.github/workflows/AzGovViz_OIDC.yml
@@ -1,13 +1,16 @@
-# AzGovViz v6_major_20220116_2
+# AzGovViz v6_major_20220319_1
 name: AzGovViz_OIDC
 
 env:
-  outputpath: wiki
-  ManagementGroupId: <managementGroupId>
+  OutputPath: wiki
+  ManagementGroupId: <Management Group Id>
+  ScriptDir: pwsh #example: 'my folder\pwsh' or 'my folder/pwsh'
+  ScriptPrereqFile: prerequisites.ps1
+  ScriptFile: AzGovVizParallel.ps1
 
 on:
   #schedule:
-  #  - cron:  '45 5,17 * * *'
+  # - cron:  '30 5 * * *'
 
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
@@ -28,23 +31,17 @@ jobs:
       - name: Connect Azure OIDC
         uses: azure/login@v1
         with:
-          client-id: ${{secrets.CLIENT_ID}}
-          tenant-id: ${{secrets.TENANT_ID}}
-          subscription-id: ${{secrets.SUBSCRIPTION_ID}} 
+          client-id: ${{secrets.CLIENT_ID}} #create this secret
+          tenant-id: ${{secrets.TENANT_ID}} #create this secret
+          subscription-id: ${{secrets.SUBSCRIPTION_ID}} #create this secret
           enable-AzPSSession: true
 
-      - name: Check prerequisites
-        uses: azure/powershell@v1
-        with:
-          inlineScript: |
-            .\pwsh\prerequisites.ps1
-          azPSVersion: "latest"
-
       - name: Run AzGovViz
         uses: azure/powershell@v1
         with:
           inlineScript: |
-            .\pwsh\AzGovVizParallel.ps1 -ManagementGroupId ${env:ManagementGroupId} -SubscriptionId4AzContext ${{secrets.SUBSCRIPTION_ID}} -outputpath ${env:outputpath}
+            . .\$($env:ScriptDir)\$($env:ScriptPrereqFile)
+            . .\$($env:ScriptDir)\$($env:ScriptFile) -ManagementGroupId ${env:ManagementGroupId} -SubscriptionId4AzContext ${{secrets.SUBSCRIPTION_ID}} -ScriptPath ${env:ScriptDir} -OutputPath ${env:OutputPath}
           azPSVersion: "latest"
 
       - name: Push AzGovViz output to repository
@@ -54,4 +51,4 @@ jobs:
           git config pull.rebase false
           git add --all
           git commit -m "$GITHUB_WORKFLOW $GITHUB_JOB"
-          git push
+          git push
diff --git a/.pipelines/AzGovViz.yml b/.pipelines/AzGovViz.yml
@@ -1,4 +1,4 @@
-# AzGovViz v6_major_20220114_1
+# AzGovViz v6_major_20220319_1
 # First things first: 
 # 1. edit line 61 and line 62 
 # 2. check line 76 and 87 if branch 'master' is applicable 
@@ -73,7 +73,7 @@ schedules:
   always: true
   branches:
     include:
-    - master #CHECK: branch 'master' is applicable? - delete me :)
+    - master #CHECK branch 'master' is applicable? - delete me :)
 
 #Running AzOps? Run AzGovViz after 'AzOps - Push' ..
 #AzOps Accellerator https://github.com/Azure/AzOps-Accelerator
@@ -107,7 +107,7 @@ jobs:
       scriptType: filePath
       pwsh: true
       scriptPath: '$(System.DefaultWorkingDirectory)/$(ScriptDir)/$(Script)'
-      scriptArguments: '-ManagementGroupId $(ManagementGroupId) -OutputPath $(WikiDir) -CsvDelimiter "$(CsvDelimiter)" -SubscriptionQuotaIdWhitelist $(SubscriptionQuotaIdWhitelist) -ExludedResourceTypesDiagnosticsCapable $(ExludedResourceTypesDiagnosticsCapable)'
+      scriptArguments: '-ManagementGroupId $(ManagementGroupId) -ScriptPath $(ScriptDir) -OutputPath $(WikiDir) -CsvDelimiter "$(CsvDelimiter)" -SubscriptionQuotaIdWhitelist $(SubscriptionQuotaIdWhitelist) -ExludedResourceTypesDiagnosticsCapable $(ExludedResourceTypesDiagnosticsCapable)'
       azurePowerShellVersion: latestVersion
     displayName: 'Run AzGovViz v6'
   - powershell: |

diff --git a/.pipelines/README.md b/.pipelines/README.md
@@ -0,0 +1,3 @@
+# [Deprecated] `AzGovViz.yml`
+
+This version of the pipeline will be deprecated. Check the new [pipeline YAML](../.azuredevops/pipelines) approach (`AzGovViz.pipeline.yml` & `AzGovViz.variables.yml`)!
diff --git a/README.md b/README.md
@@ -45,6 +45,7 @@ Listed as [security monitoring tool](https://docs.microsoft.com/en-us/azure/arch
   * [Required permissions in Azure Active Directory](#required-permissions-in-azure-active-directory)
   * [PowerShell](#powershell)
   * [Parameters](#parameters)
+  * [API](#api)
 * [Integrate with AzOps](#integrate-with-azops)
 * [Stats](#stats)
 * [Security](#security)
@@ -57,24 +58,22 @@ Listed as [security monitoring tool](https://docs.microsoft.com/en-us/azure/arch
 
 ## Release history
 
-__Changes__ (2022-Jan-31 / Major)
-
-* New __TenantSummary | RBAC__ feature - insights on all Role definitions that are capable to write Role assignments
-* __TenantSummary | Subscriptions, Resources & Defender | Subscriptions__ report (new) [Role assignment limits](https://docs.microsoft.com/en-us/azure/role-based-access-control/troubleshooting#azure-role-assignments-limit)
-* Handling orphaned Policy assignments (scope Management Group)
-* Datacollection for Management Groups process in batches (batch per Management Group level)
-* Update Dockerfile
-* Update API version for Resources, ResourceGroups and Subscriptions
-* Further enrich _PolicyDefinitions and _PolicySetDefinitions CSV outputs
-* HTML file performance optimization
-* Include instructions for GitHub Actions in the __[Setup Guide](setup.md)__
-* New [demo](https://www.azadvertizer.net/azgovvizv4/demo/AzGovViz_demo.html) uploaded
+__Changes__ (2022-Apr-25 / Major)
+
+* New JSON output *_PolicyAll.json - Contains all relations of Policy/Set definitions and Policy assignments
+* New parameter `-ShowMemoryUsage` - Shows memory usage at memory intense sections of the scripts, this shall help you determine if the the worker is well sized for AzGovViz
+* Leveraging AzAPICall PowerShell module. The AzAPICall function has been removed from the AzGovViz code base and has been published as a module to the [PoweShell Gallery](https://www.powershellgallery.com/packages/AzAPICall) ([GitHub](https://aka.ms/AzAPICall))
+* Foreach -parallel import the AzAPICall module instead of $using:
+* Optimize GitHub Actions workflows (YAML)
+* Added list of [APIs](#api) that are polled by AzGovViz
+* Microsoft Graph `v1.0/directoryObjects/getByIds` do batching is exceeds 1000 identities
+* Performance optimization
 * Bugfixes
 
-Passed tests: Powershell Core 7.2.1 on Windows  
-Passed tests: Powershell Core 7.2.1 Azure DevOps hosted agent ubuntu-18.04  
-Passed tests: Powershell Core 7.2.1 Github Actions hosted agent ubuntu-latest  
-Passed tests: Powershell Core 7.2.1 GitHub Codespaces mcr.microsoft.com/powershell:latest  
+Passed tests: Powershell Core 7.2.2 on Windows  
+Passed tests: Powershell Core 7.2.2 Azure DevOps hosted agent ubuntu-18.04  
+Passed tests: Powershell Core 7.2.2 Github Actions hosted agent ubuntu-latest  
+Passed tests: Powershell Core 7.2.2 GitHub Codespaces mcr.microsoft.com/powershell:latest  
 Passed tests: AzureCloud, AzureUSGovernment, AzureChinaCloud
 
 [Release history](history.md)
@@ -401,6 +400,11 @@ Screenshot Azure Portal
   * ~~Az.Resources~~
   * ~~Az.ResourceGraph~~
   * [Install the Azure Az PowerShell module](https://docs.microsoft.com/en-us/powershell/azure/install-az-ps)
+* Requires PowerShell Module 'AzAPICall'.  
+Running in Azure DevOps or GitHub Actions the AzAPICall PowerShell module will be installed automatically.  
+AzAPICall resources:
+  * [![PowerShell Gallery Version (including pre-releases)](https://img.shields.io/powershellgallery/v/AzAPICall?include_prereleases&label=PowerShell%20Gallery)](https://www.powershellgallery.com/packages/AzAPICall)  
+  * [GitHub Repository](https://aka.ms/AzAPICall)
 * Usage/command
   * `.\AzGovVizParallel.ps1 -ManagementGroupId <your-Management-Group-Id>`
 
@@ -446,6 +450,63 @@ Screenshot Azure Portal
   * `-StatsOptOut` - Opt out sending [stats](#stats)
   * `-NoSingleSubscriptionOutput` - Single __Scope Insights__ output per Subscription should not be created
   * `-ManagementGroupsOnly` - Collect data only for Management Groups (Subscription data such as e.g. Policy assignments etc. will not be collected)
+  * `-ShowMemoryUsage` - Shows memory usage at memory intense sections of the scripts, this shall help you determine if the the worker is well sized for AzGovViz
+
+### API
+
+AzGovViz polls the following APIs
+
+| Endpoint | API version | API name |
+| --- | --- | --- |
+| MS Graph | beta | /groups/`aadGroupId`/transitiveMembers |
+| MS Graph | v1.0 | /applications |
+| MS Graph | v1.0 | /directoryObjects/getByIds |
+| MS Graph | v1.0 | /users |
+| MS Graph | v1.0 | /groups |
+| MS Graph | v1.0 | /servicePrincipals |
+| ARM |2021-05-01-preview | /`resourceId`/providers/Microsoft.Insights/diagnosticSettingsCategories |
+| ARM |2018-11-01-preview | /`scopeId`/providers/Microsoft.Blueprint/blueprints/`blueprintName` |
+| ARM |2021-06-01 | /providers/Microsoft.Authorization/policyDefinitions |
+| ARM |2021-06-01 | /providers/Microsoft.Authorization/policySetDefinitions |
+| ARM |2018-07-01 | /providers/Microsoft.Authorization/roleDefinitions |
+| ARM |2020-02-01 | /providers/Microsoft.Management/getEntities |
+| ARM |2021-06-01 | /providers/Microsoft.Management/managementGroups/`managementGroupId`/providers/Microsoft.Authorization/policyAssignments |
+| ARM |2021-06-01 | /providers/Microsoft.Management/managementGroups/`managementGroupId`/providers/Microsoft.Authorization/policyDefinitions |
+| ARM |2020-07-01-preview | /providers/Microsoft.Management/managementGroups/`managementGroupId`/providers/Microsoft.Authorization/policyExemptions |
+| ARM |2021-06-01 | /providers/Microsoft.Management/managementGroups/`managementGroupId`/providers/Microsoft.Authorization/policySetDefinitions |
+| ARM |2015-07-01 | /providers/Microsoft.Management/managementGroups/`managementGroupId`/providers/Microsoft.Authorization/roleAssignments |
+| ARM |2020-10-01-preview | /providers/Microsoft.Management/managementGroups/`managementGroupId`/providers/Microsoft.Authorization/roleAssignmentSchedules |
+| ARM |2015-07-01 | /providers/Microsoft.Management/managementGroups/`managementGroupId`/providers/Microsoft.Authorization/roleDefinitions |
+| ARM |2018-11-01-preview | /providers/Microsoft.Management/managementGroups/`managementGroupId`/providers/Microsoft.Blueprint/blueprints |
+| ARM |2019-11-01 | /providers/Microsoft.Management/managementGroups/`managementGroupId`/providers/Microsoft.CostManagement/query |
+| ARM |2020-01-01-preview | /providers/Microsoft.Management/managementGroups/`managementGroupId`/providers/microsoft.insights/diagnosticSettings |
+| ARM |2019-10-01 | /providers/Microsoft.Management/managementGroups/`managementGroupId`/providers/Microsoft.PolicyInsights/policyStates/latest/summarize |
+| ARM |2020-05-01 | /providers/Microsoft.Management/managementGroups/`managementGroupId` |
+| ARM |2020-02-01 | /providers/Microsoft.Management/managementGroups/`tenantId`/settings |
+| ARM |2020-05-01 | /providers/Microsoft.Management/managementGroups |
+| ARM |2021-03-01 | /providers/Microsoft.ResourceGraph/resources |
+| ARM |2016-09-01 | /subscriptions/`subscriptionId`/providers/Microsoft.Authorization/locks |
+| ARM |2021-06-01 | /subscriptions/`subscriptionId`/providers/Microsoft.Authorization/policyAssignments |
+| ARM |2021-06-01 | /subscriptions/`subscriptionId`/providers/Microsoft.Authorization/policyDefinitions |
+| ARM |2020-07-01-preview | /subscriptions/`subscriptionId`/providers/Microsoft.Authorization/policyExemptions |
+| ARM |2021-06-01 | /subscriptions/`subscriptionId`/providers/Microsoft.Authorization/policySetDefinitions |
+| ARM |2015-07-01 | /subscriptions/`subscriptionId`/providers/Microsoft.Authorization/roleAssignments |
+| ARM |2020-10-01-preview | /subscriptions/`subscriptionId`/providers/Microsoft.Authorization/roleAssignmentSchedules |
+| ARM |2019-08-01-preview | /subscriptions/`subscriptionId`/providers/Microsoft.Authorization/roleAssignmentsUsageMetrics |
+| ARM |2015-07-01 | /subscriptions/`subscriptionId`/providers/Microsoft.Authorization/roleDefinitions |
+| ARM |2018-11-01-preview | /subscriptions/`subscriptionId`/providers/Microsoft.Blueprint/blueprintAssignments |
+| ARM |2018-11-01-preview | /subscriptions/`subscriptionId`/providers/Microsoft.Blueprint/blueprints |
+| ARM |2019-11-01 | /subscriptions/`subscriptionId`/providers/Microsoft.CostManagement/query |
+| ARM |2021-05-01-preview | /subscriptions/`subscriptionId`/providers/Microsoft.Insights/diagnosticSettings |
+| ARM |2019-10-01 | /subscriptions/`subscriptionId`/providers/Microsoft.PolicyInsights/policyStates/latest/summarize |
+| ARM |2020-06-01 | /subscriptions/`subscriptionId`/providers/Microsoft.Resources/tags/default |
+| ARM |2018-06-01 | /subscriptions/`subscriptionId`/providers/Microsoft.Security/pricings |
+| ARM |2020-01-01 | /subscriptions/`subscriptionId`/providers/Microsoft.Security/securescores |
+| ARM |2019-10-01 | /subscriptions/`subscriptionId`/providers |
+| ARM |2021-04-01 | /subscriptions/`subscriptionId`/resourcegroups |
+| ARM |2021-04-01 | /subscriptions/`subscriptionId`/resources |
+| ARM |2020-01-01 | /subscriptions |
+| ARM |2020-01-01 | /tenants |
 
 ## Integrate with AzOps
 

diff --git a/history.md b/history.md
@@ -4,6 +4,18 @@
 
 ### AzGovViz version 6
 
+__Changes__ (2022-Apr-25 / Major)
+
+* New JSON output *_PolicyAll.json - Contains all relations of Policy/Set definitions and Policy assignments
+* New parameter `-ShowMemoryUsage` - Shows memory usage at memory intense sections of the scripts, this shall help you determine if the the worker is well sized for AzGovViz
+* Leveraging AzAPICall PowerShell module. The AzAPICall function has been removed from the AzGovViz code base and has been published as a module to the [PoweShell Gallery](https://www.powershellgallery.com/packages/AzAPICall) ([GitHub](https://aka.ms/AzAPICall))
+* Foreach -parallel import the AzAPICall module instead of $using:
+* Optimize GitHub Actions workflows (YAML)
+* Added list of [APIs](#api) that are polled by AzGovViz
+* Microsoft Graph `v1.0/directoryObjects/getByIds` do batching is exceeds 1000 identities
+* Performance optimization
+* Bugfixes
+
 __Changes__ (2022-Jan-31 / Major)
 
 * New __TenantSummary | RBAC__ feature - insights on all Role definitions that are capable to write Role assignments