diff --git a/docs/_reporting/hubs/configure-scopes.md b/docs/_reporting/hubs/configure-scopes.md index e22dd17c1..c421bdcd8 100644 --- a/docs/_reporting/hubs/configure-scopes.md +++ b/docs/_reporting/hubs/configure-scopes.md @@ -43,7 +43,7 @@ If you cannot grant permissions for your scope, you can create Cost Management e 1. [Create a new FOCUS cost export](https://learn.microsoft.com/azure/cost-management-billing/costs/tutorial-export-acm-data?tabs=azure-portal) using the following settings: - **Type of data** = `Cost and usage details (FOCUS)`1 - - **Dataset version** = `1.0`2 + - **Dataset version** = `1.0` or `1.0r2`2 - **Frequency** = `Daily export of month-to-date costs`3 - **Storage account** = (Use subscription/resource deployed with your hub) - **Container** = `msexports` @@ -77,7 +77,7 @@ If you cannot grant permissions for your scope, you can create Cost Management e 5. Repeat steps 1-4 for each scope you want to monitor. _1) FinOps hubs 0.2 and beyond requires FOCUS cost data. As of July 2024, the option to export FOCUS cost data is only accessible from the central Cost Management experience in the Azure portal. If you do not see this option, please search for or navigate to [Cost Management Exports](https://portal.azure.com/#blade/Microsoft_Azure_CostManagement/Menu/open/exports)._ -_2) FinOps hubs 0.4 supports both FOCUS 1.0 and FOCUS 1.0 preview. Power BI reports in 0.4 are aligned to FOCUS 1.0 regardless of whether data was ingested as FOCUS 1.0 preview. If you need 1.0 preview data and reports, please use FinOps hubs 0.3._ +_2) FinOps hubs 0.4 supports FOCUS 1.0r2, 1.0, 1.0 preview. Power BI reports in 0.4 are aligned to FOCUS 1.0 regardless of whether data was ingested as FOCUS 1.0 preview. If you need 1.0 preview data and reports, please use FinOps hubs 0.3. The only difference in FOCUS 1.0r2 is the inclusion of seconds in date columns._ _3) Configuring a daily export starts in the current month. If you want to backfill historical data, create a one-time export and set the start/end dates to the desired date range._ _4) While most settings are required, overwriting is optional. We recommend **not** overwriting files so you can monitor your ingestion pipeline using the [Data ingestion](../power-bi/data-ingestion.md) report. If you do not plan to use that report, please enable overwriting._ _5) Export paths can be any value but must be unique per scope. We recommended using a path that identifies the source scope (e.g., subscription or billing account). If 2 scopes share the same path, there could be ingestion errors._ diff --git a/docs/_resources/changelog.md b/docs/_resources/changelog.md index a23766dd8..32ed9bae5 100644 --- a/docs/_resources/changelog.md +++ b/docs/_resources/changelog.md @@ -129,10 +129,6 @@ Legend: đŸĻ FinOps hubs {: .fs-5 .fw-500 .mt-4 mb-0 } -> ➕ Added: -> -> 1. Infrastructure encryption - Added an optional enableInfrastructureEncryption template parameter to support storage account infrastructure encryption. - **Breaking change** {: .label .label-red .pt-0 .pl-3 .pr-3 .m-0 } @@ -151,6 +147,7 @@ Legend: > - Added param to disable external access to Azure Data Lake and Azure Data Explorer. > - Added param to specify subnet range of virtual network - minimum size = /26 > 1. Support for storage account infrastructure encryption. +> 1. Published a [schema file](https://aka.ms/finops/hubs/settings-schema) for the hub settings.json file. > > ✏ī¸ Changed: > @@ -176,7 +173,7 @@ Legend: > ➕ Added: > > - [Optimization workbook](../_optimize/workbooks/optimization/README.md) -> 1. On the Storagetab, included the **RSVaultBackup** tag in the list of non-idle disks. +> 1. On the Storage tab, included the **RSVaultBackup** tag in the list of non-idle disks. > > 🛠ī¸ Fixed: > diff --git a/src/power-bi/kql/CostSummary.Report/report.json b/src/power-bi/kql/CostSummary.Report/report.json index efdeb4167..40c58151d 100644 --- a/src/power-bi/kql/CostSummary.Report/report.json +++ b/src/power-bi/kql/CostSummary.Report/report.json @@ -154,7 +154,7 @@ "z": 1000.00 }, { - "config": "{\"name\":\"907968064288c59539c5\",\"layouts\":[{\"id\":0,\"position\":{\"x\":15.562310030395137,\"y\":16,\"z\":0,\"width\":1264.0000000000002,\"height\":96,\"tabOrder\":5000}}],\"singleVisual\":{\"visualType\":\"textbox\",\"drillFilterOtherVisuals\":true,\"objects\":{\"general\":[{\"properties\":{\"paragraphs\":[{\"textRuns\":[{\"value\":\"Cost summary report\",\"textStyle\":{\"fontWeight\":\"bold\",\"fontSize\":\"42pt\"}},{\"value\":\" v24.11.28\",\"textStyle\":{\"color\":\"#808080\"}}]}]}}]},\"vcObjects\":{\"visualHeader\":[{\"properties\":{\"show\":{\"expr\":{\"Literal\":{\"Value\":\"false\"}}}}}],\"background\":[{\"properties\":{\"show\":{\"expr\":{\"Literal\":{\"Value\":\"false\"}}}}}]}}}", + "config": "{\"name\":\"907968064288c59539c5\",\"layouts\":[{\"id\":0,\"position\":{\"x\":15.562310030395137,\"y\":16,\"z\":0,\"width\":1264.0000000000002,\"height\":96,\"tabOrder\":5000}}],\"singleVisual\":{\"visualType\":\"textbox\",\"drillFilterOtherVisuals\":true,\"objects\":{\"general\":[{\"properties\":{\"paragraphs\":[{\"textRuns\":[{\"value\":\"Cost summary report\",\"textStyle\":{\"fontWeight\":\"bold\",\"fontSize\":\"42pt\"}},{\"value\":\" v24.11.30\",\"textStyle\":{\"color\":\"#808080\"}}]}]}}]},\"vcObjects\":{\"visualHeader\":[{\"properties\":{\"show\":{\"expr\":{\"Literal\":{\"Value\":\"false\"}}}}}],\"background\":[{\"properties\":{\"show\":{\"expr\":{\"Literal\":{\"Value\":\"false\"}}}}}]}}}", "filters": "[]", "height": 96.00, "width": 1264.00, diff --git a/src/power-bi/kql/CostSummary.SemanticModel/definition/cultures/en-US.tmdl b/src/power-bi/kql/CostSummary.SemanticModel/definition/cultures/en-US.tmdl index 9393adbc9..68911a66a 100644 --- a/src/power-bi/kql/CostSummary.SemanticModel/definition/cultures/en-US.tmdl +++ b/src/power-bi/kql/CostSummary.SemanticModel/definition/cultures/en-US.tmdl @@ -19844,82 +19844,6 @@ cultureInfo en-US } ] }, - "tag_cost.tag": { - "Definition": { - "Binding": { - "ConceptualEntity": "TagCosts", - "ConceptualProperty": "Tags" - } - }, - "State": "Generated", - "Terms": [ - { - "tag": { - "State": "Generated" - } - }, - { - "device": { - "Type": "Noun", - "State": "Suggested", - "Source": { - "Agent": "Thesaurus" - }, - "Weight": 0.491 - } - }, - { - "ticket": { - "Type": "Noun", - "State": "Suggested", - "Source": { - "Agent": "Thesaurus" - }, - "Weight": 0.476 - } - }, - { - "tab": { - "Type": "Noun", - "State": "Suggested", - "Source": { - "Agent": "Thesaurus" - }, - "Weight": 0.476 - } - }, - { - "docket": { - "Type": "Noun", - "State": "Suggested", - "Source": { - "Agent": "Thesaurus" - }, - "Weight": 0.476 - } - }, - { - "chip": { - "Type": "Noun", - "State": "Suggested", - "Source": { - "Agent": "Thesaurus" - }, - "Weight": 0.476 - } - }, - { - "mark": { - "Type": "Noun", - "State": "Suggested", - "Source": { - "Agent": "Thesaurus" - }, - "Weight": 0.476 - } - } - ] - }, "tag_cost.x_commitment_discount_savings": { "Definition": { "Binding": { diff --git a/src/power-bi/kql/CostSummary.SemanticModel/definition/tables/ChargeBreakdown.tmdl b/src/power-bi/kql/CostSummary.SemanticModel/definition/tables/ChargeBreakdown.tmdl index 45dd34e76..a4b5aa9b4 100644 --- a/src/power-bi/kql/CostSummary.SemanticModel/definition/tables/ChargeBreakdown.tmdl +++ b/src/power-bi/kql/CostSummary.SemanticModel/definition/tables/ChargeBreakdown.tmdl @@ -299,6 +299,24 @@ table ChargeBreakdown annotation PBI_FormatHint = {"isGeneralNumber":true} + column x_ChargePeriodMonth + dataType: dateTime + formatString: General Date + lineageTag: 487aae09-ebbf-43b1-baa2-5220b477575d + summarizeBy: none + sourceColumn: x_ChargePeriodMonth + + annotation SummarizationSetBy = Automatic + + column x_ReportingDate + dataType: dateTime + formatString: General Date + lineageTag: 1c5359f9-6c85-4157-9274-8a4bb4f040b9 + summarizeBy: none + sourceColumn: x_ReportingDate + + annotation SummarizationSetBy = Automatic + partition ChargeBreakdown = m mode: import queryGroup: 'Data Explorer' @@ -307,18 +325,20 @@ table ChargeBreakdown lookback = Text.From(if #"Number of Months" = "" or #"Number of Months" = null then 999 else #"Number of Months"), Source = AzureDataExplorer.Contents(#"Cluster URI", "Hub", " + let numberOfMonths = " & lookback & "; + let monthlyGranularity = " & Text.From(#"Daily or Monthly" = "Monthly") & "; + let allowCostManagementAllocationRules = false; + let summarizeAfterRowCount = 400000; Costs_v1_0 - | where ChargePeriodStart >= monthsago(" & lookback & ") - // - // Filter out cost allocation changes - | where isempty(x_CostAllocationRuleName) - // - // Fix missing costs - | extend ContractedCost = iff(isempty(ContractedCost) or ContractedCost == 0, EffectiveCost, ContractedCost) - | extend ListCost = iff(isempty(ListCost) or ListCost == 0, ContractedCost, ListCost) + | where ChargePeriodStart >= monthsago(numberOfMonths) + | where allowCostManagementAllocationRules or isempty(x_CostAllocationRuleName) + | extend x_ChargePeriodMonth = startofmonth(ChargePeriodStart) + | extend x_ReportingDate = iff(monthlyGranularity, x_ChargePeriodMonth, startofday(ChargePeriodStart)) // | summarize BilledCost = sum(BilledCost), + ChargePeriodEnd = max(ChargePeriodEnd), + ChargePeriodStart = min(ChargePeriodStart), ContractedCost = sum(ContractedCost), EffectiveCost = sum(EffectiveCost), ListCost = sum(ListCost) @@ -328,8 +348,6 @@ table ChargeBreakdown BillingCurrency, ChargeCategory, ChargeClass, - ChargePeriodEnd, - ChargePeriodStart, CommitmentDiscountCategory, CommitmentDiscountStatus, CommitmentDiscountType, @@ -340,9 +358,11 @@ table ChargeBreakdown ServiceCategory, ServiceName, SubAccountName, + x_ChargePeriodMonth, x_Operation, x_PricingSubcategory, x_PublisherCategory, + x_ReportingDate, x_SkuMeterCategory, x_SkuMeterName, x_SkuMeterSubcategory, @@ -373,11 +393,13 @@ table ChargeBreakdown ServiceCategory, ServiceName, SubAccountName, + x_ChargePeriodMonth, x_CommitmentDiscountSavings = ContractedCost - EffectiveCost, x_NegotiatedDiscountSavings = ListCost - ContractedCost, x_Operation, x_PricingSubcategory, x_PublisherCategory, + x_ReportingDate, x_SkuMeterCategory, x_SkuMeterName, x_SkuMeterSubcategory, @@ -387,8 +409,53 @@ table ChargeBreakdown x_SkuTier, x_TotalSavings = ListCost - EffectiveCost, x_UsageType + | serialize + | extend ROW = row_number() + | as allData + | limit summarizeAfterRowCount + | union ( + allData + | where ROW > summarizeAfterRowCount + | summarize + BilledCost = sum(BilledCost), + ChargePeriodEnd = max(ChargePeriodEnd), + ChargePeriodStart = min(ChargePeriodStart), + ContractedCost = sum(ContractedCost), + EffectiveCost = sum(EffectiveCost), + ListCost = sum(ListCost) + by + BillingAccountId, + BillingAccountName, + BillingCurrency, + ChargeCategory, + ChargeClass, + CommitmentDiscountCategory, + CommitmentDiscountStatus, + CommitmentDiscountType, + PricingCategory, + ProviderName, + PublisherName, + ResourceType = '(multiple)', + ServiceCategory, + ServiceName, + SubAccountName = '(multiple)', + x_ChargePeriodMonth, + x_Operation = '(multiple)', + x_PricingSubcategory, + x_PublisherCategory, + x_ReportingDate, + x_SkuMeterCategory = '(multiple)', + x_SkuMeterName = '(multiple)', + x_SkuMeterSubcategory = '(multiple)', + x_SkuPartNumber = '(multiple)', + x_SkuServiceFamily = '(multiple)', + x_SkuTerm, + x_SkuTier = '(multiple)', + x_UsageType = '(multiple)' + ) + | project-away ROW - ", [MaxRows=null, MaxSize=null, NoTruncate=false, AdditionalSetStatements=null, ClientRequestIdPrefix="ftk-Resources-PurchaseCosts"]) + ", [MaxRows=null, MaxSize=1073741824, NoTruncate=false, AdditionalSetStatements=null, ClientRequestIdPrefix="ftk-CostSummary-ChargeBreakdown"]) in Source diff --git a/src/power-bi/kql/CostSummary.SemanticModel/definition/tables/PurchaseCosts.tmdl b/src/power-bi/kql/CostSummary.SemanticModel/definition/tables/PurchaseCosts.tmdl index 34623fc85..2d405407f 100644 --- a/src/power-bi/kql/CostSummary.SemanticModel/definition/tables/PurchaseCosts.tmdl +++ b/src/power-bi/kql/CostSummary.SemanticModel/definition/tables/PurchaseCosts.tmdl @@ -340,6 +340,24 @@ table PurchaseCosts annotation SummarizationSetBy = Automatic + column x_ChargePeriodMonth + dataType: dateTime + formatString: General Date + lineageTag: 2b3da636-6d41-44ad-aa7f-8291ed1fbe08 + summarizeBy: none + sourceColumn: x_ChargePeriodMonth + + annotation SummarizationSetBy = Automatic + + column x_ReportingDate + dataType: dateTime + formatString: General Date + lineageTag: 6e3e6796-d89c-44a7-a4f3-ecff9545c819 + summarizeBy: none + sourceColumn: x_ReportingDate + + annotation SummarizationSetBy = Automatic + partition PurchaseCosts = m mode: import queryGroup: 'Data Explorer' @@ -348,17 +366,78 @@ table PurchaseCosts lookback = Text.From(if #"Number of Months" = "" or #"Number of Months" = null then 999 else #"Number of Months"), Source = AzureDataExplorer.Contents(#"Cluster URI", "Hub", " + let numberOfMonths = " & lookback & "; + let monthlyGranularity = " & Text.From(#"Daily or Monthly" = "Monthly") & "; + let allowCostManagementAllocationRules = false; + let summarizeAfterRowCount = 400000; Costs_v1_0 - | where ChargePeriodStart >= monthsago(" & lookback & ") - // - // Filter out cost allocation changes - | where isempty(x_CostAllocationRuleName) - // - // Fix missing costs - | extend ContractedCost = iff(isempty(ContractedCost) or ContractedCost == 0, EffectiveCost, ContractedCost) - | extend ListCost = iff(isempty(ListCost) or ListCost == 0, ContractedCost, ListCost) + | where ChargePeriodStart >= monthsago(numberOfMonths) + | where allowCostManagementAllocationRules or isempty(x_CostAllocationRuleName) + | extend x_ChargePeriodMonth = startofmonth(ChargePeriodStart) + | extend x_ReportingDate = iff(monthlyGranularity, x_ChargePeriodMonth, startofday(ChargePeriodStart)) // | where ChargeCategory == 'Purchase' + // + | summarize + BilledCost = sum(BilledCost), + ChargePeriodEnd = max(ChargePeriodEnd), + ChargePeriodStart = min(ChargePeriodStart), + ContractedCost = sum(ContractedCost), + EffectiveCost = sum(EffectiveCost), + ListCost = sum(ListCost), + PricingQuantity = sum(PricingQuantity), + x_ServicePeriodEnd = max(x_ServicePeriodEnd), + x_ServicePeriodStart = min(x_ServicePeriodStart) + by + BillingAccountId, + BillingAccountName, + BillingCurrency, + BillingPeriodEnd, + BillingPeriodStart, + ChargeClass, + // ChargeDescription, + ChargeFrequency, + CommitmentDiscountCategory, + // CommitmentDiscountId, + // CommitmentDiscountName, + CommitmentDiscountType, + ContractedUnitPrice, + ListUnitPrice, + PricingCategory, + PricingUnit, + ProviderName, + PublisherName, + // ServiceCategory, + // ServiceName, + SkuId, + SkuPriceId, + SubAccountId, + SubAccountName, + // x_BillingAccountAgreement, + // x_BillingAccountId, + // x_BillingAccountName, + // x_BillingProfileId, + // x_BillingProfileName, + x_ChargePeriodMonth, + x_EffectiveUnitPrice, + x_Operation, + x_PricingBlockSize, + x_PricingUnitDescription, + x_PublisherCategory, + x_PublisherId, + x_ReportingDate, + x_SkuDescription, + // x_SkuDetails, + // x_SkuIsCreditEligible, + // x_SkuMeterCategory, + // x_SkuMeterId, + // x_SkuMeterName, + // x_SkuMeterSubcategory, + x_SkuOrderId, + x_SkuOrderName, + // x_SkuServiceFamily, + x_SkuTerm, + x_UsageType | project BilledCost, BillingAccountId, @@ -396,12 +475,14 @@ table PurchaseCosts // x_BillingAccountName, // x_BillingProfileId, // x_BillingProfileName, + x_ChargePeriodMonth, x_EffectiveUnitPrice, x_Operation, x_PricingBlockSize, x_PricingUnitDescription, x_PublisherCategory, x_PublisherId, + x_ReportingDate, x_ServicePeriodEnd, x_ServicePeriodStart, x_SkuDescription, @@ -416,8 +497,67 @@ table PurchaseCosts // x_SkuServiceFamily, x_SkuTerm, x_UsageType + | serialize + | extend ROW = row_number() + | as allData + | limit summarizeAfterRowCount + | union ( + allData + | where ROW > summarizeAfterRowCount + | summarize + BilledCost = sum(BilledCost), + ChargePeriodEnd = max(ChargePeriodEnd), + ChargePeriodStart = min(ChargePeriodStart), + ContractedCost = sum(ContractedCost), + EffectiveCost = sum(EffectiveCost), + ListCost = sum(ListCost), + PricingQuantity = sum(PricingQuantity), + x_ServicePeriodEnd = max(x_ServicePeriodEnd), + x_ServicePeriodStart = min(x_ServicePeriodStart) + by + BillingAccountId, + BillingAccountName, + BillingCurrency, + BillingPeriodEnd, + BillingPeriodStart, + ChargeClass, + ChargeFrequency, + CommitmentDiscountCategory, + CommitmentDiscountType, + ContractedUnitPrice = todecimal(''), + ListUnitPrice = todecimal(''), + PricingCategory, + PricingUnit, + ProviderName, + PublisherName, + SkuId = '(multiple)', + SkuPriceId = '(multiple)', + SubAccountId = '(multiple)', + SubAccountName = '(multiple)', + x_ChargePeriodMonth, + x_EffectiveUnitPrice = todecimal(''), + x_Operation = '(multiple)', + x_PricingBlockSize = todecimal(''), + x_PricingUnitDescription = '(multiple)', + x_PublisherCategory, + x_PublisherId, + x_ReportingDate, + x_SkuDescription = '(multiple)', + // x_SkuDetails, + // x_SkuIsCreditEligible, + // x_SkuMeterCategory, + // x_SkuMeterId, + // x_SkuMeterName, + // x_SkuMeterSubcategory, + x_SkuOrderId = '(multiple)', + x_SkuOrderName = '(multiple)', + // x_SkuServiceFamily, + x_SkuTerm = toint(''), + x_UsageType = '(multiple)' + ) + | project-away ROW - ", [MaxRows=null, MaxSize=null, NoTruncate=false, AdditionalSetStatements=null, ClientRequestIdPrefix="ftk-Resources-PurchaseCosts"]) + ", [MaxRows=null, MaxSize=1073741824, NoTruncate=false, AdditionalSetStatements=null, ClientRequestIdPrefix="ftk-CostSummary-PurchaseCosts"]) in Source diff --git a/src/power-bi/kql/CostSummary.SemanticModel/definition/tables/ResourceCosts.tmdl b/src/power-bi/kql/CostSummary.SemanticModel/definition/tables/ResourceCosts.tmdl index e78907821..ece4b886d 100644 --- a/src/power-bi/kql/CostSummary.SemanticModel/definition/tables/ResourceCosts.tmdl +++ b/src/power-bi/kql/CostSummary.SemanticModel/definition/tables/ResourceCosts.tmdl @@ -307,21 +307,22 @@ table ResourceCosts lookback = Text.From(if #"Number of Months" = "" or #"Number of Months" = null then 999 else #"Number of Months"), Source = AzureDataExplorer.Contents(#"Cluster URI", "Hub", " + let numberOfMonths = " & lookback & "; + let monthlyGranularity = " & Text.From(#"Daily or Monthly" = "Monthly") & "; + let allowCostManagementAllocationRules = false; + let summarizeAfterRowCount = 400000; Costs_v1_0 - | where ChargePeriodStart >= monthsago(" & lookback & ") - // - // Filter out cost allocation changes - | where isempty(x_CostAllocationRuleName) - // - // Fix missing costs - | extend ContractedCost = iff(isempty(ContractedCost) or ContractedCost == 0, EffectiveCost, ContractedCost) - | extend ListCost = iff(isempty(ListCost) or ListCost == 0, ContractedCost, ListCost) + | where ChargePeriodStart >= monthsago(numberOfMonths) + | where allowCostManagementAllocationRules or isempty(x_CostAllocationRuleName) + | extend x_ChargePeriodMonth = startofmonth(ChargePeriodStart) + | extend x_ReportingDate = iff(monthlyGranularity, x_ChargePeriodMonth, startofday(ChargePeriodStart)) // | summarize AvailabilityZone = take_any(AvailabilityZone), BilledCost = sum(BilledCost), BillingAccountName = take_any(BillingAccountName), - " & (if #"Daily or Monthly" = "Daily" then "" else "ChargePeriodEnd = max(ChargePeriodEnd), ChargePeriodStart = min(ChargePeriodStart),") & " + ChargePeriodEnd = max(ChargePeriodEnd), + ChargePeriodStart = min(ChargePeriodStart), ContractedCost = sum(ContractedCost), EffectiveCost = sum(EffectiveCost), ListCost = sum(ListCost), @@ -338,18 +339,15 @@ table ResourceCosts by BillingAccountId, BillingCurrency, - " & (if #"Daily or Monthly" = "Daily" then "ChargePeriodEnd, ChargePeriodStart," else "") & " - x_ChargePeriodMonth = startofmonth(ChargePeriodStart), CommitmentDiscountCategory, CommitmentDiscountId, CommitmentDiscountName, CommitmentDiscountType, CommitmentDiscountStatus, ResourceId, - SubAccountId - // Tags, - // x_CostCategories, - // x_CostCenter + SubAccountId, + x_ChargePeriodMonth, + x_ReportingDate | project AvailabilityZone, BilledCost, @@ -382,12 +380,52 @@ table ResourceCosts // x_CostCategories, // x_CostCenter x_NegotiatedDiscountSavings = ListCost - ContractedCost, - x_ReportingDate = " & (if #"Daily or Monthly" = "Daily" then "startofday(ChargePeriodStart)" else "x_ChargePeriodMonth") & ", + x_ReportingDate, x_ResourceGroupName, x_ResourceType, x_TotalSavings = ListCost - EffectiveCost + | serialize + | extend ROW = row_number() + | as allData + | limit summarizeAfterRowCount + | union ( + allData + | where ROW > summarizeAfterRowCount + | summarize + BilledCost = sum(BilledCost), + ChargePeriodEnd = max(ChargePeriodEnd), + ChargePeriodStart = min(ChargePeriodStart), + ContractedCost = sum(ContractedCost), + EffectiveCost = sum(EffectiveCost), + ListCost = sum(ListCost) + by + AvailabilityZone = '(multiple)', + BillingAccountId, + BillingAccountName, + BillingCurrency, + CommitmentDiscountCategory, + CommitmentDiscountId = '(multiple)', + CommitmentDiscountName = '(multiple)', + CommitmentDiscountType, + CommitmentDiscountStatus, + ProviderName, + RegionId = '(multiple)', + RegionName = '(multiple)', + ResourceId = '(multiple)', + ResourceName = '(multiple)', + ResourceType = '(multiple)', + ServiceCategory = '(multiple)', + ServiceName = '(multiple)', + SubAccountName = '(multiple)', + SubAccountId = '(multiple)', + x_ChargePeriodMonth, + x_ReportingDate, + x_ResourceGroupName = '(multiple)', + x_ResourceType = '(multiple)' + ) + | project-away ROW - ", [MaxRows=null, MaxSize=null, NoTruncate=false, AdditionalSetStatements=null, ClientRequestIdPrefix="ftk-Resources-ResourceCosts"]) + ", [MaxRows=null, MaxSize=1073741824, NoTruncate=false, AdditionalSetStatements=null, ClientRequestIdPrefix="ftk-CostSummary-ResourceCosts"]) in Source diff --git a/src/power-bi/kql/CostSummary.SemanticModel/definition/tables/SkuPrices.tmdl b/src/power-bi/kql/CostSummary.SemanticModel/definition/tables/SkuPrices.tmdl index 4c281b4a7..4c39c7019 100644 --- a/src/power-bi/kql/CostSummary.SemanticModel/definition/tables/SkuPrices.tmdl +++ b/src/power-bi/kql/CostSummary.SemanticModel/definition/tables/SkuPrices.tmdl @@ -288,7 +288,7 @@ table SkuPrices Source = AzureDataExplorer.Contents(#"Cluster URI", "Hub", " let numberOfMonths = " & lookback & "; - let monthlyGranularity = " & Text.From(#"Daily or Monthly" = "Daily") & "; + let monthlyGranularity = " & Text.From(#"Daily or Monthly" = "Monthly") & "; let allowCostManagementAllocationRules = false; let usageOnly = false; let committedOnly = false; @@ -360,7 +360,7 @@ table SkuPrices x_SkuTermLabel, x_TotalSavings = ListCost - EffectiveCost - ", [MaxRows=null, MaxSize=null, NoTruncate=false, AdditionalSetStatements=null, ClientRequestIdPrefix="ftk-Resources-SkuPrices"]) + ", [MaxRows=null, MaxSize=1073741824, NoTruncate=false, AdditionalSetStatements=null, ClientRequestIdPrefix="ftk-CostSummary-SkuPrices"]) in Source diff --git a/src/power-bi/kql/CostSummary.SemanticModel/definition/tables/TagCosts.tmdl b/src/power-bi/kql/CostSummary.SemanticModel/definition/tables/TagCosts.tmdl index 555013f6f..ff51890b7 100644 --- a/src/power-bi/kql/CostSummary.SemanticModel/definition/tables/TagCosts.tmdl +++ b/src/power-bi/kql/CostSummary.SemanticModel/definition/tables/TagCosts.tmdl @@ -68,14 +68,6 @@ table TagCosts annotation PBI_FormatHint = {"isGeneralNumber":true} - column Tags - dataType: string - lineageTag: 1d040e0e-7534-4760-a493-8be211cb8344 - summarizeBy: none - sourceColumn: Tags - - annotation SummarizationSetBy = Automatic - column x_CommitmentDiscountSavings dataType: double lineageTag: 81651eba-c48f-44b1-9d81-a2c4d2b7b910 @@ -187,6 +179,64 @@ table TagCosts annotation SummarizationSetBy = Automatic + column CommitmentDiscountCategory + dataType: string + lineageTag: ee9e68f8-6449-4d7b-addb-fe74fdd7cc2b + summarizeBy: none + sourceColumn: CommitmentDiscountCategory + + annotation SummarizationSetBy = Automatic + + column CommitmentDiscountId + dataType: string + lineageTag: 05f9fe5a-b12d-4692-b6ce-87f0a7608369 + summarizeBy: none + sourceColumn: CommitmentDiscountId + + annotation SummarizationSetBy = Automatic + + column CommitmentDiscountName + dataType: string + lineageTag: dc924f77-d8ec-4f55-834a-440b08a9f25c + summarizeBy: none + sourceColumn: CommitmentDiscountName + + annotation SummarizationSetBy = Automatic + + column CommitmentDiscountType + dataType: string + lineageTag: 2183563b-f74d-4fe0-aa42-ae6553eecb5a + summarizeBy: none + sourceColumn: CommitmentDiscountType + + annotation SummarizationSetBy = Automatic + + column x_ChargePeriodMonth + dataType: dateTime + formatString: General Date + lineageTag: 5d26fbf1-9c38-4e6e-a4be-7006dcb6d83d + summarizeBy: none + sourceColumn: x_ChargePeriodMonth + + annotation SummarizationSetBy = Automatic + + column tag_Untagged + dataType: boolean + formatString: """TRUE"";""TRUE"";""FALSE""" + lineageTag: c35e42aa-281b-4852-9c77-e37a23ab671c + summarizeBy: none + sourceColumn: tag_Untagged + + annotation SummarizationSetBy = Automatic + + column tag_Product + dataType: string + lineageTag: 693b17fa-35c6-4aa5-b63f-0c75f5fc184c + summarizeBy: none + sourceColumn: tag_Product + + annotation SummarizationSetBy = Automatic + partition TagCosts = m mode: import queryGroup: 'Data Explorer' @@ -195,6 +245,11 @@ table TagCosts lookback = Text.From(if #"Number of Months" = "" or #"Number of Months" = null then 999 else #"Number of Months"), Source = AzureDataExplorer.Contents(#"Cluster URI", "Hub", " + let numberOfMonths = " & lookback & "; + let monthlyGranularity = " & Text.From(#"Daily or Monthly" = "Monthly") & "; + let allowCostManagementAllocationRules = false; + let summarizeAfterRowCount = 400000; + // // Set the tags you want to promote as 'tag_*' columns in this array let promotedTags = dynamic(['Application', 'BusinessUnit', 'CostCenter', 'Department', 'Division', 'Env', 'Owner', 'Product', 'Project', 'Purpose', 'Service']); // @@ -209,87 +264,68 @@ table TagCosts iff(toscalar(tagColumn | count) > 0, toscalar(tagColumn | limit 1), '') }; Costs_v1_0 - | where ChargePeriodStart >= monthsago(" & lookback & ") - // | where isnotempty(Tags) - // - // Filter out cost allocation changes - | where isempty(x_CostAllocationRuleName) - // - // Fix missing costs - | extend ContractedCost = iff(isempty(ContractedCost) or ContractedCost == 0, EffectiveCost, ContractedCost) - | extend ListCost = iff(isempty(ListCost) or ListCost == 0, ContractedCost, ListCost) + | where ChargePeriodStart >= monthsago(numberOfMonths) + | where allowCostManagementAllocationRules or isempty(x_CostAllocationRuleName) + | extend x_ChargePeriodMonth = startofmonth(ChargePeriodStart) + | extend x_ReportingDate = iff(monthlyGranularity, x_ChargePeriodMonth, startofday(ChargePeriodStart)) // + | extend Tags = tostring(Tags) | summarize BilledCost = sum(BilledCost), - // BillingAccountName = take_any(BillingAccountName), - " & (if #"Daily or Monthly" = "Daily" then "" else "ChargePeriodEnd = max(ChargePeriodEnd), ChargePeriodStart = min(ChargePeriodStart),") & " + ChargePeriodEnd = max(ChargePeriodEnd), + ChargePeriodStart = min(ChargePeriodStart), ContractedCost = sum(ContractedCost), EffectiveCost = sum(EffectiveCost), ListCost = sum(ListCost) - // ProviderName = take_any(ProviderName), - // SubAccountName = take_any(SubAccountName), by - // BillingAccountId, BillingCurrency, - " & (if #"Daily or Monthly" = "Daily" then "ChargePeriodEnd, ChargePeriodStart," else "") & " CommitmentDiscountCategory, CommitmentDiscountId, CommitmentDiscountName, CommitmentDiscountType, - CommitmentDiscountStatus, - ResourceId, - // SubAccountId - Tags = tostring(Tags), - // x_BillingAccountId, - // x_BillingAccountName, - // x_BillingProfileId, - // x_BillingProfileName, - x_ChargePeriodMonth = startofmonth(ChargePeriodStart), + Tags, + x_ChargePeriodMonth, x_CostCategories = tostring(x_CostCategories), - x_CostCenter - // x_InvoiceSectionId, - // x_InvoiceSectionName, - // x_Project - | project + x_CostCenter, + x_ReportingDate, + tag_Untagged = isempty(Tags) or Tags == '{}' + // + // Extract tags + | extend Tags = parse_json(Tags) + | mv-apply Tags on ( + extend TagName = tostring(bag_keys(Tags)[0]) + | extend cleanTagName = iff(trimSpaces, trim(' ', TagName), TagName) + | where cleanTagName in (promotedTags) or (caseInsensitive and cleanTagName in~ (promotedTags)) + | extend tagColumn = strcat('tag_', promotedTags[array_index_of(parse_json(tolower(promotedTags)), tolower(cleanTagName))]) + | where isnotempty(tagColumn) + | summarize Tags = make_bag(bag_pack(tagColumn, tostring(Tags[TagName]))) + ) + | extend x_CommitmentDiscountSavings = ContractedCost - EffectiveCost + | extend x_NegotiatedDiscountSavings = ListCost - ContractedCost + | extend x_TotalSavings = ListCost - EffectiveCost + | evaluate bag_unpack(Tags) + | project-reorder BilledCost, - // BillingAccountId, - // BillingAccountName, BillingCurrency, ChargePeriodEnd, ChargePeriodStart, + CommitmentDiscountCategory, + CommitmentDiscountId, + CommitmentDiscountName, + CommitmentDiscountType, ContractedCost, EffectiveCost, ListCost, - // SubAccountId, - // SubAccountName, - Tags, - // x_BillingAccountId, - // x_BillingAccountName, - // x_BillingProfileId, - // x_BillingProfileName, - x_CommitmentDiscountSavings = ContractedCost - EffectiveCost, + x_ChargePeriodMonth, + x_CommitmentDiscountSavings, x_CostCategories, x_CostCenter, - // x_InvoiceSectionId, - // x_InvoiceSectionName, - // x_Project - x_NegotiatedDiscountSavings = ListCost - ContractedCost, - x_ReportingDate = " & (if #"Daily or Monthly" = "Daily" then "startofday(ChargePeriodStart)" else "x_ChargePeriodMonth") & ", - x_TotalSavings = ListCost - EffectiveCost - // - // Extract tags - | extend tmp_Tags = parse_json(Tags) - | mv-apply tmp_Tags on ( - extend TagName = tostring(bag_keys(tmp_Tags)[0]) - | extend cleanTagName = iff(trimSpaces, trim(' ', TagName), TagName) - | where cleanTagName in (promotedTags) or (caseInsensitive and cleanTagName in~ (promotedTags)) - | extend tagColumn = strcat('tag_', promotedTags[array_index_of(parse_json(tolower(promotedTags)), tolower(cleanTagName))]) - | where isnotempty(tagColumn) - | summarize tmp_Tags = make_bag(bag_pack(tagColumn, tostring(tmp_Tags[TagName]))) - ) - | evaluate bag_unpack(tmp_Tags) + x_NegotiatedDiscountSavings, + x_ReportingDate, + x_TotalSavings, + tag_Untagged - ", [MaxRows=null, MaxSize=null, NoTruncate=false, AdditionalSetStatements=null, ClientRequestIdPrefix="ftk-Resources-TagCosts"]) + ", [MaxRows=null, MaxSize=1073741824, NoTruncate=false, AdditionalSetStatements=null, ClientRequestIdPrefix="ftk-CostSummary-TagCosts"]) in Source diff --git a/src/powershell/Tests/Integration/Toolkit.Tests.ps1 b/src/powershell/Tests/Integration/Toolkit.Tests.ps1 index 1810009bd..883b09b7e 100644 --- a/src/powershell/Tests/Integration/Toolkit.Tests.ps1 +++ b/src/powershell/Tests/Integration/Toolkit.Tests.ps1 @@ -36,19 +36,23 @@ Describe 'Get-FinOpsToolkitVersion' { # Templates CheckFile "finops-hub-v$verStr.zip" $null $null - CheckFile "governance-workbook-v$verStr.zip" '0.1' $null + CheckFile "finops-workbook-v$verStr.zip" '0.6' $null + CheckFile "governance-workbook-v$verStr.zip" '0.1' '0.5' CheckFile "optimization-engine-v$verStr.zip" '0.4' $null - CheckFile "optimization-workbook-v$verStr.zip" $null $null + CheckFile "optimization-workbook-v$verStr.zip" $null '0.5' # Power BI + CheckFile "PowerBI-demo.zip" '0.7' $null + CheckFile "PowerBI-kql.zip" '0.7' $null + CheckFile "PowerBI-storage.zip" '0.7' $null CheckFile "CostManagementConnector.pbix" '0.2' $null CheckFile "CostManagementTemplateApp.pbix" '0.2' $null - CheckFile "CostSummary.pbit" '0.2' $null - CheckFile "CostSummary.pbix" $null $null - CheckFile "DataIngestion.pbit" '0.3' $null - CheckFile "DataIngestion.pbix" '0.3' $null - CheckFile "RateOptimization.pbit" '0.4' $null - CheckFile "RateOptimization.pbix" '0.4' $null + CheckFile "CostSummary.pbit" '0.2' '0.6' + CheckFile "CostSummary.pbix" $null '0.6' + CheckFile "DataIngestion.pbit" '0.3' '0.6' + CheckFile "DataIngestion.pbix" '0.3' '0.6' + CheckFile "RateOptimization.pbit" '0.4' '0.6' + CheckFile "RateOptimization.pbix" '0.4' '0.6' # Open data CheckFile "dataset-examples.zip" '0.4' $null diff --git a/src/scripts/Package-Toolkit.ps1 b/src/scripts/Package-Toolkit.ps1 index b50984368..afc8e223b 100644 --- a/src/scripts/Package-Toolkit.ps1 +++ b/src/scripts/Package-Toolkit.ps1 @@ -14,31 +14,39 @@ .PARAMETER Build Optional. Indicates whether the Build-Toolkit command should be executed first. Default = false. - .PARAMETER PowerBI - Optional. Indicates whether to open Power BI files as part of the packaging process. Default = false. + .PARAMETER CopyFiles + Optional. Indicates whether to copy templates and open data files. Default = false. + + .PARAMETER OpenPBI + Optional. Indicates that Power BI projects should be opened as part of the packaging process. Default = false. + + .PARAMETER ZipPBI + Optional. Indicates that prepared PBIX files should be packaged into release files. Default = false. .PARAMETER Preview Optional. Indicates that the template(s) should be saved as a preview only. Does not package other files. Default = false. .EXAMPLE - ./Package-Toolkit + ./Package-Toolkit -CopyFiles Generates ZIP files for each template using an existing build. .EXAMPLE - ./Package-Toolkit -Build + ./Package-Toolkit -CopyFiles -Build Builds the latest code and generates ZIP files for each template. .EXAMPLE - ./Package-Toolkit -Build -PowerBI + ./Package-Toolkit -CopyFiles -Build -OpenPBI Builds the latest code, generates ZIP files for each template, and opens Power BI projects to be saved as PBIX files. #> Param( [Parameter(Position = 0)][string]$Template = "*", [switch]$Build, - [switch]$PowerBI, + [switch]$CopyFiles, + [switch]$OpenPBI, + [switch]$ZipPBI, [switch]$Preview ) @@ -62,98 +70,95 @@ if ($Template -ne "*" -and -not (Test-Path $relDir)) return } -# Package templates -$version = & "$PSScriptRoot/Get-Version" -Write-Host "Packaging $(if ($Template) { "$Template v$version template" } else { "v$version templates" })..." - -$isPrerelease = $version -like '*-*' +function Copy-TemplateFiles() +{ + Write-Host "Packaging $(if ($Template) { "$Template v$version template" } else { "v$version templates" })..." -Write-Verbose "Removing existing ZIP files..." -Remove-Item "$relDir/*.zip" -Force + Write-Verbose "Removing existing ZIP files..." + Remove-Item "$relDir/*.zip" -Force -$templates = Get-ChildItem "$relDir/$Template*" -Directory ` -| ForEach-Object { - Write-Verbose ("Packaging $_" -replace (Get-Item $relDir).FullName, '.') - $srcPath = $_ - $templateName = $srcPath.Name - $versionSubFolder = (Join-Path $srcPath $version) - $zip = Join-Path (Get-Item $relDir) "$templateName-v$version.zip" + return Get-ChildItem "$relDir/$Template*" -Directory ` + | Where-Object { $_.Name -ne 'pbix' } ` + | ForEach-Object { + Write-Verbose ("Packaging $_" -replace (Get-Item $relDir).FullName, '.') + $srcPath = $_ + $templateName = $srcPath.Name + $versionSubFolder = (Join-Path $srcPath $version) + $zip = Join-Path (Get-Item $relDir) "$templateName-v$version.zip" - Write-Verbose "Checking for a nested version folder: $versionSubFolder" - if ((Test-Path -Path $versionSubFolder -PathType Container) -eq $true) - { - Write-Verbose " Switching to sub folder" - $srcPath = $versionSubFolder - } + Write-Verbose "Checking for a nested version folder: $versionSubFolder" + if ((Test-Path -Path $versionSubFolder -PathType Container) -eq $true) + { + Write-Verbose " Switching to sub folder" + $srcPath = $versionSubFolder + } - # Skip if template is a Bicep Registry module - Write-Verbose "Checking version.json to see if it's targeting the Bicep Registry" - if (Test-Path $srcPath/version.json) - { - $versionSchema = (Get-Content "$srcPath\version.json" -Raw | ConvertFrom-Json | Select-Object -ExpandProperty '$schema') - if ($versionSchema -like '*bicep-registry-module*') + # Skip if template is a Bicep Registry module + Write-Verbose "Checking version.json to see if it's targeting the Bicep Registry" + if (Test-Path $srcPath/version.json) { - Write-Verbose "Skipping Bicep Registry module (not included in releases)" - return + $versionSchema = (Get-Content "$srcPath\version.json" -Raw | ConvertFrom-Json | Select-Object -ExpandProperty '$schema') + if ($versionSchema -like '*bicep-registry-module*') + { + Write-Verbose "Skipping Bicep Registry module (not included in releases)" + return + } } - } - Write-Verbose "Updating $templateName deployment files in docs..." + Write-Verbose "Updating $templateName deployment files in docs..." - function Copy-DeploymentFiles($suffix) - { - $packageManifestPath = "$srcPath/package-manifest.json" - if (Test-Path $packageManifestPath) + function Copy-DeploymentFiles($suffix) { - # Read files/directories from package-manifest.json - $packageManifest = Get-Content $packageManifestPath -Raw | ConvertFrom-Json - - # Create release directory - $targetDir = "$deployDir/$templateName/$suffix" - & "$PSScriptRoot/New-Directory" $targetDir - - # Copy files and directories - $packageManifest.deployment.Files | ForEach-Object { Copy-Item "$srcPath/$($_.source)" "$targetDir/$($_.destination)" -Force } - $packageManifest.deployment.Directories | ForEach-Object { - & "$PSScriptRoot/New-Directory" "$targetDir/$($_.destination)" - Get-ChildItem "$srcPath/$($_.source)" | Copy-Item -Destination "$targetDir/$($_.destination)" -Recurse -Force + $packageManifestPath = "$srcPath/package-manifest.json" + if (Test-Path $packageManifestPath) + { + # Read files/directories from package-manifest.json + $packageManifest = Get-Content $packageManifestPath -Raw | ConvertFrom-Json + + # Create release directory + $targetDir = "$deployDir/$templateName/$suffix" + & "$PSScriptRoot/New-Directory" $targetDir + + # Copy files and directories + $packageManifest.deployment.Files | ForEach-Object { Copy-Item "$srcPath/$($_.source)" "$targetDir/$($_.destination)" -Force } + $packageManifest.deployment.Directories | ForEach-Object { + & "$PSScriptRoot/New-Directory" "$targetDir/$($_.destination)" + Get-ChildItem "$srcPath/$($_.source)" | Copy-Item -Destination "$targetDir/$($_.destination)" -Recurse -Force + } + } + else + { + # Copy azuredeploy.json to docs/deploy folder + Copy-Item "$srcPath/azuredeploy.json" "$deployDir/$templateName-$suffix.json" + Copy-Item "$srcPath/createUiDefinition.json" "$deployDir/$templateName-$suffix.ui.json" } } + + if ($Preview) + { + Copy-DeploymentFiles "preview" + } else { - # Copy azuredeploy.json to docs/deploy folder - Copy-Item "$srcPath/azuredeploy.json" "$deployDir/$templateName-$suffix.json" - Copy-Item "$srcPath/createUiDefinition.json" "$deployDir/$templateName-$suffix.ui.json" + Copy-DeploymentFiles $version + Copy-DeploymentFiles "latest" } - } - if ($Preview) - { - Copy-DeploymentFiles "preview" + Write-Verbose ("Compressing $srcPath to $zip" -replace (Get-Item $relDir).FullName, '.') + Compress-Archive -Path "$srcPath/*" -DestinationPath $zip + return $zip } - else - { - Copy-DeploymentFiles $version - Copy-DeploymentFiles "latest" - } - - Write-Verbose ("Compressing $srcPath to $zip" -replace (Get-Item $relDir).FullName, '.') - Compress-Archive -Path "$srcPath/*" -DestinationPath $zip - return $zip } -Write-Host "✅ $($templates.Count) template$(if ($templates.Count -ne 1) { 's' })" -Write-Host "ℹī¸ Deployment files updated... Please commit the changes manually..." -# Only package remaining files if not preview -if (-not $Preview) +function Copy-OpenDataFiles() { - # Copy open data files Write-Verbose "Copying open data files..." Copy-Item "$PSScriptRoot/../open-data/*.csv" $relDir Copy-Item "$PSScriptRoot/../open-data/*.json" $relDir - Write-Host "✅ $((@(Get-ChildItem "$relDir/*.csv") + @(Get-ChildItem "$relDir/*.json")).Count) open data files" - - # Package sample data files together +} + +function Copy-OpenDataFolders() +{ Write-Verbose "Packaging open data files..." Get-ChildItem -Path "$PSScriptRoot/../open-data" -Directory ` | ForEach-Object { @@ -161,44 +166,101 @@ if (-not $Preview) Compress-Archive -Path "$dir/*.*" -DestinationPath "$relDir/$($dir.BaseName).zip" Write-Host "✅ $((Get-ChildItem "$dir/*.*").Count) $($dir.BaseName) files" } - - # Copy PBIX files - Write-Verbose "Copying PBIX files..." - Copy-Item "$PSScriptRoot/../power-bi/*.pbix" $relDir -Force - Write-Host "✅ $((Get-ChildItem "$PSScriptRoot/../power-bi/*.pbix").Count) PBIX files" - - # Open Power BI projects - $pbi = Get-ChildItem "$PSScriptRoot/../power-bi/*.pbip" - if ($PowerBI) - { - Write-Host "ℹī¸ $($pbi.Count) Power BI projects must be converted manually... Opening..." - $pbi | Invoke-Item - } - elseif ($isPrerelease) - { - Write-Host "✖ī¸ Skipping $($pbi.Count) Power BI projects for prerelease version" - } - else +} + +$version = & "$PSScriptRoot/Get-Version" + +if ($CopyFiles -or $Build -or $Preview -or -not ($OpenPBI -or $ZipPBI)) +{ + # Package templates + $templates = Copy-TemplateFiles + Write-Host "✅ $($templates.Count) template$(if ($templates.Count -ne 1) { 's' })" + Write-Host "ℹī¸ Deployment files updated... Please commit the changes manually..." + + # Only package remaining files if not preview + if (-not $Preview) { - Write-Host "⚠ī¸ $($pbi.Count) Power BI projects must be converted manually!" - Write-Host ' To open them, run: ' -NoNewline - Write-Host './Package-Toolkit -PowerBI' -ForegroundColor Cyan - } + # Copy open data files + Copy-OpenDataFiles + Write-Host "✅ $((@(Get-ChildItem "$relDir/*.csv") + @(Get-ChildItem "$relDir/*.json")).Count) open data files" - # Update version in docs - $docVersionPath = "$PSScriptRoot/../../docs/_includes/ftkver.txt" - $versionInDocs = Get-Content $docVersionPath -Raw - if ($versionInDocs -eq $version) - { - Write-Host "✅ Version in docs ($versionInDocs) already up-to-date" - } - else - { - Write-Verbose "Updating version in docs..." - $version | Out-File $docVersionPath -NoNewline - Write-Host "ℹī¸ Version updated in docs... Please commit the changes manually..." + # Package sample data files together + Copy-OpenDataFolders + + # Copy PBIX files + Write-Verbose "Copying PBIX files..." + Copy-Item "$PSScriptRoot/../power-bi/cm-connector/*.pbix" "$relDir" -Force + Write-Host "✅ $((Get-ChildItem "$PSScriptRoot/../power-bi/cm-connector/*.pbix").Count) PBIX files" + + # Update version in docs + $docVersionPath = "$PSScriptRoot/../../docs/_includes/ftkver.txt" + $versionInDocs = Get-Content $docVersionPath -Raw + if ($versionInDocs -eq $version) + { + Write-Host "✅ Version in docs ($versionInDocs) already up-to-date" + } + else + { + Write-Verbose "Updating version in docs..." + $version | Out-File $docVersionPath -NoNewline + Write-Host "ℹī¸ Version updated in docs... Please commit the changes manually..." + } } } +$pbip = Get-ChildItem -Path "$PSScriptRoot/../power-bi" -Include "*.pbip" -Recurse +if (-not ($OpenPBI -or $ZipPBI)) +{ + Write-Host "⚠ī¸ $($pbip.Count) Power BI projects must be converted manually!" + Write-Host ' To open them, run: ' -NoNewline + Write-Host './Package-Toolkit -OpenPBI' -ForegroundColor Cyan +} +elseif ($OpenPBI) +{ + # Recreate temp pbix folder + Remove-Item -Path "$relDir/pbix" -Recurse -Force -ErrorAction SilentlyContinue + & "$PSScriptRoot/New-Directory" "$relDir/pbix" + + # Open Power BI projects + Write-Host "ℹī¸ $($pbip.Count) Power BI projects must be converted manually... Opening..." + $pbip | Invoke-Item + Write-Host ' Save as PBIX then run: ' -NoNewline + Write-Host './Package-Toolkit -ZipPBI' -ForegroundColor Cyan +} +elseif ($ZipPBI) +{ + # Clean PBIX files + $pbixFiles = Get-ChildItem -Path "$relDir/pbix/*.pbix" + Write-Verbose "Processing $($pbixFiles.Count) files..." + $pbixFiles | ForEach-Object { + # Expand PBIX files for cleanup + $pbix = $_ + $pbixDir = $pbix.FullName.Replace('.pbix', '') + # Write-Verbose "Expanding $pbixDir..." + # Expand-Archive -Path $pbix -DestinationPath $pbixDir + + # TODO: Remove queries for storage files + # TODO: Remove sensitivity + + # Zip as PBIX for demo + if ($pbixDir.EndsWith('.kql')) + { + Write-Verbose "Saving demo $pbixDir.pbix..." + # Compress-Archive -Path $pbixDir -DestinationPath "$pbixDir.kql.pbix" + } + + # TODO: Remove data + + # Zip as PBIT + Write-Verbose "Saving $pbixDir.pbit..." + # Compress-Archive -Path $pbixDir -DestinationPath "$pbixDir.pbit" + } + + # Zip release files + # Compress-Archive -Path "$relDir/pbix/*.storage.pbit" -DestinationPath "$relDir/PowerBI-storage.zip" + # Compress-Archive -Path "$relDir/pbix/*.kql.pbix" -DestinationPath "$relDir/PowerBI-demo.zip" + # Compress-Archive -Path "$relDir/pbix/*.kql.pbit" -DestinationPath "$relDir/PowerBI-kql.zip" +} + Write-Host '...done!' Write-Host '' diff --git a/src/templates/finops-hub/README.md b/src/templates/finops-hub/README.md index bb875e2a4..0bfc64a9d 100644 --- a/src/templates/finops-hub/README.md +++ b/src/templates/finops-hub/README.md @@ -5,6 +5,7 @@ products: - azure - azure-blob-storage - azure-cost-management +- azure-data-explorer - azure-data-factory - azure-data-lake - azure-key-vault @@ -36,7 +37,8 @@ This template creates a new **FinOps hub** instance. FinOps hubs are a foundatio FinOps hubs include: -- Data Lake storage to host cost data. +- Data Explorer cluster for analytics. +- Data Lake storage for staging. - Data Factory for data processing and orchestration. - Key Vault for storing secrets. @@ -65,25 +67,27 @@ If you run into any issues, see [Troubleshooting FinOps hubs](https://aka.ms/fin 2. Deploy the template > [![Deploy To Azure](https://raw.githubusercontent.com/Azure/azure-quickstart-templates/master/1-CONTRIBUTION-GUIDE/images/deploytoazure.svg?sanitize=true)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2Fazure-quickstart-templates%2Fmaster%2Fquickstarts%2Fmicrosoft.costmanagement%2Ffinops-hub%2Fazuredeploy.json/createUIDefinitionUri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2Fazure-quickstart-templates%2Fmaster%2Fquickstarts%2Fmicrosoft.costmanagement%2Ffinops-hub%2FcreateUiDefinition.json)   [![Deploy To Azure US Gov](https://raw.githubusercontent.com/Azure/azure-quickstart-templates/master/1-CONTRIBUTION-GUIDE/images/deploytoazuregov.svg?sanitize=true)](https://portal.azure.us/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2Fazure-quickstart-templates%2Fmaster%2Fquickstarts%2Fmicrosoft.costmanagement%2Ffinops-hub%2Fazuredeploy.json/createUIDefinitionUri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure%2Fazure-quickstart-templates%2Fmaster%2Fquickstarts%2Fmicrosoft.costmanagement%2Ffinops-hub%2FcreateUiDefinition.json) 3. [Create a new cost export](https://learn.microsoft.com/azure/cost-management-billing/costs/tutorial-export-acm-data?tabs=azure-portal) using the following settings: - - **Metric** = `Amortized cost` - - **Export type** = `Daily export of month-to-date costs` + - **Type of data** = `Cost and usage details (FOCUS)` + - **Dataset version** = `1.0` or `1.0r2` + - **Frequency** = `Daily export of month-to-date costs` + - **Storage account** = (Use subscription/resource deployed with your hub) + - **Container** = `msexports` + - **Format** = `Parquet` + - **Compression** = `Snappy` + - **Directory** = (Specify a unique path for this scope) + - _**EA billing account:** `billingAccounts/{enrollment-number}`_ + - _**MCA billing profile:** `billingProfiles/{billing-profile-id}`_ + - _**Subscription:** `subscriptions/{subscription-id}`_ + - _**Resource group:** `subscriptions/{subscription-id}/resourceGroups/{rg-name}`_ - **File partitioning** = On - **Overwrite data** = Off -
- _While most settings are required, overwriting is optional. We recommend **not** overwriting files so you can monitor your ingestion pipeline using the [Data ingestion](https://aka.ms/ftk/DataIngestion) report. If you do not plan to use that report, please enable overwriting._ -
- - **Storage account** = (Use subscription/resource from step 1) - - **Container** = `msexports` - - **Directory** = (Use the resource ID of the scope1 you're exporting without the first "/") 4. Run your export using the **Run now** command > Your data should be available within 15 minutes or so, depending on how big your account is. -5. Connect to the data in Azure Data Lake Storage +5. Connect to the data in Azure Data Explorer or Data Lake Storage > Consider using [available Power BI reports](https://aka.ms/finops/hubs/reports) If you run into any issues, see [Troubleshooting FinOps hubs](https://aka.ms/finops/hubs/troubleshoot). -_1) A "scope" is an Azure construct that contains resources or enables purchasing services, like a resource group, subscription, management group, or billing account. The resource ID for a scope will be the Azure Resource Manager URI that identifies the scope (e.g., "/subscriptions/###" for a subscription or "/providers/Microsoft.Billing/billingAccounts/###" for a billing account). To learn more, see [Understand and work with scopes](https://aka.ms/costmgmt/scopes)._ -
## 🧰 About the FinOps toolkit @@ -94,4 +98,4 @@ To contribute to the FinOps toolkit, [join us on GitHub](https://aka.ms/ftk).
-`Tags: finops, cost, Microsoft.CostManagement/exports, Microsoft.Storage/storageAccounts, Microsoft.DataFactory/factories` +`Tags: finops, cost, Microsoft.CostManagement/exports, Microsoft.Kusto/clusters, Microsoft.Storage/storageAccounts, Microsoft.DataFactory/factories` diff --git a/src/templates/finops-hub/metadata.json b/src/templates/finops-hub/metadata.json index d4307920f..34438946c 100644 --- a/src/templates/finops-hub/metadata.json +++ b/src/templates/finops-hub/metadata.json @@ -5,5 +5,5 @@ "summary": "Create a new FinOps hub instance", "description": "This template creates a new FinOps hub instance, including Data Explorer, Data Lake storage, and Data Factory.", "githubUsername": "flanakin", - "dateUpdated": "2024-10-02" + "dateUpdated": "2024-11-30" } diff --git a/src/templates/finops-hub/schemas/settings.json b/src/templates/finops-hub/schemas/settings.json index 42de3177e..92783c083 100644 --- a/src/templates/finops-hub/schemas/settings.json +++ b/src/templates/finops-hub/schemas/settings.json @@ -12,7 +12,7 @@ }, "version": { "type": "string", - "enum": ["0.0.1", "0.1", "0.1.1", "0.2", "0.2.1", "0.3", "0.4", "0.5", "0.6"] + "enum": ["0.0.1", "0.1", "0.1.1", "0.2", "0.2.1", "0.3", "0.4", "0.5", "0.6", "0.7"] }, "learnMore": { "type": "string", @@ -58,6 +58,28 @@ }, "additionalProperties": true, "required": ["months"] + }, + "raw": { + "type": "object", + "properties": { + "days": { + "type": "integer", + "minimum": 0 + } + }, + "additionalProperties": true, + "required": ["days"] + }, + "final": { + "type": "object", + "properties": { + "months": { + "type": "integer", + "minimum": 0 + } + }, + "additionalProperties": true, + "required": ["months"] } }, "additionalProperties": true,