From 2deda073e4d59e89650841b99a2a3bbf41c10115 Mon Sep 17 00:00:00 2001 From: michaelglenister Date: Fri, 14 Apr 2023 13:18:53 +0200 Subject: [PATCH 1/7] Get spend summary data from V2 cube --- scorecard/profile_data/api_data.py | 31 ++++++++---- scorecard/profile_data/indicators/__init__.py | 49 +++++++++++++++++++ 2 files changed, 70 insertions(+), 10 deletions(-) diff --git a/scorecard/profile_data/api_data.py b/scorecard/profile_data/api_data.py index 77a5463ac..b1006235a 100644 --- a/scorecard/profile_data/api_data.py +++ b/scorecard/profile_data/api_data.py @@ -446,11 +446,7 @@ def get_queries(self): "cube": "incexp_v2", "aggregate": "amount.sum", "cut": { - "item.code": [ - "2000", "2100", "2200", "2300", "2400", - "2500", "2600", "2700", "2800", "2900", - "3000", - ], + "item.code": V2_SPENDING_CODES, "amount_type.code": ["ADJB"], "demarcation.code": [self.geo_code], "period_length.length": ["year"], @@ -480,11 +476,7 @@ def get_queries(self): "cube": "incexp_v2", "aggregate": "amount.sum", "cut": { - "item.code": [ - "2000", "2100", "2200", "2300", "2400", - "2500", "2600", "2700", "2800", "2900", - "3000", - ], + "item.code": V2_SPENDING_CODES, "amount_type.code": ["AUDA"], "demarcation.code": [self.geo_code], "period_length.length": ["year"], @@ -863,6 +855,25 @@ def get_queries(self): "results_structure": self.noop_structure, "order": "financial_year_end.year:desc,function.category_label:asc", }, + "expenditure_functional_breakdown_v2": { + "cube": "incexp_v2", + "aggregate": "amount.sum", + "cut": { + "item.code": V2_SPENDING_CODES, + "amount_type.code": ["AUDA"], + "demarcation.code": [self.geo_code], + "period_length.length": ["year"], + "financial_year_end.year": self.years + [self.budget_year], + }, + "drilldown": [ + "function.category_label", + "financial_year_end.year", + "amount_type.code", + ], + "query_type": "aggregate", + "results_structure": self.noop_structure, + "order": "financial_year_end.year:desc,function.category_label:asc", + }, "expenditure_trends": { "cube": "incexp", "aggregate": "amount.sum", diff --git a/scorecard/profile_data/indicators/__init__.py b/scorecard/profile_data/indicators/__init__.py index fffe0c2bb..2ba5f3d82 100644 --- a/scorecard/profile_data/indicators/__init__.py +++ b/scorecard/profile_data/indicators/__init__.py @@ -24,6 +24,9 @@ ) from collections import defaultdict +import logging +logger = logging.Logger(__name__) + def get_indicator_calculators(has_comparisons=None): calculators = [ CashCoverage, @@ -313,4 +316,50 @@ def get_muni_specifics(cls, api_data): grouped_results = sorted( grouped_results, key=lambda r: (r["date"], r["item"])) + + # Get data from V2 cube + results = api_data.results["expenditure_functional_breakdown_v2"] + + for year, yeargroup in groupby(results, lambda r: r["financial_year_end.year"]): + yeargroup_list = list(yeargroup) + if len(yeargroup_list) == 0: + continue + total = sum(x["amount.sum"] for x in yeargroup_list) + try: + GAPD_total = 0.0 + year_name = ( + "%d" % year + if year != api_data.budget_year + else ("%s budget" % year) + ) + + for result in yeargroup_list: + # only do budget for budget year, use AUDA for others + if api_data.check_budget_actual(year, result["amount_type.code"]): + if result["function.category_label"] in GAPD_categories: + GAPD_total += result["amount.sum"] + else: + grouped_results.append( + { + "amount": result["amount.sum"], + "percent": percent(result["amount.sum"], total), + "item": result["function.category_label"], + "date": year_name, + } + ) + + grouped_results.append( + { + "amount": GAPD_total, + "percent": percent(GAPD_total, total), + "item": GAPD_label, + "date": year_name, + } + ) + except (KeyError, IndexError): + continue + + grouped_results = sorted( + grouped_results, key=lambda r: (r["date"], r["item"])) + return {"values": grouped_results} From 64fc3fa262e3837df67d10b8e0a53057c830a1d7 Mon Sep 17 00:00:00 2001 From: michaelglenister Date: Mon, 17 Apr 2023 15:09:04 +0200 Subject: [PATCH 2/7] Get spend summary data from V2 cube --- scorecard/profile_data/indicators/__init__.py | 48 +------------------ 1 file changed, 1 insertion(+), 47 deletions(-) diff --git a/scorecard/profile_data/indicators/__init__.py b/scorecard/profile_data/indicators/__init__.py index 2ba5f3d82..44f1411e6 100644 --- a/scorecard/profile_data/indicators/__init__.py +++ b/scorecard/profile_data/indicators/__init__.py @@ -24,8 +24,6 @@ ) from collections import defaultdict -import logging -logger = logging.Logger(__name__) def get_indicator_calculators(has_comparisons=None): calculators = [ @@ -273,53 +271,9 @@ def get_muni_specifics(cls, api_data): GAPD_label = "Governance, Administration, Planning and Development" results = api_data.results["expenditure_functional_breakdown"] + results.extend(api_data.results["expenditure_functional_breakdown_v2"]) grouped_results = [] - for year, yeargroup in groupby(results, lambda r: r["financial_year_end.year"]): - yeargroup_list = list(yeargroup) - if len(yeargroup_list) == 0: - continue - total = sum(x["amount.sum"] for x in yeargroup_list) - try: - GAPD_total = 0.0 - year_name = ( - "%d" % year - if year != api_data.budget_year - else ("%s budget" % year) - ) - - for result in yeargroup_list: - # only do budget for budget year, use AUDA for others - if api_data.check_budget_actual(year, result["amount_type.code"]): - if result["function.category_label"] in GAPD_categories: - GAPD_total += result["amount.sum"] - else: - grouped_results.append( - { - "amount": result["amount.sum"], - "percent": percent(result["amount.sum"], total), - "item": result["function.category_label"], - "date": year_name, - } - ) - - grouped_results.append( - { - "amount": GAPD_total, - "percent": percent(GAPD_total, total), - "item": GAPD_label, - "date": year_name, - } - ) - except (KeyError, IndexError): - continue - - grouped_results = sorted( - grouped_results, key=lambda r: (r["date"], r["item"])) - - # Get data from V2 cube - results = api_data.results["expenditure_functional_breakdown_v2"] - for year, yeargroup in groupby(results, lambda r: r["financial_year_end.year"]): yeargroup_list = list(yeargroup) if len(yeargroup_list) == 0: From 9bd7afccfaacc0a5ca8278c5aac79243a18f9624 Mon Sep 17 00:00:00 2001 From: michaelglenister Date: Mon, 17 Apr 2023 15:11:00 +0200 Subject: [PATCH 3/7] Format --- scorecard/profile_data/indicators/__init__.py | 30 ++++++++++++++----- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/scorecard/profile_data/indicators/__init__.py b/scorecard/profile_data/indicators/__init__.py index 44f1411e6..5b3d73ec7 100644 --- a/scorecard/profile_data/indicators/__init__.py +++ b/scorecard/profile_data/indicators/__init__.py @@ -72,8 +72,12 @@ def get_muni_specifics(cls, api_data): items = v1_data[year] results = { - "local": {"amount": 0, }, - "government": {"amount": 0, }, + "local": { + "amount": 0, + }, + "government": { + "amount": 0, + }, "year": year, "ref": api_data.references["lges"], } @@ -111,7 +115,6 @@ class LocalRevenueBreakdown(IndicatorCalculator): @classmethod def get_muni_specifics(cls, api_data): - v1_groups = [ ("Property rates", ["0200", "0300"]), ("Service Charges", ["0400"]), @@ -195,7 +198,11 @@ def get_muni_specifics(cls, api_data): contracting_code = "4200" total = sum(x["amount.sum"] for x in results) - contracting_items = [x["amount.sum"] for x in results if x["item.code"] == contracting_code] + contracting_items = [ + x["amount.sum"] + for x in results + if x["item.code"] == contracting_code + ] contracting = percent(contracting_items[0], total) # Prefer KeyError but crash before we use it in case we have more than expectexd assert len(contracting_items) <= 1 @@ -204,7 +211,11 @@ def get_muni_specifics(cls, api_data): contracting = None values.append( - {"date": year, "result": contracting, "rating": "", } + { + "date": year, + "result": contracting, + "rating": "", + } ) return { @@ -247,7 +258,11 @@ def get_muni_specifics(cls, api_data): staff = None values.append( - {"date": year, "result": staff, "rating": "", } + { + "date": year, + "result": staff, + "rating": "", + } ) return { @@ -313,7 +328,6 @@ def get_muni_specifics(cls, api_data): except (KeyError, IndexError): continue - grouped_results = sorted( - grouped_results, key=lambda r: (r["date"], r["item"])) + grouped_results = sorted(grouped_results, key=lambda r: (r["date"], r["item"])) return {"values": grouped_results} From ca3f23d70a9dd625f0458fd8ede3dcbf44beae47 Mon Sep 17 00:00:00 2001 From: michaelglenister Date: Mon, 17 Apr 2023 15:35:33 +0200 Subject: [PATCH 4/7] Test expenditure functional breakdown --- .../test_expenditure_functional_breakdown.py | 319 ++++++++++++++++++ 1 file changed, 319 insertions(+) create mode 100644 scorecard/tests/indicators/test_expenditure_functional_breakdown.py diff --git a/scorecard/tests/indicators/test_expenditure_functional_breakdown.py b/scorecard/tests/indicators/test_expenditure_functional_breakdown.py new file mode 100644 index 000000000..1466fd92c --- /dev/null +++ b/scorecard/tests/indicators/test_expenditure_functional_breakdown.py @@ -0,0 +1,319 @@ +from ...profile_data import ApiData +from ...profile_data.indicators import ( + ExpenditureFunctionalBreakdown, +) + +from . import ( + import_data, + _IndicatorTestCase, +) +from .resources import ( + GeographyResource, + IncexpFactsV1Resource, + IncexpFactsV2Resource, +) + + +# expenditure_functional_breakdown +class TestExpenditureFunctionalBreakdown(_IndicatorTestCase): + def test_result(self): + # Load sample data + import_data( + GeographyResource, "operating_budget_spending/scorecard_geography.csv" + ) + import_data( + IncexpFactsV1Resource, + "operating_budget_spending/income_expenditure_facts_v1.csv", + ) + import_data( + IncexpFactsV2Resource, + "operating_budget_spending/income_expenditure_facts_v2.csv", + ) + # Fetch data from API + api_data = ApiData(self.api_client, "CPT", 2020, 2020, 2020, "2020q4") + api_data.fetch_data( + [ + "expenditure_functional_breakdown", + "expenditure_functional_breakdown_v2", + ] + ) + # Provide data to indicator + result = ExpenditureFunctionalBreakdown.get_muni_specifics(api_data) + + self.assertEqual( + { + "values": [ + { + "amount": 642335159.0, + "percent": 1.95, + "item": "Community & Social Services", + "date": "2017", + }, + { + "amount": 9764952152.0, + "percent": 29.57, + "item": "Electricity ", + "date": "2017", + }, + { + "amount": 113416863.0, + "percent": 0.34, + "item": "Environmental Protection", + "date": "2017", + }, + { + "amount": 6205745326.0, + "percent": 18.79, + "item": "Governance, Administration, Planning and Development", + "date": "2017", + }, + { + "amount": 999459766.0, + "percent": 3.03, + "item": "Health", + "date": "2017", + }, + { + "amount": 1343676621.0, + "percent": 4.07, + "item": "Housing", + "date": "2017", + }, + { + "amount": 242541316.0, + "percent": 0.73, + "item": "Other", + "date": "2017", + }, + { + "amount": 2697688277.0, + "percent": 8.17, + "item": "Public Safety", + "date": "2017", + }, + { + "amount": 2962138419.0, + "percent": 8.97, + "item": "Road Transport", + "date": "2017", + }, + { + "amount": 1350117891.0, + "percent": 4.09, + "item": "Sport And Recreation", + "date": "2017", + }, + { + "amount": 1968026884.0, + "percent": 5.96, + "item": "Waste Management", + "date": "2017", + }, + { + "amount": 1436777514.0, + "percent": 4.35, + "item": "Waste Water Management", + "date": "2017", + }, + { + "amount": 3296754955.0, + "percent": 9.98, + "item": "Water", + "date": "2017", + }, + { + "amount": 837312699.0, + "percent": 2.44, + "item": "Community & Social Services", + "date": "2018", + }, + { + "amount": 9385677420.0, + "percent": 27.35, + "item": "Electricity ", + "date": "2018", + }, + { + "amount": 125801713.0, + "percent": 0.37, + "item": "Environmental Protection", + "date": "2018", + }, + { + "amount": 7598669618.0, + "percent": 22.14, + "item": "Governance, Administration, Planning and Development", + "date": "2018", + }, + { + "amount": 1068081082.0, + "percent": 3.11, + "item": "Health", + "date": "2018", + }, + { + "amount": 1181977044.0, + "percent": 3.44, + "item": "Housing", + "date": "2018", + }, + { + "amount": 744322793.0, + "percent": 2.17, + "item": "Other", + "date": "2018", + }, + { + "amount": 656863605.0, + "percent": 1.91, + "item": "Public Safety", + "date": "2018", + }, + { + "amount": 5284650476.0, + "percent": 15.4, + "item": "Road Transport", + "date": "2018", + }, + { + "amount": 1114225213.0, + "percent": 3.25, + "item": "Sport And Recreation", + "date": "2018", + }, + { + "amount": 2111879736.0, + "percent": 6.15, + "item": "Waste Management", + "date": "2018", + }, + { + "amount": 1181386852.0, + "percent": 3.44, + "item": "Waste Water Management", + "date": "2018", + }, + { + "amount": 3026137787.0, + "percent": 8.82, + "item": "Water", + "date": "2018", + }, + { + "amount": 883385270.0, + "percent": 2.42, + "item": "Community & Social Services", + "date": "2019", + }, + { + "amount": 9934887545.0, + "percent": 27.22, + "item": "Electricity ", + "date": "2019", + }, + { + "amount": 144546505.0, + "percent": 0.4, + "item": "Environmental Protection", + "date": "2019", + }, + { + "amount": 8950230060.0, + "percent": 24.52, + "item": "Governance, Administration, Planning and Development", + "date": "2019", + }, + { + "amount": 1156258229.0, + "percent": 3.17, + "item": "Health", + "date": "2019", + }, + { + "amount": 1236545059.0, + "percent": 3.39, + "item": "Housing", + "date": "2019", + }, + { + "amount": 318842626.0, + "percent": 0.87, + "item": "Other", + "date": "2019", + }, + { + "amount": 643571112.0, + "percent": 1.76, + "item": "Public Safety", + "date": "2019", + }, + { + "amount": 5701801276.0, + "percent": 15.62, + "item": "Road Transport", + "date": "2019", + }, + { + "amount": 1187603633.0, + "percent": 3.25, + "item": "Sport And Recreation", + "date": "2019", + }, + { + "amount": 2300992268.0, + "percent": 6.3, + "item": "Waste Management", + "date": "2019", + }, + { + "amount": 1252179065.0, + "percent": 3.43, + "item": "Waste Water Management", + "date": "2019", + }, + { + "amount": 2785243355.0, + "percent": 7.63, + "item": "Water", + "date": "2019", + }, + { + "amount": 7288097448.0, + "percent": 20.15, + "item": "Community and public safety", + "date": "2020", + }, + { + "amount": 4439158290.0, + "percent": 12.27, + "item": "Economic and environmental services", + "date": "2020", + }, + { + "amount": 0.0, + "percent": 0.0, + "item": "Governance, Administration, Planning and Development", + "date": "2020", + }, + { + "amount": 7973764641.0, + "percent": 22.05, + "item": "Municipal governance and administration", + "date": "2020", + }, + { + "amount": 381585830.0, + "percent": 1.06, + "item": "Other", + "date": "2020", + }, + { + "amount": 16081731467.0, + "percent": 44.47, + "item": "Trading services", + "date": "2020", + }, + ] + }, + result, + ) From e3757669961250c83db557a2f76c797b1430bb4c Mon Sep 17 00:00:00 2001 From: michaelglenister Date: Mon, 17 Apr 2023 16:27:57 +0200 Subject: [PATCH 5/7] Remove comment --- .../tests/indicators/test_expenditure_functional_breakdown.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scorecard/tests/indicators/test_expenditure_functional_breakdown.py b/scorecard/tests/indicators/test_expenditure_functional_breakdown.py index 1466fd92c..31e574f39 100644 --- a/scorecard/tests/indicators/test_expenditure_functional_breakdown.py +++ b/scorecard/tests/indicators/test_expenditure_functional_breakdown.py @@ -14,7 +14,6 @@ ) -# expenditure_functional_breakdown class TestExpenditureFunctionalBreakdown(_IndicatorTestCase): def test_result(self): # Load sample data From af17d88081d8224f5c9e98cb745b022f235dc246 Mon Sep 17 00:00:00 2001 From: michaelglenister Date: Mon, 22 May 2023 11:41:22 +0200 Subject: [PATCH 6/7] mscoa years for spend summary --- scorecard/profile_data/indicators/__init__.py | 18 +++++++++--------- scorecard/profiles.py | 9 +++++---- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/scorecard/profile_data/indicators/__init__.py b/scorecard/profile_data/indicators/__init__.py index 5b3d73ec7..600a05232 100644 --- a/scorecard/profile_data/indicators/__init__.py +++ b/scorecard/profile_data/indicators/__init__.py @@ -317,17 +317,17 @@ def get_muni_specifics(cls, api_data): } ) - grouped_results.append( - { - "amount": GAPD_total, - "percent": percent(GAPD_total, total), - "item": GAPD_label, - "date": year_name, - } - ) + if GAPD_total > 0: + grouped_results.append( + { + "amount": GAPD_total, + "percent": percent(GAPD_total, total), + "item": GAPD_label, + "date": year_name, + } + ) except (KeyError, IndexError): continue grouped_results = sorted(grouped_results, key=lambda r: (r["date"], r["item"])) - return {"values": grouped_results} diff --git a/scorecard/profiles.py b/scorecard/profiles.py index 355dae519..6cc5f0873 100644 --- a/scorecard/profiles.py +++ b/scorecard/profiles.py @@ -93,8 +93,9 @@ def get_muni_comparison(geo, objects): for calculator in get_indicator_calculators(has_comparisons=True): name = calculator.name objects.get(group_id='national') - national = national_group[name][geo.miif_category] - comparisons[name]['national']['dev_cat'] = national - provincial = provincial_group[name][geo.province_code][geo.miif_category] - comparisons[name]['provincial']['dev_cat'] = provincial + if name in national_group and geo.miif_category in provincial_group[name][geo.province_code]: + national = national_group[name][geo.miif_category] + comparisons[name]['national']['dev_cat'] = national + provincial = provincial_group[name][geo.province_code][geo.miif_category] + comparisons[name]['provincial']['dev_cat'] = provincial return comparisons From 431e14997237f39558c7d87521a2698f56c823ed Mon Sep 17 00:00:00 2001 From: michaelglenister Date: Mon, 22 May 2023 12:21:33 +0200 Subject: [PATCH 7/7] mscoa years for spend summary --- .../indicators/test_expenditure_functional_breakdown.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/scorecard/tests/indicators/test_expenditure_functional_breakdown.py b/scorecard/tests/indicators/test_expenditure_functional_breakdown.py index 31e574f39..0afbe5772 100644 --- a/scorecard/tests/indicators/test_expenditure_functional_breakdown.py +++ b/scorecard/tests/indicators/test_expenditure_functional_breakdown.py @@ -288,12 +288,6 @@ def test_result(self): "item": "Economic and environmental services", "date": "2020", }, - { - "amount": 0.0, - "percent": 0.0, - "item": "Governance, Administration, Planning and Development", - "date": "2020", - }, { "amount": 7973764641.0, "percent": 22.05,