Merge branch 'sandbox' of https://github.com/publichealthengland/coro…

…navirus-dashboard-api-v2-server into sandbox
UKHSA-Internal · Mar 15, 2023 · 5444f41 · 5444f41
2 parents ef27ddf + be64a98
commit 5444f41
Show file tree

Hide file tree

Showing 5 changed files with 37 additions and 37 deletions.
diff --git a/app/engine/from_db/generic.py b/app/engine/from_db/generic.py
@@ -31,20 +31,26 @@ def process_generic_data(results: Iterable[Record], request: Request) -> DataFra
         for metric in filter(response_metrics.__contains__, MetricData.generic_dtypes)
     }
 
-    payload = (
-        df
-        .pivot_table(
-            values="value",
-            index=MetricData.base_metrics,
-            columns="metric",
-            aggfunc='first'
+    try:
+        payload = (
+            df
+            .pivot_table(
+                values="value",
+                index=MetricData.base_metrics,
+                columns="metric",
+                aggfunc='first'
+            )
+            .reset_index()
+            .sort_values(["date", "areaCode"], ascending=[False, True])
+            .pipe(format_dtypes, column_types=column_types)
+            .loc[:, [*MetricData.base_metrics, *response_metrics]]
+            .pipe(format_msoas, request=request)
+            .pipe(format_data, response_metrics=response_metrics)
         )
-        .reset_index()
-        .sort_values(["date", "areaCode"], ascending=[False, True])
-        .pipe(format_dtypes, column_types=column_types)
-        .loc[:, [*MetricData.base_metrics, *response_metrics]]
-        .pipe(format_msoas, request=request)
-        .pipe(format_data, response_metrics=response_metrics)
-    )
+    except KeyError as err:
+        # This can happen if there are only null values in the df
+        # then some operations on the dataframe can't be performed
+        # Return the expected Dataframe object
+        payload = DataFrame()
 
     return payload
diff --git a/app/engine/from_db/nested.py b/app/engine/from_db/nested.py
@@ -26,27 +26,17 @@ def process_nested_data(results: Iterable[Record], request: Request) -> DataFram
         df = json_normalize(
             map(dict, results),
             nested_metric_name,
-            [*base_columns, MetricData.nested_struct[nested_metric_name]],
-            errors='ignore'
+            base_columns,
         )
-
-        df.rename(
-            columns={col: col.removeprefix(f"{nested_metric_name}.") for col in df.columns},
-            inplace=True
-        )
-
-        columns = [*base_columns, *MetricData.nested_struct[nested_metric_name]]
     else:
         df = DataFrame(
             results,
             columns=["areaCode", "areaType", "areaName", "date", "metric", nested_metric_name]
         )
-        columns = df.columns
 
     payload = (
         df
         .sort_values(["date", "areaCode"], ascending=[False, True])
-        .loc[:, columns]
     )
     payload = payload.where(payload.notnull(), None)
 

diff --git a/app/engine/from_db/utils.py b/app/engine/from_db/utils.py
@@ -106,7 +106,8 @@ async def cache_response(func, *, request: Request, **kwargs) -> bool:
 
 
 def format_dtypes(df: DataFrame, column_types: Dict[str, object]) -> DataFrame:
-    json_columns = MetricData.json_dtypes.intersection(column_types)
+    # passing a list instead of a set to avid compatibility with new pandas version
+    json_columns = list(MetricData.json_dtypes.intersection(column_types))
 
     # Replace `null` string with None. This happens because
     # some DB queries convert `null` to `"null"` for type

diff --git a/app/utils/assets.py b/app/utils/assets.py
@@ -38,7 +38,7 @@
 
 
 query = """\
-SELECT rr.timestamp 
+SELECT rr.timestamp
 FROM covid19.release_reference AS rr
 JOIN covid19.release_category AS rc ON rc.release_id = rr.id
 WHERE DATE(rr.timestamp) = $1
@@ -146,16 +146,12 @@ class MetricData:
             "cumVaccinationThirdInjectionUptakeByVaccinationDatePercentage",
             "cumVaccinationCompleteCoverageByVaccinationDatePercentage",
 
+            "newPeopleVaccinatedAutumn22ByVaccinationDate",
+            "cumPeopleVaccinatedAutumn22ByVaccinationDate",
+            "cumVaccinationAutumn22UptakeByVaccinationDatePercentage",
             "newPeopleVaccinatedSpring22ByVaccinationDate",
             "cumPeopleVaccinatedSpring22ByVaccinationDate",
             "cumVaccinationSpring22UptakeByVaccinationDatePercentage",
-
-            'newPeopleVaccinatedSpring22ByVaccinationDate',
-            'newPeopleVaccinatedAutumn22ByVaccinationDate',
-            'cumPeopleVaccinatedSpring22ByVaccinationDate',
-            'cumPeopleVaccinatedAutumn22ByVaccinationDate',
-            'cumVaccinationAutumn22UptakeByVaccinationDatePercentage',
-            'cumVaccinationSpring22UptakeByVaccinationDatePercentage',
         ]
     }
 

diff --git a/app/utils/operations/response.py b/app/utils/operations/response.py
@@ -8,7 +8,8 @@
 
 # 3rd party:
 
-# Internal: 
+# Internal:
+from app.config import Settings
 from ..assets import get_latest_timestamp
 from .request import Request
 
@@ -21,7 +22,7 @@
 
 
 API_PREFIX = "/api/"
-API_URL = "coronavirus.data.gov.uk"
+API_URL = Settings.service_domain
 
 ResponseContentType = Union[None, bytes, AsyncGenerator[bytes, None]]
 
@@ -38,7 +39,13 @@ def __init__(self, request, container, path):
         host: str = request.base_request.headers.get("X-Forwarded-Host", API_URL)
         host = host.removeprefix("https://").removeprefix("api.")
 
-        self.location = f"https://api.{host}/downloads/{container}/{path}"
+        # TODO: remove this IF statement when routing rule for rr-apiv2cache frontend
+        #       domain name has been change
+        self.location = (
+            f"https://api.{host}/downloads/{container}/{path}"
+            if not host.startswith("sandbox")
+            else f"https://api-{host}/downloads/{container}/{path}"
+        )
 
         permalink = f"https://{API_URL}/apiv2cache/{request.path}"