Merge pull request #63 from smart-on-fhir/mikix/lib4.0

Update for Library 4.0
smart-on-fhir · Nov 19, 2024 · aacc0ea · aacc0ea
2 parents 1544754 + 5c6f1dc
commit aacc0ea
Show file tree

Hide file tree

Showing 23 changed files with 132 additions and 109 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,7 +1,7 @@
 default_install_hook_types: [pre-commit, pre-push]
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.4.4  # sync with pyproject.toml
+    rev: v0.7.4  # sync with pyproject.toml
     hooks:
       - name: Ruff formatting
         id: ruff-format

diff --git a/cumulus_library_data_metrics/__init__.py b/cumulus_library_data_metrics/__init__.py
@@ -1,3 +1,3 @@
 """Data Metrics study for Cumulus Library"""
 
-__version__ = "5.1.0"
+__version__ = "6.0.0"
diff --git a/cumulus_library_data_metrics/base.py b/cumulus_library_data_metrics/base.py
@@ -19,6 +19,7 @@ class MetricMixin:
     def __init__(self):
         super().__init__()
         self.display_text = f"Creating {self.name} tables…"
+        self.study_prefix = "data_metrics"
         self.output_mode = "cube"
         self.summary_entries = {}
         # A "group" is a value in the second column - a secondary characteristic like "field"
@@ -103,8 +104,10 @@ def prepare_queries(
         self,
         *args,
         config: cumulus_library.StudyConfig,
+        manifest: cumulus_library.StudyManifest,
         **kwargs,
     ) -> None:
+        self.study_prefix = manifest.get_study_prefix()
         self.output_mode = self.get_output_mode(config)
         self._query_schema(config)
         self.extra_schema_checks(config)
@@ -136,11 +139,12 @@ def render_sql(self, template: str, **kwargs) -> str:
 
         # See how we should combine counts.
         kwargs["output_mode"] = self.output_mode
+        kwargs["study_prefix"] = self.study_prefix
 
         with open(f"{path}/{self.name}/{template}.jinja") as file:
             template = file.read()
             loader = jinja2.FileSystemLoader(path)
-            env = jinja2.Environment(loader=loader).from_string(template)
+            env = jinja2.Environment(loader=loader).from_string(template)  # noqa: S701
             sql = env.render(**kwargs)
             # print(sql)
             return sql
diff --git a/cumulus_library_data_metrics/base.summary.jinja b/cumulus_library_data_metrics/base.summary.jinja
@@ -1,9 +1,9 @@
-CREATE TABLE data_metrics__{{ metric }}_summary AS (
+CREATE TABLE {{ study_prefix }}__{{ metric }}_summary AS (
     WITH
     {%- for entry_key, summary_info in entries.items() %}
     {{ entry_key }}_numerator AS (
         SELECT COUNT(id) AS numerator
-        FROM data_metrics__{{ metric }}_{{ entry_key }}
+        FROM {{ study_prefix }}__{{ metric }}_{{ entry_key }}
     ),
     {{ entry_key }}_denominator AS (
         {% if summary_info["denominator"] %}
@@ -23,7 +23,7 @@ CREATE TABLE data_metrics__{{ metric }}_summary AS (
     {{ all_entry_key }}_numerator_ids AS (
         {% for group_entry_key, summary_info in entries.items() %}
         {% if summary_info["src"] == resource %}
-        SELECT id FROM data_metrics__{{ metric }}_{{ group_entry_key }}
+        SELECT id FROM {{ study_prefix }}__{{ metric }}_{{ group_entry_key }}
         UNION
         {% endif %}
         {% endfor %}

diff --git a/cumulus_library_data_metrics/c_pt_count/c_pt_count.jinja b/cumulus_library_data_metrics/c_pt_count/c_pt_count.jinja
@@ -1,7 +1,7 @@
 {% import 'utils.jinja' as utils %}
 {% import 'us_core_v4/patient_utils.jinja' as patient_utils %}
 
-CREATE TABLE data_metrics__count_c_pt_count AS (
+CREATE TABLE {{ study_prefix }}__count_c_pt_count AS (
 WITH
 src_status AS {{ utils.extract_status('Patient') }},
 

diff --git a/cumulus_library_data_metrics/c_pt_deceased_count/c_pt_deceased_count.jinja b/cumulus_library_data_metrics/c_pt_deceased_count/c_pt_deceased_count.jinja
@@ -1,6 +1,6 @@
 {% import 'utils.jinja' as utils %}
 
-CREATE TABLE data_metrics__count_c_pt_deceased_count AS (
+CREATE TABLE {{ study_prefix }}__count_c_pt_deceased_count AS (
 WITH
 
 parsed AS (

diff --git a/cumulus_library_data_metrics/c_resource_count/c_resource_count.jinja b/cumulus_library_data_metrics/c_resource_count/c_resource_count.jinja
@@ -1,6 +1,6 @@
 {% import 'utils.jinja' as utils %}
 
-CREATE TABLE data_metrics__count_c_resource_count_{{ src|lower }}_{{ period|lower }} AS (
+CREATE TABLE {{ study_prefix }}__count_c_resource_count_{{ src|lower }}_{{ period|lower }} AS (
 {%- set orig_cat_field = cat_field %}
 WITH
 src_status AS {{ utils.extract_status(src) }},

diff --git a/cumulus_library_data_metrics/c_resources_per_pt/c_resources_per_pt.jinja b/cumulus_library_data_metrics/c_resources_per_pt/c_resources_per_pt.jinja
@@ -1,6 +1,6 @@
 {% import 'utils.jinja' as utils %}
 
-CREATE TABLE data_metrics__c_resources_per_pt_summary AS (
+CREATE TABLE {{ study_prefix }}__c_resources_per_pt_summary AS (
 
 WITH
 patient_refs AS (

diff --git a/cumulus_library_data_metrics/c_system_use/c_system_use.jinja b/cumulus_library_data_metrics/c_system_use/c_system_use.jinja
@@ -1,6 +1,6 @@
 {% import 'utils.jinja' as utils %}
 
-CREATE TABLE data_metrics__count_c_system_use_{{ src|lower }}_{{ field|lower }} AS (
+CREATE TABLE {{ study_prefix }}__count_c_system_use_{{ src|lower }}_{{ field|lower }} AS (
 
 WITH
 src_status AS {{ utils.extract_status(src) }},

diff --git a/cumulus_library_data_metrics/c_us_core_v4_count/mandatory.jinja b/cumulus_library_data_metrics/c_us_core_v4_count/mandatory.jinja
@@ -2,7 +2,7 @@
 
 {% set ns = namespace(fields=[]) %}
 
-CREATE TABLE data_metrics__count_c_us_core_v4_count_{{ profile_name }}_mandatory{{ table_num|default('') }} AS (
+CREATE TABLE {{ study_prefix }}__count_c_us_core_v4_count_{{ profile_name }}_mandatory{{ table_num|default('') }} AS (
 WITH
 src_status AS {{ utils.extract_status(src) }},
 

diff --git a/cumulus_library_data_metrics/c_us_core_v4_count/must_support.jinja b/cumulus_library_data_metrics/c_us_core_v4_count/must_support.jinja
@@ -2,7 +2,7 @@
 
 {% set ns = namespace(fields=[]) %}
 
-CREATE TABLE data_metrics__count_c_us_core_v4_count_{{ profile_name }}_must_support AS (
+CREATE TABLE {{ study_prefix }}__count_c_us_core_v4_count_{{ profile_name }}_must_support AS (
 WITH
 src_status AS {{ utils.extract_status(src) }},
 

diff --git a/cumulus_library_data_metrics/gen_prefix.py b/cumulus_library_data_metrics/gen_prefix.py
@@ -0,0 +1,17 @@
+#!/usr/bin/env python3
+
+# TODO: once we implement study-cohort slicing,
+#  uncomment this file to allow writing to a study-specific prefix.
+#  But for now, just always use boring old data_metrics__.
+print("data_metrics")
+
+# import argparse
+#
+# parser = argparse.ArgumentParser()
+# parser.add_argument("--study")
+# args, _rest = parser.parse_known_args()
+#
+# if args.study:
+#     print(f"data_metrics_{args.study}")
+# else:
+#     print("data_metrics")
diff --git a/cumulus_library_data_metrics/manifest.toml b/cumulus_library_data_metrics/manifest.toml
@@ -1,6 +1,7 @@
 study_prefix = "data_metrics"
+dynamic_study_prefix = "gen_prefix.py"
 
-[table_builder_config]
+[file_config]
 file_names = [
     "meta/meta.py",
     "c_pt_count/c_pt_count.py",
@@ -17,81 +18,85 @@ file_names = [
 ]
 
 [export_config]
-export_list = [
-    "data_metrics__count_c_pt_count",
-    "data_metrics__count_c_pt_deceased_count",
-    "data_metrics__count_c_resource_count_allergyintolerance_month",
-    "data_metrics__count_c_resource_count_allergyintolerance_year",
-    "data_metrics__count_c_resource_count_condition_month",
-    "data_metrics__count_c_resource_count_condition_year",
-    "data_metrics__count_c_resource_count_device_all",
-    "data_metrics__count_c_resource_count_diagnosticreport_month",
-    "data_metrics__count_c_resource_count_diagnosticreport_year",
-    "data_metrics__count_c_resource_count_documentreference_month",
-    "data_metrics__count_c_resource_count_documentreference_year",
-    "data_metrics__count_c_resource_count_encounter_month",
-    "data_metrics__count_c_resource_count_encounter_year",
-    "data_metrics__count_c_resource_count_immunization_month",
-    "data_metrics__count_c_resource_count_immunization_year",
-    "data_metrics__count_c_resource_count_medication_all",
-    "data_metrics__count_c_resource_count_medicationrequest_month",
-    "data_metrics__count_c_resource_count_medicationrequest_year",
-    "data_metrics__count_c_resource_count_observation_month",
-    "data_metrics__count_c_resource_count_observation_year",
-    "data_metrics__count_c_resource_count_patient_all",
-    "data_metrics__count_c_resource_count_procedure_month",
-    "data_metrics__count_c_resource_count_procedure_year",
-    "data_metrics__c_resources_per_pt_summary",
-    "data_metrics__count_c_system_use_allergyintolerance_code",
-    "data_metrics__count_c_system_use_condition_code",
-    "data_metrics__count_c_system_use_device_type",
-    "data_metrics__count_c_system_use_diagnosticreport_code",
-    "data_metrics__count_c_system_use_documentreference_type",
-    "data_metrics__count_c_system_use_encounter_class",
-    "data_metrics__count_c_system_use_encounter_type",
-    "data_metrics__count_c_system_use_immunization_vaccinecode",
-    "data_metrics__count_c_system_use_medication_code",
-    "data_metrics__count_c_system_use_medicationrequest_medicationcodeableconcept",
-    "data_metrics__count_c_system_use_observation_code",
-    "data_metrics__count_c_system_use_observation_valuecodeableconcept",
-    "data_metrics__count_c_system_use_procedure_code",
-    "data_metrics__count_c_us_core_v4_count_allergyintolerance_mandatory",
-    "data_metrics__count_c_us_core_v4_count_allergyintolerance_must_support",
-    "data_metrics__count_c_us_core_v4_count_condition_mandatory",
-    "data_metrics__count_c_us_core_v4_count_condition_must_support",
-    "data_metrics__count_c_us_core_v4_count_diagnosticreport_lab_mandatory",
-    "data_metrics__count_c_us_core_v4_count_diagnosticreport_lab_must_support",
-    "data_metrics__count_c_us_core_v4_count_diagnosticreport_note_mandatory",
-    "data_metrics__count_c_us_core_v4_count_diagnosticreport_note_must_support",
-    "data_metrics__count_c_us_core_v4_count_documentreference_mandatory",
-    "data_metrics__count_c_us_core_v4_count_documentreference_must_support",
-    "data_metrics__count_c_us_core_v4_count_encounter_mandatory",
-    "data_metrics__count_c_us_core_v4_count_encounter_must_support",
-    "data_metrics__count_c_us_core_v4_count_immunization_mandatory",
-    "data_metrics__count_c_us_core_v4_count_immunization_must_support",
-    "data_metrics__count_c_us_core_v4_count_medication_mandatory",
-    "data_metrics__count_c_us_core_v4_count_medication_must_support",
-    "data_metrics__count_c_us_core_v4_count_medicationrequest_mandatory",
-    "data_metrics__count_c_us_core_v4_count_medicationrequest_must_support",
-    "data_metrics__count_c_us_core_v4_count_observation_laboratory_mandatory1",
-    "data_metrics__count_c_us_core_v4_count_observation_laboratory_mandatory2",
-    "data_metrics__count_c_us_core_v4_count_observation_laboratory_must_support",
-    "data_metrics__count_c_us_core_v4_count_observation_smoking_status_mandatory1",
-    "data_metrics__count_c_us_core_v4_count_observation_smoking_status_mandatory2",
-    "data_metrics__count_c_us_core_v4_count_observation_smoking_status_must_support",
-    "data_metrics__count_c_us_core_v4_count_observation_vital_signs_mandatory1",
-    "data_metrics__count_c_us_core_v4_count_observation_vital_signs_mandatory2",
-    "data_metrics__count_c_us_core_v4_count_observation_vital_signs_mandatory3",
-    "data_metrics__count_c_us_core_v4_count_observation_vital_signs_must_support",
-    "data_metrics__count_c_us_core_v4_count_patient_mandatory",
-    "data_metrics__count_c_us_core_v4_count_patient_must_support",
-    "data_metrics__count_c_us_core_v4_count_procedure_mandatory",
-    "data_metrics__count_c_us_core_v4_count_procedure_must_support",
-    "data_metrics__q_date_recent_summary",
-    "data_metrics__q_ref_target_pop_summary",
-    "data_metrics__q_ref_target_valid_summary",
-    "data_metrics__q_system_use_summary",
-    "data_metrics__q_valid_us_core_v4_summary",
-    "data_metrics__meta_date",
-    "data_metrics__meta_version",
+count_list = [
+    "count_c_pt_count",
+    "count_c_pt_deceased_count",
+    "count_c_resource_count_allergyintolerance_month",
+    "count_c_resource_count_allergyintolerance_year",
+    "count_c_resource_count_condition_month",
+    "count_c_resource_count_condition_year",
+    "count_c_resource_count_device_all",
+    "count_c_resource_count_diagnosticreport_month",
+    "count_c_resource_count_diagnosticreport_year",
+    "count_c_resource_count_documentreference_month",
+    "count_c_resource_count_documentreference_year",
+    "count_c_resource_count_encounter_month",
+    "count_c_resource_count_encounter_year",
+    "count_c_resource_count_immunization_month",
+    "count_c_resource_count_immunization_year",
+    "count_c_resource_count_medication_all",
+    "count_c_resource_count_medicationrequest_month",
+    "count_c_resource_count_medicationrequest_year",
+    "count_c_resource_count_observation_month",
+    "count_c_resource_count_observation_year",
+    "count_c_resource_count_patient_all",
+    "count_c_resource_count_procedure_month",
+    "count_c_resource_count_procedure_year",
+    "count_c_system_use_allergyintolerance_code",
+    "count_c_system_use_condition_code",
+    "count_c_system_use_device_type",
+    "count_c_system_use_diagnosticreport_code",
+    "count_c_system_use_documentreference_type",
+    "count_c_system_use_encounter_class",
+    "count_c_system_use_encounter_type",
+    "count_c_system_use_immunization_vaccinecode",
+    "count_c_system_use_medication_code",
+    "count_c_system_use_medicationrequest_medicationcodeableconcept",
+    "count_c_system_use_observation_code",
+    "count_c_system_use_observation_valuecodeableconcept",
+    "count_c_system_use_procedure_code",
+    "count_c_us_core_v4_count_allergyintolerance_mandatory",
+    "count_c_us_core_v4_count_allergyintolerance_must_support",
+    "count_c_us_core_v4_count_condition_mandatory",
+    "count_c_us_core_v4_count_condition_must_support",
+    "count_c_us_core_v4_count_diagnosticreport_lab_mandatory",
+    "count_c_us_core_v4_count_diagnosticreport_lab_must_support",
+    "count_c_us_core_v4_count_diagnosticreport_note_mandatory",
+    "count_c_us_core_v4_count_diagnosticreport_note_must_support",
+    "count_c_us_core_v4_count_documentreference_mandatory",
+    "count_c_us_core_v4_count_documentreference_must_support",
+    "count_c_us_core_v4_count_encounter_mandatory",
+    "count_c_us_core_v4_count_encounter_must_support",
+    "count_c_us_core_v4_count_immunization_mandatory",
+    "count_c_us_core_v4_count_immunization_must_support",
+    "count_c_us_core_v4_count_medication_mandatory",
+    "count_c_us_core_v4_count_medication_must_support",
+    "count_c_us_core_v4_count_medicationrequest_mandatory",
+    "count_c_us_core_v4_count_medicationrequest_must_support",
+    "count_c_us_core_v4_count_observation_laboratory_mandatory1",
+    "count_c_us_core_v4_count_observation_laboratory_mandatory2",
+    "count_c_us_core_v4_count_observation_laboratory_must_support",
+    "count_c_us_core_v4_count_observation_smoking_status_mandatory1",
+    "count_c_us_core_v4_count_observation_smoking_status_mandatory2",
+    "count_c_us_core_v4_count_observation_smoking_status_must_support",
+    "count_c_us_core_v4_count_observation_vital_signs_mandatory1",
+    "count_c_us_core_v4_count_observation_vital_signs_mandatory2",
+    "count_c_us_core_v4_count_observation_vital_signs_mandatory3",
+    "count_c_us_core_v4_count_observation_vital_signs_must_support",
+    "count_c_us_core_v4_count_patient_mandatory",
+    "count_c_us_core_v4_count_patient_must_support",
+    "count_c_us_core_v4_count_procedure_mandatory",
+    "count_c_us_core_v4_count_procedure_must_support",
+]
+flat_list = [
+    "c_resources_per_pt_summary",
+    "q_date_recent_summary",
+    "q_ref_target_pop_summary",
+    "q_ref_target_valid_summary",
+    "q_system_use_summary",
+    "q_valid_us_core_v4_summary",
+]
+meta_list = [
+    "meta_date",
+    "meta_version",
 ]
diff --git a/cumulus_library_data_metrics/meta/dates.jinja b/cumulus_library_data_metrics/meta/dates.jinja
@@ -1,6 +1,6 @@
 {% import 'utils.jinja' as utils %}
 
-CREATE TABLE data_metrics__meta_date AS (
+CREATE TABLE {{ study_prefix }}__meta_date AS (
 
 WITH
 

diff --git a/cumulus_library_data_metrics/meta/version.jinja b/cumulus_library_data_metrics/meta/version.jinja
@@ -1,2 +1,2 @@
-CREATE TABLE data_metrics__meta_version AS
+CREATE TABLE {{ study_prefix }}__meta_version AS
 SELECT 1 AS data_package_version;
diff --git a/cumulus_library_data_metrics/q_date_recent/q_date_recent.jinja b/cumulus_library_data_metrics/q_date_recent/q_date_recent.jinja
@@ -1,6 +1,6 @@
 {% import 'utils.jinja' as utils %}
 
-CREATE TABLE data_metrics__q_date_recent_{{ src|lower }} AS (
+CREATE TABLE {{ study_prefix }}__q_date_recent_{{ src|lower }} AS (
 WITH
 src_status AS {{ utils.extract_status(src) }}
 

diff --git a/cumulus_library_data_metrics/q_ref_target_pop/q_ref_target_pop.jinja b/cumulus_library_data_metrics/q_ref_target_pop/q_ref_target_pop.jinja
@@ -1,6 +1,6 @@
 {% import 'utils.jinja' as utils %}
 
-CREATE TABLE data_metrics__q_ref_target_pop_{{ src|lower }}_{{ dest|lower }} AS (
+CREATE TABLE {{ study_prefix }}__q_ref_target_pop_{{ src|lower }}_{{ dest|lower }} AS (
 WITH
 src_status AS {{ utils.extract_status(src) }}
 

diff --git a/cumulus_library_data_metrics/q_ref_target_valid/q_ref_target_valid.jinja b/cumulus_library_data_metrics/q_ref_target_valid/q_ref_target_valid.jinja
@@ -1,6 +1,6 @@
 {% import 'utils.jinja' as utils %}
 
-CREATE TABLE data_metrics__q_ref_target_valid_{{ src|lower }}_{{ dest|lower }} AS (
+CREATE TABLE {{ study_prefix }}__q_ref_target_valid_{{ src|lower }}_{{ dest|lower }} AS (
 
 {% if field == 'context.encounter' and not schema['context']['encounter'] %}
     SELECT id FROM {{ src }} WHERE 1=0  -- return an empty table

diff --git a/cumulus_library_data_metrics/q_system_use/q_system_use.jinja b/cumulus_library_data_metrics/q_system_use/q_system_use.jinja
@@ -2,7 +2,7 @@
 
 {%- set system_list = "('" + systems|join("', '") + "')" %}
 
-CREATE TABLE data_metrics__q_system_use_{{ src|lower }}_{{ field|lower }} AS (
+CREATE TABLE {{ study_prefix }}__q_system_use_{{ src|lower }}_{{ field|lower }} AS (
 WITH
 src_status AS {{ utils.extract_status(src) }},
 

diff --git a/cumulus_library_data_metrics/q_valid_us_core_v4/q_valid_us_core_v4.jinja b/cumulus_library_data_metrics/q_valid_us_core_v4/q_valid_us_core_v4.jinja
@@ -2,7 +2,7 @@
 
 {% set ns = namespace(fields=[]) %}
 
-CREATE TABLE data_metrics__q_valid_us_core_v4_{{ profile_name }} AS (
+CREATE TABLE {{ study_prefix }}__q_valid_us_core_v4_{{ profile_name }} AS (
 WITH
 src_status AS {{ utils.extract_status(src) }},
 

diff --git a/cumulus_library_data_metrics/t_us_core_v4/mandatory.jinja b/cumulus_library_data_metrics/t_us_core_v4/mandatory.jinja
@@ -2,6 +2,6 @@
 
 {% set ns = namespace(fields=[]) %}
 
-CREATE TABLE data_metrics__t_us_core_v4_{{ profile_name }}_mandatory AS (
+CREATE TABLE {{ study_prefix }}__t_us_core_v4_{{ profile_name }}_mandatory AS (
     {% include 'us_core_v4/' + profile_name + '_mandatory.jinja' %}
 );
diff --git a/cumulus_library_data_metrics/t_us_core_v4/must_support.jinja b/cumulus_library_data_metrics/t_us_core_v4/must_support.jinja
@@ -2,6 +2,6 @@
 
 {% set ns = namespace(fields=[]) %}
 
-CREATE TABLE data_metrics__t_us_core_v4_{{ profile_name }}_must_support AS (
+CREATE TABLE {{ study_prefix }}__t_us_core_v4_{{ profile_name }}_must_support AS (
     {% include 'us_core_v4/' + profile_name + '_must_support.jinja' %}
 );