Merge pull request #1850 from TEAMSchools/1842-deprecate-teamster_uti…

…ls-dbt-package 1842 deprecate teamster utils dbt package
TEAMSchools · Sep 27, 2024 · 2c08ba9 · 2c08ba9
2 parents 938ba77 + 267730b
commit 2c08ba9
Show file tree

Hide file tree

Showing 128 changed files with 2,472 additions and 1,921 deletions.
diff --git a/.devcontainer/tpl/.env.tpl b/.devcontainer/tpl/.env.tpl
@@ -1,3 +1,12 @@
+DBT_DEV_CLOUD_STORAGE_URI_BASE=gs://teamster-kippnewark/dagster/kippnewark
+DBT_DEV_DEANSLIST_SCHEMA=_dev_deanslist
+DBT_DEV_EDPLAN_SCHEMA=_dev_edplan
+DBT_DEV_IREADY_SCHEMA=_dev_iready
+DBT_DEV_OVERGRAD_SCHEMA=_dev_overgrad
+DBT_DEV_PEARSON_SCHEMA=_dev_pearson
+DBT_DEV_POWERSCHOOL_SCHEMA=_dev_powerschool
+DBT_DEV_RENLEARN_SCHEMA=_dev_renlearn
+DBT_DEV_TITAN_SCHEMA=_dev_titan
 ADP_SFTP_HOST_IP=op://Data Team/ADP Workforce Now SFTP/ip
 ADP_SFTP_PASSWORD=op://Data Team/ADP Workforce Now SFTP/password
 ADP_SFTP_USERNAME=op://Data Team/ADP Workforce Now SFTP/username

diff --git a/.github/workflows/deploy-prod-kippcamden.yaml b/.github/workflows/deploy-prod-kippcamden.yaml
@@ -16,7 +16,6 @@ on:
       - src/dbt/overgrad/**
       - src/dbt/pearson/**
       - src/dbt/powerschool/**
-      - src/dbt/teamster_utils/**
       - src/dbt/titan/**
       - src/teamster/code_locations/kippcamden/**
       - src/teamster/core/**

diff --git a/.github/workflows/deploy-prod-kippmiami.yaml b/.github/workflows/deploy-prod-kippmiami.yaml
@@ -15,7 +15,6 @@ on:
       - src/dbt/kippmiami/**
       - src/dbt/powerschool/**
       - src/dbt/renlearn/**
-      - src/dbt/teamster_utils/**
       - src/teamster/code_locations/kippmiami/**
       - src/teamster/core/**
       - src/teamster/libraries/couchdrop/**

diff --git a/.github/workflows/deploy-prod-kippnewark.yaml b/.github/workflows/deploy-prod-kippnewark.yaml
@@ -18,7 +18,6 @@ on:
       - src/dbt/pearson/**
       - src/dbt/powerschool/**
       - src/dbt/renlearn/**
-      - src/dbt/teamster_utils/**
       - src/dbt/titan/**
       - src/teamster/code_locations/kippnewark/**
       - src/teamster/core/**

diff --git a/.github/workflows/deploy-prod-kipptaf.yaml b/.github/workflows/deploy-prod-kipptaf.yaml
@@ -11,7 +11,6 @@ on:
       - Dockerfile
       - requirements.txt
       - src/dbt/kipptaf/**
-      - src/dbt/teamster_utils/**
       - src/teamster/code_locations/kipptaf/**
       - src/teamster/core/**
       - src/teamster/libraries/adp/**

diff --git a/scripts/fivetran-schema.py b/scripts/fivetran-schema.py
diff --git a/scripts/render_source_to_models.py b/scripts/render_source_to_models.py
diff --git a/scripts/scaffold_models.py b/scripts/scaffold_models.py
diff --git a/src/dbt/deanslist/dbt_project.yml b/src/dbt/deanslist/dbt_project.yml
@@ -27,8 +27,4 @@ clean-targets: # directories to be removed by `dbt clean`
 # Configuring models
 # Full documentation: https://docs.getdbt.com/docs/configuring-models
 models:
-  deanslist:
-    +materialized: table
-
-vars:
-  cloud_storage_uri_base: null
+  +schema: deanslist
diff --git a/src/dbt/deanslist/models/sources.yml b/src/dbt/deanslist/models/sources.yml
@@ -2,9 +2,8 @@ version: 2
 
 sources:
   - name: deanslist
-    schema: "{{ var('deanslist_schema', 'deanslist') }}"
-    database:
-      "{% if target.type != 'spark' %}{{ var('deanslist_database', target.database) }}{% endif %}"
+    schema: "{{ var('deanslist_schema', env_var('DBT_DEV_DEANSLIST_SCHEMA', 'deanslist')) }}"
+    database: "{{ var('deanslist_database', target.database) }}"
     tags:
       - stage_external_sources
     tables:
@@ -16,9 +15,13 @@ sources:
               - deanslist
               - behavior
         external:
-          location: "{{ var('cloud_storage_uri_base') }}/deanslist/behavior/*"
+          location:
+            "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', ''))
+            }}/deanslist/behavior/*"
           options:
-            hive_partition_uri_prefix: "{{ var('cloud_storage_uri_base') }}/deanslist/behavior/"
+            hive_partition_uri_prefix:
+              "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', ''))
+              }}/deanslist/behavior/"
             format: AVRO
             enable_logical_types: true
       - name: src_deanslist__comm_log
@@ -29,9 +32,13 @@ sources:
               - deanslist
               - comm_log
         external:
-          location: "{{ var('cloud_storage_uri_base') }}/deanslist/comm_log/*"
+          location:
+            "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', ''))
+            }}/deanslist/comm_log/*"
           options:
-            hive_partition_uri_prefix: "{{ var('cloud_storage_uri_base') }}/deanslist/comm_log/"
+            hive_partition_uri_prefix:
+              "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', ''))
+              }}/deanslist/comm_log/"
             format: AVRO
             enable_logical_types: true
       - name: src_deanslist__followups
@@ -42,9 +49,13 @@ sources:
               - deanslist
               - followups
         external:
-          location: "{{ var('cloud_storage_uri_base') }}/deanslist/followups/*"
+          location:
+            "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', ''))
+            }}/deanslist/followups/*"
           options:
-            hive_partition_uri_prefix: "{{ var('cloud_storage_uri_base') }}/deanslist/followups/"
+            hive_partition_uri_prefix:
+              "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', ''))
+              }}/deanslist/followups/"
             format: AVRO
             enable_logical_types: true
       - name: src_deanslist__homework
@@ -55,9 +66,13 @@ sources:
               - deanslist
               - homework
         external:
-          location: "{{ var('cloud_storage_uri_base') }}/deanslist/homework/*"
+          location:
+            "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', ''))
+            }}/deanslist/homework/*"
           options:
-            hive_partition_uri_prefix: "{{ var('cloud_storage_uri_base') }}/deanslist/homework/"
+            hive_partition_uri_prefix:
+              "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', ''))
+              }}/deanslist/homework/"
             format: AVRO
             enable_logical_types: true
       - name: src_deanslist__incidents
@@ -68,9 +83,13 @@ sources:
               - deanslist
               - incidents
         external:
-          location: "{{ var('cloud_storage_uri_base') }}/deanslist/incidents/*"
+          location:
+            "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', ''))
+            }}/deanslist/incidents/*"
           options:
-            hive_partition_uri_prefix: "{{ var('cloud_storage_uri_base') }}/deanslist/incidents/"
+            hive_partition_uri_prefix:
+              "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', ''))
+              }}/deanslist/incidents/"
             format: AVRO
             enable_logical_types: true
       - name: src_deanslist__lists
@@ -81,9 +100,13 @@ sources:
               - deanslist
               - lists
         external:
-          location: "{{ var('cloud_storage_uri_base') }}/deanslist/lists/*"
+          location:
+            "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', ''))
+            }}/deanslist/lists/*"
           options:
-            hive_partition_uri_prefix: "{{ var('cloud_storage_uri_base') }}/deanslist/lists/"
+            hive_partition_uri_prefix:
+              "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', ''))
+              }}/deanslist/lists/"
             format: AVRO
             enable_logical_types: true
       - name: src_deanslist__roster_assignments
@@ -95,11 +118,11 @@ sources:
               - roster_assignments
         external:
           location:
-            "gs://teamster-{{ project_name }}/dagster/{{ project_name
+            "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', ''))
             }}/deanslist/roster_assignments/*"
           options:
             hive_partition_uri_prefix:
-              "gs://teamster-{{ project_name }}/dagster/{{ project_name
+              "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', ''))
               }}/deanslist/roster_assignments/"
             format: AVRO
             enable_logical_types: true
@@ -111,9 +134,13 @@ sources:
               - deanslist
               - rosters
         external:
-          location: "{{ var('cloud_storage_uri_base') }}/deanslist/rosters/*"
+          location:
+            "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', ''))
+            }}/deanslist/rosters/*"
           options:
-            hive_partition_uri_prefix: "{{ var('cloud_storage_uri_base') }}/deanslist/rosters/"
+            hive_partition_uri_prefix:
+              "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', ''))
+              }}/deanslist/rosters/"
             format: AVRO
             enable_logical_types: true
       - name: src_deanslist__terms
@@ -124,9 +151,13 @@ sources:
               - deanslist
               - terms
         external:
-          location: "{{ var('cloud_storage_uri_base') }}/deanslist/terms/*"
+          location:
+            "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', ''))
+            }}/deanslist/terms/*"
           options:
-            hive_partition_uri_prefix: "{{ var('cloud_storage_uri_base') }}/deanslist/terms/"
+            hive_partition_uri_prefix:
+              "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', ''))
+              }}/deanslist/terms/"
             format: AVRO
             enable_logical_types: true
       - name: src_deanslist__users
@@ -137,9 +168,13 @@ sources:
               - deanslist
               - users
         external:
-          location: "{{ var('cloud_storage_uri_base') }}/deanslist/users/*"
+          location:
+            "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', ''))
+            }}/deanslist/users/*"
           options:
-            hive_partition_uri_prefix: "{{ var('cloud_storage_uri_base') }}/deanslist/users/"
+            hive_partition_uri_prefix:
+              "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', ''))
+              }}/deanslist/users/"
             format: AVRO
             enable_logical_types: true
       - name: src_deanslist__students
@@ -150,9 +185,13 @@ sources:
               - deanslist
               - students
         external:
-          location: "{{ var('cloud_storage_uri_base') }}/deanslist/students/*"
+          location:
+            "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', ''))
+            }}/deanslist/students/*"
           options:
-            hive_partition_uri_prefix: "{{ var('cloud_storage_uri_base') }}/deanslist/students/"
+            hive_partition_uri_prefix:
+              "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', ''))
+              }}/deanslist/students/"
             format: AVRO
             enable_logical_types: true
       - name: src_deanslist__dff_stats
@@ -163,8 +202,12 @@ sources:
               - deanslist
               - dff_stats
         external:
-          location: "{{ var('cloud_storage_uri_base') }}/deanslist/dff_stats/*"
+          location:
+            "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', ''))
+            }}/deanslist/dff_stats/*"
           options:
-            hive_partition_uri_prefix: "{{ var('cloud_storage_uri_base') }}/deanslist/dff_stats/"
+            hive_partition_uri_prefix:
+              "{{ var('cloud_storage_uri_base', env_var('DBT_DEV_CLOUD_STORAGE_URI_BASE', ''))
+              }}/deanslist/dff_stats/"
             format: AVRO
             enable_logical_types: true