Merge pull request #1957 from TEAMSchools/1948-staging-table-for-i-re…

…ady-pro-lessons-needed

refactor: iready instruction by lesson, with pro

.devcontainer/tpl/.env.tpl

@@ -1,3 +1,4 @@
+DBT_DEV=true
 DBT_DEV_CLOUD_STORAGE_URI_BASE=gs://teamster-kippnewark/dagster/kippnewark
 DBT_DEV_DEANSLIST_SCHEMA=_dev_deanslist
 DBT_DEV_EDPLAN_SCHEMA=_dev_edplan

.vscode/settings.json

@@ -46,7 +46,7 @@
   },
   "dbt.deferConfigPerProject": {
     "src/dbt/kipptaf": {
-      "deferToProduction": true,
+      "deferToProduction": false,
       "favorState": false,
       "manifestPathForDeferral": "/workspaces/teamster/src/dbt/kipptaf/target",
       "manifestPathType": "local"

src/dbt/iready/models/sources.yml

@@ -58,7 +58,7 @@ sources:
               }}/iready/instructional_usage_data/"
             format: AVRO
             enable_logical_types: true
-      - name: src_iready__personalized_instruction_by_lesson
+      - name: src_iready__instruction_by_lesson
         meta:
           dagster:
             parent_asset_key_path:
@@ -75,7 +75,7 @@ sources:
               }}/iready/personalized_instruction_by_lesson/"
             format: AVRO
             enable_logical_types: true
-      - name: src_iready__instruction_by_lesson
+      - name: src_iready__instruction_by_lesson_pro
         meta:
           dagster:
             parent_asset_key_path:

src/dbt/iready/models/staging/stg_iready__instruction_by_lesson.sql

@@ -1 +1,31 @@
-select *, from {{ source("iready", "src_iready__instruction_by_lesson") }}
+select
+    _dagster_partition_academic_year as academic_year_int,
+    student_id,
+    academic_year,
+    school,
+
+    `subject`,
+    domain,
+    lesson_grade,
+    lesson_level,
+    lesson_id,
+    lesson_name,
+    lesson_objective,
+    lesson_language,
+    passed_or_not_passed,
+    teacher_assigned_lesson,
+
+    coalesce(
+        student_grade.string_value, cast(student_grade.long_value as string)
+    ) as student_grade,
+
+    coalesce(cast(score.double_value as int), score.long_value) as score,
+    coalesce(
+        cast(total_time_on_lesson_min.double_value as int),
+        total_time_on_lesson_min.long_value
+    ) as total_time_on_lesson_min,
+
+    parse_date('%m/%d/%Y', completion_date) as completion_date,
+
+    if(passed_or_not_passed = 'Passed', 1.0, 0.0) as passed_or_not_passed_numeric,
+from {{ source("iready", "src_iready__instruction_by_lesson") }}

src/dbt/iready/models/staging/stg_iready__instruction_by_lesson_pro.sql

@@ -0,0 +1,26 @@
+select
+    _dagster_partition_academic_year as academic_year_int,
+    student_id,
+    school,
+
+    `subject`,
+    `level`,
+    topic,
+    lesson,
+    lesson_status,
+    lesson_result,
+    lesson_time_on_task_min,
+    lesson_language,
+    skills_completed,
+    skills_successful,
+
+    parse_date('%m/%d/%Y', completion_date) as completion_date,
+
+    coalesce(
+        student_grade.string_value, cast(student_grade.long_value as string)
+    ) as student_grade,
+
+    coalesce(
+        percent_skills_successful.long_value, percent_skills_successful.double_value
+    ) as percent_skills_successful,
+from {{ source("iready", "src_iready__instruction_by_lesson_pro") }}

src/dbt/kipptaf/models/extracts/deanslist/rpt_deanslist__iready_lessons.sql

@@ -1,21 +1,25 @@
-select
-    pl.student_id,
-    pl.subject,
+with
+    instruction_by_lesson as (
+        select
+            pl.student_id,
+            pl.subject,
 
-    t.name as term,
+            t.name as term,
 
-    sum(pl.passed_or_not_passed_numeric) as lessons_passed,
-    count(pl.lesson_id) as total_lessons,
+            sum(pl.passed_or_not_passed_numeric) as lessons_passed,
+            count(pl.lesson_id) as total_lessons,
+        from {{ ref("stg_iready__instruction_by_lesson") }} as pl
+        inner join
+            {{ ref("stg_people__location_crosswalk") }} as sc on pl.school = sc.name
+        inner join
+            {{ ref("stg_reporting__terms") }} as t
+            on sc.powerschool_school_id = t.school_id
+            and pl.academic_year_int = t.academic_year
+            and pl.completion_date between t.start_date and t.end_date
+            and t.type = 'RT'
+        where pl.completion_date >= '{{ var("current_academic_year") }}-07-01'
+        group by pl.student_id, pl.subject, t.name
+    )
 
-    round(sum(pl.passed_or_not_passed_numeric) / count(pl.lesson_id), 2)
-    * 100 as pct_passed,
-from {{ ref("stg_iready__personalized_instruction_by_lesson") }} as pl
-inner join {{ ref("stg_people__location_crosswalk") }} as sc on pl.school = sc.name
-inner join
-    {{ ref("stg_reporting__terms") }} as t
-    on sc.powerschool_school_id = t.school_id
-    and pl.academic_year_int = t.academic_year
-    and pl.completion_date between t.start_date and t.end_date
-    and t.type = 'RT'
-where pl.completion_date >= '{{ var("current_academic_year") }}-07-01'
-group by pl.student_id, pl.subject, t.name
+select *, round(lessons_passed / total_lessons, 2) * 100 as pct_passed,
+from instruction_by_lesson

src/dbt/kipptaf/models/extracts/tableau/intermediate/int_tableau__student_enrollments.sql

@@ -94,7 +94,6 @@ select
         when e.region = 'Miami'
         then 'FL'
     end as `state`,
-
 from {{ ref("base_powerschool__student_enrollments") }} as e
 left join
     ms_grad_sub as m