diff --git a/.devcontainer/tpl/.env.tpl b/.devcontainer/tpl/.env.tpl index 7434c4525..2c13f1689 100644 --- a/.devcontainer/tpl/.env.tpl +++ b/.devcontainer/tpl/.env.tpl @@ -1,3 +1,4 @@ +DBT_DEV=true DBT_DEV_CLOUD_STORAGE_URI_BASE=gs://teamster-kippnewark/dagster/kippnewark DBT_DEV_DEANSLIST_SCHEMA=_dev_deanslist DBT_DEV_EDPLAN_SCHEMA=_dev_edplan diff --git a/.vscode/settings.json b/.vscode/settings.json index 7921b6f20..fdaccdd79 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -46,7 +46,7 @@ }, "dbt.deferConfigPerProject": { "src/dbt/kipptaf": { - "deferToProduction": true, + "deferToProduction": false, "favorState": false, "manifestPathForDeferral": "/workspaces/teamster/src/dbt/kipptaf/target", "manifestPathType": "local" diff --git a/src/dbt/iready/models/sources.yml b/src/dbt/iready/models/sources.yml index 386ab9639..8d1c13988 100644 --- a/src/dbt/iready/models/sources.yml +++ b/src/dbt/iready/models/sources.yml @@ -58,7 +58,7 @@ sources: }}/iready/instructional_usage_data/" format: AVRO enable_logical_types: true - - name: src_iready__personalized_instruction_by_lesson + - name: src_iready__instruction_by_lesson meta: dagster: parent_asset_key_path: @@ -75,7 +75,7 @@ sources: }}/iready/personalized_instruction_by_lesson/" format: AVRO enable_logical_types: true - - name: src_iready__instruction_by_lesson + - name: src_iready__instruction_by_lesson_pro meta: dagster: parent_asset_key_path: diff --git a/src/dbt/iready/models/staging/stg_iready__instruction_by_lesson.sql b/src/dbt/iready/models/staging/stg_iready__instruction_by_lesson.sql index 3d317aefe..7ce821ad0 100644 --- a/src/dbt/iready/models/staging/stg_iready__instruction_by_lesson.sql +++ b/src/dbt/iready/models/staging/stg_iready__instruction_by_lesson.sql @@ -1 +1,31 @@ -select *, from {{ source("iready", "src_iready__instruction_by_lesson") }} +select + _dagster_partition_academic_year as academic_year_int, + student_id, + academic_year, + school, + + `subject`, + domain, + lesson_grade, + lesson_level, + lesson_id, + lesson_name, + lesson_objective, + lesson_language, + passed_or_not_passed, + teacher_assigned_lesson, + + coalesce( + student_grade.string_value, cast(student_grade.long_value as string) + ) as student_grade, + + coalesce(cast(score.double_value as int), score.long_value) as score, + coalesce( + cast(total_time_on_lesson_min.double_value as int), + total_time_on_lesson_min.long_value + ) as total_time_on_lesson_min, + + parse_date('%m/%d/%Y', completion_date) as completion_date, + + if(passed_or_not_passed = 'Passed', 1.0, 0.0) as passed_or_not_passed_numeric, +from {{ source("iready", "src_iready__instruction_by_lesson") }} diff --git a/src/dbt/iready/models/staging/stg_iready__instruction_by_lesson_pro.sql b/src/dbt/iready/models/staging/stg_iready__instruction_by_lesson_pro.sql new file mode 100644 index 000000000..3e6161007 --- /dev/null +++ b/src/dbt/iready/models/staging/stg_iready__instruction_by_lesson_pro.sql @@ -0,0 +1,26 @@ +select + _dagster_partition_academic_year as academic_year_int, + student_id, + school, + + `subject`, + `level`, + topic, + lesson, + lesson_status, + lesson_result, + lesson_time_on_task_min, + lesson_language, + skills_completed, + skills_successful, + + parse_date('%m/%d/%Y', completion_date) as completion_date, + + coalesce( + student_grade.string_value, cast(student_grade.long_value as string) + ) as student_grade, + + coalesce( + percent_skills_successful.long_value, percent_skills_successful.double_value + ) as percent_skills_successful, +from {{ source("iready", "src_iready__instruction_by_lesson_pro") }} diff --git a/src/dbt/iready/models/staging/stg_iready__personalized_instruction_by_lesson.sql b/src/dbt/iready/models/staging/stg_iready__personalized_instruction_by_lesson.sql deleted file mode 100644 index 73df0507e..000000000 --- a/src/dbt/iready/models/staging/stg_iready__personalized_instruction_by_lesson.sql +++ /dev/null @@ -1,33 +0,0 @@ -{% set src_model = source( - "iready", "src_iready__personalized_instruction_by_lesson" -) %} - -select - {{ - dbt_utils.star( - from=src_model, - except=[ - "completion_date", - "score", - "student_grade", - "total_time_on_lesson_min", - ], - ) - }}, - - cast(left(academic_year, 4) as int) as academic_year_int, - - coalesce( - student_grade.string_value, cast(student_grade.long_value as string) - ) as student_grade, - - coalesce(safe_cast(score.double_value as int), score.long_value) as score, - coalesce( - safe_cast(total_time_on_lesson_min.double_value as int), - total_time_on_lesson_min.long_value - ) as total_time_on_lesson_min, - - parse_date('%m/%d/%Y', completion_date) as completion_date, - - if(passed_or_not_passed = 'Passed', 1.0, 0.0) as passed_or_not_passed_numeric, -from {{ src_model }} diff --git a/src/dbt/kipptaf/models/extracts/deanslist/rpt_deanslist__iready_lessons.sql b/src/dbt/kipptaf/models/extracts/deanslist/rpt_deanslist__iready_lessons.sql index 420973560..1202a5768 100644 --- a/src/dbt/kipptaf/models/extracts/deanslist/rpt_deanslist__iready_lessons.sql +++ b/src/dbt/kipptaf/models/extracts/deanslist/rpt_deanslist__iready_lessons.sql @@ -1,21 +1,25 @@ -select - pl.student_id, - pl.subject, +with + instruction_by_lesson as ( + select + pl.student_id, + pl.subject, - t.name as term, + t.name as term, - sum(pl.passed_or_not_passed_numeric) as lessons_passed, - count(pl.lesson_id) as total_lessons, + sum(pl.passed_or_not_passed_numeric) as lessons_passed, + count(pl.lesson_id) as total_lessons, + from {{ ref("stg_iready__instruction_by_lesson") }} as pl + inner join + {{ ref("stg_people__location_crosswalk") }} as sc on pl.school = sc.name + inner join + {{ ref("stg_reporting__terms") }} as t + on sc.powerschool_school_id = t.school_id + and pl.academic_year_int = t.academic_year + and pl.completion_date between t.start_date and t.end_date + and t.type = 'RT' + where pl.completion_date >= '{{ var("current_academic_year") }}-07-01' + group by pl.student_id, pl.subject, t.name + ) - round(sum(pl.passed_or_not_passed_numeric) / count(pl.lesson_id), 2) - * 100 as pct_passed, -from {{ ref("stg_iready__personalized_instruction_by_lesson") }} as pl -inner join {{ ref("stg_people__location_crosswalk") }} as sc on pl.school = sc.name -inner join - {{ ref("stg_reporting__terms") }} as t - on sc.powerschool_school_id = t.school_id - and pl.academic_year_int = t.academic_year - and pl.completion_date between t.start_date and t.end_date - and t.type = 'RT' -where pl.completion_date >= '{{ var("current_academic_year") }}-07-01' -group by pl.student_id, pl.subject, t.name +select *, round(lessons_passed / total_lessons, 2) * 100 as pct_passed, +from instruction_by_lesson diff --git a/src/dbt/kipptaf/models/extracts/tableau/intermediate/int_tableau__student_enrollments.sql b/src/dbt/kipptaf/models/extracts/tableau/intermediate/int_tableau__student_enrollments.sql index f4010cb91..f41c3925a 100644 --- a/src/dbt/kipptaf/models/extracts/tableau/intermediate/int_tableau__student_enrollments.sql +++ b/src/dbt/kipptaf/models/extracts/tableau/intermediate/int_tableau__student_enrollments.sql @@ -94,7 +94,6 @@ select when e.region = 'Miami' then 'FL' end as `state`, - from {{ ref("base_powerschool__student_enrollments") }} as e left join ms_grad_sub as m diff --git a/src/dbt/kipptaf/models/extracts/tableau/rpt_tableau__ddi_dashboard.sql b/src/dbt/kipptaf/models/extracts/tableau/rpt_tableau__ddi_dashboard.sql index 46d4eed02..d2839b0d9 100644 --- a/src/dbt/kipptaf/models/extracts/tableau/rpt_tableau__ddi_dashboard.sql +++ b/src/dbt/kipptaf/models/extracts/tableau/rpt_tableau__ddi_dashboard.sql @@ -3,19 +3,21 @@ with select il.academic_year_int, il.student_id, - lower(il.subject) as subject, cw.powerschool_school_id, pw.week_start_monday, + lower(il.subject) as `subject`, + if( sum(il.passed_or_not_passed_numeric) >= 2, 1, 0 ) as is_pass_2_lessons_int, + if( sum(il.passed_or_not_passed_numeric) >= 4, 1, 0 ) as is_pass_4_lessons_int, - from {{ ref("stg_iready__personalized_instruction_by_lesson") }} as il + from {{ ref("stg_iready__instruction_by_lesson") }} as il inner join {{ ref("stg_people__location_crosswalk") }} as cw on il.school = cw.name inner join @@ -44,7 +46,7 @@ with from iready_weekly pivot ( max(is_pass_2_lessons_int) as is_pass_2_lessons_int, - max(is_pass_4_lessons_int) as is_pass_4_lessons_int for subject + max(is_pass_4_lessons_int) as is_pass_4_lessons_int for `subject` in ('reading', 'math') ) ), @@ -101,6 +103,8 @@ with lc.head_of_school_preferred_name_lastfirst as head_of_school, + cast(r.assessment_id as string) as assessment_id, + if(co.lep_status, 'ML', 'Not ML') as ml_status, if(co.is_504, 'Has 504', 'No 504') as status_504, if( @@ -115,11 +119,11 @@ with false ) as is_current_week, - safe_cast(r.assessment_id as string) as assessment_id, + if(r.date_taken is not null, 1, 0) as is_complete, + case when r.is_mastery then 1 when not r.is_mastery then 0 end as is_mastery_int, - if(r.date_taken is not null, 1, 0) as is_complete, from {{ ref("base_powerschool__student_enrollments") }} as co inner join {{ ref("int_powerschool__calendar_week") }} as w @@ -162,46 +166,96 @@ with where co.enroll_status = 0 and co.academic_year >= {{ var("current_academic_year") - 1 }} - /* Manual filter to avoid dashboard roll-up #TODO: Remove SY26 */ + {# TODO: Remove SY26 #} + /* Manual filter to avoid dashboard roll-up */ and sc.module_type != 'WPP' ), + microgoals_long as ( + select + u.internal_id_int as employee_number, + + m.goal_name, + + date_add( + date(timestamp_trunc(timestamp(a.created), week)), interval 1 day + ) as week_start_monday, + from {{ ref("stg_schoolmint_grow__users") }} as u + left join + {{ ref("stg_schoolmint_grow__assignments") }} as a on u.user_id = a.user_id + left join + {{ ref("stg_schoolmint_grow__assignments__tags") }} as t + on a.assignment_id = t.assignment_id + left join + {{ ref("stg_schoolmint_grow__microgoals") }} as m + on t.tag_id = m.goal_tag_id + where m.goal_code is not null + ), + microgoals as ( select employee_number, week_start_monday, + string_agg(goal_name, ', ') as microgoals, - from - ( - select - u.internal_id_int as employee_number, - - m.goal_name, - - date_add( - date(timestamp_trunc(timestamp(a.created), week)), - interval 1 day - ) as week_start_monday, - from {{ ref("stg_schoolmint_grow__users") }} as u - left join - {{ ref("stg_schoolmint_grow__assignments") }} as a - on u.user_id = a.user_id - left join - {{ ref("stg_schoolmint_grow__assignments__tags") }} as t - on a.assignment_id = t.assignment_id - left join - {{ ref("stg_schoolmint_grow__microgoals") }} as m - on t.tag_id = m.goal_tag_id - where m.goal_code is not null - ) + from microgoals_long group by employee_number, week_start_monday ) /* ES/MS assessments */ select - co.*, - - qbl.qbl, + co.student_number, + co.student_name, + co.academic_year, + co.schoolid, + co.school, + co.region, + co.grade_level, + co.enroll_status, + co.cohort, + co.school_level, + co.gender, + co.ethnicity, + co.week_start_monday, + co.week_end_sunday, + co.days_in_session, + co.term, + co.title, + co.subject_area, + co.administered_at, + co.module_type, + co.module_code, + co.date_taken, + co.response_type, + co.response_type_code, + co.response_type_description, + co.response_type_root_description, + co.percent_correct, + co.is_mastery, + co.performance_band_label_number, + co.performance_band_label, + co.is_replacement, + co.standard_domain, + co.course_section, + co.course_name, + co.course_number, + co.course_credittype, + co.course_teachernumber, + co.course_teacher_name, + co.homeroom_section, + co.homeroom_teachernumber, + co.homeroom_teacher_name, + co.head_of_school, + co.ml_status, + co.status_504, + co.self_contained_status, + co.iep_status, + co.is_current_week, + co.assessment_id, + co.is_mastery_int, + co.is_complete, + + qbls.qbl, g.grade_goal, g.school_goal, @@ -211,7 +265,7 @@ select sf.nj_student_tier, sf.dibels_most_recent_composite, - if(qbl.qbl is not null, true, false) as is_qbl, + if(qbls.qbl is not null, true, false) as is_qbl, coalesce(ip.is_pass_2_lessons_int_reading, 0) as is_passed_iready_2plus_reading_int, coalesce(ip.is_pass_4_lessons_int_reading, 0) as is_passed_iready_4plus_reading_int, @@ -224,14 +278,14 @@ left join and co.academic_year = sf.academic_year and co.subject_area = sf.illuminate_subject_area left join - {{ ref("stg_assessments__qbls_power_standards") }} as qbl - on co.academic_year = qbl.academic_year - and co.term = qbl.term_name - and co.region = qbl.region - and co.grade_level = qbl.grade_level - and co.response_type_code = qbl.standard_code - and co.subject_area = qbl.illuminate_subject_area - and qbl.qbl is not null + {{ ref("stg_assessments__qbls_power_standards") }} as qbls + on co.academic_year = qbls.academic_year + and co.term = qbls.term_name + and co.region = qbls.region + and co.grade_level = qbls.grade_level + and co.response_type_code = qbls.standard_code + and co.subject_area = qbls.illuminate_subject_area + and qbls.qbl is not null left join {{ ref("int_assessments__academic_goals") }} as g on co.schoolid = g.school_id @@ -249,9 +303,58 @@ union all /* HS assessments */ select - co.*, - - qbl.qbl, + co.student_number, + co.student_name, + co.academic_year, + co.schoolid, + co.school, + co.region, + co.grade_level, + co.enroll_status, + co.cohort, + co.school_level, + co.gender, + co.ethnicity, + co.week_start_monday, + co.week_end_sunday, + co.days_in_session, + co.term, + co.title, + co.subject_area, + co.administered_at, + co.module_type, + co.module_code, + co.date_taken, + co.response_type, + co.response_type_code, + co.response_type_description, + co.response_type_root_description, + co.percent_correct, + co.is_mastery, + co.performance_band_label_number, + co.performance_band_label, + co.is_replacement, + co.standard_domain, + co.course_section, + co.course_name, + co.course_number, + co.course_credittype, + co.course_teachernumber, + co.course_teacher_name, + co.homeroom_section, + co.homeroom_teachernumber, + co.homeroom_teacher_name, + co.head_of_school, + co.ml_status, + co.status_504, + co.self_contained_status, + co.iep_status, + co.is_current_week, + co.assessment_id, + co.is_mastery_int, + co.is_complete, + + qbls.qbl, g.grade_goal, g.school_goal, @@ -259,9 +362,10 @@ select g.organization_goal, sf.nj_student_tier, + null as dibels_most_recent_composite, - if(qbl.qbl is not null, true, false) as is_qbl, + if(qbls.qbl is not null, true, false) as is_qbl, null as is_passed_iready_2plus_reading_int, null as is_passed_iready_4plus_reading_int, @@ -274,12 +378,12 @@ left join and co.academic_year = sf.academic_year and co.course_credittype = sf.powerschool_credittype left join - {{ ref("stg_assessments__qbls_power_standards") }} as qbl - on co.academic_year = qbl.academic_year - and co.term = qbl.term_name - and co.region = qbl.region - and co.response_type_code = qbl.standard_code - and co.subject_area = qbl.illuminate_subject_area + {{ ref("stg_assessments__qbls_power_standards") }} as qbls + on co.academic_year = qbls.academic_year + and co.term = qbls.term_name + and co.region = qbls.region + and co.response_type_code = qbls.standard_code + and co.subject_area = qbls.illuminate_subject_area left join {{ ref("int_assessments__academic_goals") }} as g on co.schoolid = g.school_id @@ -288,6 +392,7 @@ left join where co.grade_level between 9 and 12 union all + /* walkthrough data */ select null as student_number, @@ -325,20 +430,22 @@ select o.rubric_name as title, m.microgoals as subject_area, o.observed_at as administered_at, + null as module_type, null as module_code, - null as date_taken, 'walkthrough' as response_type, null as response_type_code, + o.measurement_name as response_type_description, o.strand_name as response_type_root_description, o.observation_score as percent_correct, + if(o.row_score = 1, true, false) as is_mastery, + null as performance_band_label_number, null as performance_band_label, null as is_replacement, - null as standard_domain, null as course_section, null as course_name, @@ -346,10 +453,12 @@ select null as course_credittype, r.powerschool_teacher_number as course_teachernumber, + t.lastfirst as course_teacher_name, null as homeroom_section, null as homeroom_teachernumber, + r.report_to_user_principal_name as homeroom_teacher_name, lc.head_of_school_preferred_name_lastfirst as head_of_school, @@ -368,19 +477,16 @@ select o.observation_id as assessment_id, o.row_score as is_mastery_int, - null as is_complete, + null as is_complete, null as qbl, null as grade_goal, null as school_goal, null as region_goal, null as organization_goal, - null as nj_student_tier, null as dibels_most_recent_composite, - null as is_qbl, - null as is_passed_iready_2plus_reading_int, null as is_passed_iready_4plus_reading_int, null as is_passed_iready_2plus_math_int, diff --git a/src/dbt/kipptaf/models/extracts/tableau/rpt_tableau__iready_apm.sql b/src/dbt/kipptaf/models/extracts/tableau/rpt_tableau__iready_apm.sql index 96d63c8f6..96e1a4a5a 100644 --- a/src/dbt/kipptaf/models/extracts/tableau/rpt_tableau__iready_apm.sql +++ b/src/dbt/kipptaf/models/extracts/tableau/rpt_tableau__iready_apm.sql @@ -2,8 +2,9 @@ with expanded_terms as ( select academic_year, - name, - start_date, + `name`, + `start_date`, + case region when 'KIPP Miami' @@ -13,10 +14,10 @@ with when 'KIPP Cooper Norcross Academy' then 'Camden' end as region, - -- end_date contains test window end date - -- this needs continuous dates + + /* end_date contains test window end date this needs continuous dates */ coalesce( - lead(start_date, 1) over ( + lead(`start_date`, 1) over ( partition by academic_year, region order by code asc ) - 1, @@ -44,7 +45,7 @@ select co.school, co.gifted_and_talented, - subj as subject, + subj as `subject`, w.week_start_monday, w.week_end_sunday, @@ -99,9 +100,11 @@ select true, false ) as is_curterm, + concat( left(co.student_first_name, 1), '. ', co.student_last_name ) as student_name_short, + dr.mid_on_grade_level_scale_score - dr.overall_scale_score as scale_pts_to_mid_on_grade_level, from {{ ref("int_tableau__student_enrollments") }} as co @@ -118,7 +121,7 @@ left join and co.region = rt.region and w.week_start_monday between rt.start_date and rt.end_date left join - {{ ref("stg_iready__personalized_instruction_by_lesson") }} as il + {{ ref("stg_iready__instruction_by_lesson") }} as il on co.student_number = il.student_id and subj = il.subject and il.completion_date between w.week_start_monday and w.week_end_sunday diff --git a/src/dbt/kipptaf/models/extracts/tableau/rpt_tableau__miami_fast.sql b/src/dbt/kipptaf/models/extracts/tableau/rpt_tableau__miami_fast.sql index fdffb4031..1d092e1e5 100644 --- a/src/dbt/kipptaf/models/extracts/tableau/rpt_tableau__miami_fast.sql +++ b/src/dbt/kipptaf/models/extracts/tableau/rpt_tableau__miami_fast.sql @@ -18,10 +18,11 @@ with iready_lessons as ( select student_id, - subject, + `subject`, + count(distinct lesson_id) as total_lessons, sum(passed_or_not_passed_numeric) as lessons_passed, - from {{ ref("stg_iready__personalized_instruction_by_lesson") }} + from {{ ref("stg_iready__instruction_by_lesson") }} where completion_date in ( select date_value, @@ -36,7 +37,7 @@ with ) ) as date_value ) - group by student_id, subject + group by student_id, `subject` ), pre_filter_qaf as ( @@ -55,28 +56,34 @@ with scale_crosswalk as ( select - 2022 as academic_year, administration_window, - 'FAST' as source_system, + academic_year, + + 'FAST_NEW' as source_system, 'FL' as destination_system, - from unnest(['PM1', 'PM2']) as administration_window + from + unnest( + generate_array(2023, {{ var("current_academic_year") }}) + ) as academic_year + cross join unnest(['PM1', 'PM2', 'PM3']) as administration_window + union all + select + administration_window, + 2022 as academic_year, - 'PM3' as administration_window, - 'FAST_NEW' as source_system, + 'FAST' as source_system, 'FL' as destination_system, + from unnest(['PM1', 'PM2']) as administration_window + union all + select - academic_year, - administration_window, + 'PM3' as administration_window, + 2022 as academic_year, 'FAST_NEW' as source_system, 'FL' as destination_system, - from - unnest( - generate_array(2023, {{ var("current_academic_year") }}) - ) as academic_year - cross join unnest(['PM1', 'PM2', 'PM3']) as administration_window ), prev_pm3 as ( @@ -90,6 +97,7 @@ with i.sublevel_number as prev_pm3_sublevel_number, f.academic_year + 1 as academic_year_next, + round( rank() over ( partition by diff --git a/src/dbt/kipptaf/models/iready/sources-kippmiami.yml b/src/dbt/kipptaf/models/iready/sources-kippmiami.yml index b730e9f95..169ef6fb1 100644 --- a/src/dbt/kipptaf/models/iready/sources-kippmiami.yml +++ b/src/dbt/kipptaf/models/iready/sources-kippmiami.yml @@ -2,6 +2,9 @@ version: 2 sources: - name: kippmiami_iready + schema: + "{% if env_var('DBT_DEV') == 'true' %}_dev_kippmiami_iready{% else %}kippmiami_iready{% endif + %}" tables: - name: stg_iready__diagnostic_results meta: @@ -19,19 +22,19 @@ sources: - kippmiami - iready - stg_iready__instructional_usage_data - - name: stg_iready__personalized_instruction_by_lesson + - name: stg_iready__instruction_by_lesson meta: dagster: group: iready asset_key: - kippmiami - iready - - stg_iready__personalized_instruction_by_lesson - - name: stg_iready__instruction_by_lesson + - stg_iready__instruction_by_lesson + - name: stg_iready__instruction_by_lesson_pro meta: dagster: group: iready asset_key: - kippmiami - iready - - stg_iready__instruction_by_lesson + - stg_iready__instruction_by_lesson_pro diff --git a/src/dbt/kipptaf/models/iready/sources-kippnj.yml b/src/dbt/kipptaf/models/iready/sources-kippnj.yml index fc02afc0e..986aa174d 100644 --- a/src/dbt/kipptaf/models/iready/sources-kippnj.yml +++ b/src/dbt/kipptaf/models/iready/sources-kippnj.yml @@ -2,7 +2,9 @@ version: 2 sources: - name: kippnj_iready - schema: kippnewark_iready + schema: + "{% if env_var('DBT_DEV') == 'true' %}_dev_kippnewark_iready{% else %}kippnewark_iready{% + endif %}" tables: - name: stg_iready__diagnostic_results meta: @@ -20,19 +22,19 @@ sources: - kippnewark - iready - stg_iready__instructional_usage_data - - name: stg_iready__personalized_instruction_by_lesson + - name: stg_iready__instruction_by_lesson meta: dagster: group: iready asset_key: - kippnewark - iready - - stg_iready__personalized_instruction_by_lesson - - name: stg_iready__instruction_by_lesson + - stg_iready__instruction_by_lesson + - name: stg_iready__instruction_by_lesson_pro meta: dagster: group: iready asset_key: - kippnewark - iready - - stg_iready__instruction_by_lesson + - stg_iready__instruction_by_lesson_pro diff --git a/src/dbt/kipptaf/models/iready/staging/stg_iready__personalized_instruction_by_lesson.sql b/src/dbt/kipptaf/models/iready/staging/stg_iready__instruction_by_lesson_pro.sql similarity index 100% rename from src/dbt/kipptaf/models/iready/staging/stg_iready__personalized_instruction_by_lesson.sql rename to src/dbt/kipptaf/models/iready/staging/stg_iready__instruction_by_lesson_pro.sql diff --git a/src/teamster/code_locations/kippmiami/iready/assets.py b/src/teamster/code_locations/kippmiami/iready/assets.py index 484ded446..79a6a3101 100644 --- a/src/teamster/code_locations/kippmiami/iready/assets.py +++ b/src/teamster/code_locations/kippmiami/iready/assets.py @@ -20,29 +20,29 @@ current_fiscal_year=CURRENT_FISCAL_YEAR.fiscal_year, ) -personalized_instruction_by_lesson = build_iready_sftp_asset( - asset_key=[*key_prefix, "personalized_instruction_by_lesson"], +instructional_usage_data = build_iready_sftp_asset( + asset_key=[*key_prefix, "instructional_usage_data"], region_subfolder=region_subfolder, remote_file_regex=( - r"iready_instruction_by_lesson_(?Pela|math)(_CONFIDENTIAL)?\.csv" + r"instructional_usage_data_(?Pela|math)(_CONFIDENTIAL)?\.csv" ), - avro_schema=PERSONALIZED_INSTRUCTION_BY_LESSON_SCHEMA, + avro_schema=INSTRUCTIONAL_USAGE_DATA_SCHEMA, start_fiscal_year=2023, current_fiscal_year=CURRENT_FISCAL_YEAR.fiscal_year, ) -instructional_usage_data = build_iready_sftp_asset( - asset_key=[*key_prefix, "instructional_usage_data"], +instruction_by_lesson = build_iready_sftp_asset( + asset_key=[*key_prefix, "personalized_instruction_by_lesson"], region_subfolder=region_subfolder, remote_file_regex=( - r"instructional_usage_data_(?Pela|math)(_CONFIDENTIAL)?\.csv" + r"iready_instruction_by_lesson_(?Pela|math)(_CONFIDENTIAL)?\.csv" ), - avro_schema=INSTRUCTIONAL_USAGE_DATA_SCHEMA, + avro_schema=PERSONALIZED_INSTRUCTION_BY_LESSON_SCHEMA, start_fiscal_year=2023, current_fiscal_year=CURRENT_FISCAL_YEAR.fiscal_year, ) -instruction_by_lesson = build_iready_sftp_asset( +instruction_by_lesson_pro = build_iready_sftp_asset( asset_key=[*key_prefix, "instruction_by_lesson"], region_subfolder=region_subfolder, remote_file_regex=( @@ -67,6 +67,6 @@ assets = [ diagnostic_results, instruction_by_lesson, + instruction_by_lesson_pro, instructional_usage_data, - personalized_instruction_by_lesson, ] diff --git a/src/teamster/code_locations/kippnewark/iready/assets.py b/src/teamster/code_locations/kippnewark/iready/assets.py index 89ac9ce20..948fbdfb2 100644 --- a/src/teamster/code_locations/kippnewark/iready/assets.py +++ b/src/teamster/code_locations/kippnewark/iready/assets.py @@ -20,7 +20,19 @@ current_fiscal_year=CURRENT_FISCAL_YEAR.fiscal_year, ) -personalized_instruction_by_lesson = build_iready_sftp_asset( + +instructional_usage_data = build_iready_sftp_asset( + asset_key=[*key_prefix, "instructional_usage_data"], + region_subfolder=region_subfolder, + remote_file_regex=( + r"instructional_usage_data_(?Pela|math)(_CONFIDENTIAL)?\.csv" + ), + avro_schema=INSTRUCTIONAL_USAGE_DATA_SCHEMA, + start_fiscal_year=2023, + current_fiscal_year=CURRENT_FISCAL_YEAR.fiscal_year, +) + +instruction_by_lesson = build_iready_sftp_asset( asset_key=[*key_prefix, "personalized_instruction_by_lesson"], region_subfolder=region_subfolder, remote_file_regex=( @@ -38,18 +50,7 @@ }, ) -instructional_usage_data = build_iready_sftp_asset( - asset_key=[*key_prefix, "instructional_usage_data"], - region_subfolder=region_subfolder, - remote_file_regex=( - r"instructional_usage_data_(?Pela|math)(_CONFIDENTIAL)?\.csv" - ), - avro_schema=INSTRUCTIONAL_USAGE_DATA_SCHEMA, - start_fiscal_year=2023, - current_fiscal_year=CURRENT_FISCAL_YEAR.fiscal_year, -) - -instruction_by_lesson = build_iready_sftp_asset( +instruction_by_lesson_pro = build_iready_sftp_asset( asset_key=[*key_prefix, "instruction_by_lesson"], region_subfolder=region_subfolder, remote_file_regex=( @@ -74,6 +75,6 @@ assets = [ diagnostic_results, instruction_by_lesson, + instruction_by_lesson_pro, instructional_usage_data, - personalized_instruction_by_lesson, ]