diff --git a/_samples/dbt_cloud_api_v2/trigger_job.py b/_samples/dbt_cloud_api_v2/trigger_job.py new file mode 100644 index 0000000..754a26e --- /dev/null +++ b/_samples/dbt_cloud_api_v2/trigger_job.py @@ -0,0 +1,25 @@ +# Note: the easiest way to run this is through this Hex app because it has the token stored as a secret +# https://app.hex.tech/fishtown/hex/be772c9d-fce1-4994-9bb9-423429318943/draft/logic + +import requests + +account_id = 26712 +job_id = 38569 +token = 'XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX' + +headers = { + 'Authorization': f'Token {token}', + 'Content-Type': 'application/json' +} + +json_data = { + 'cause': 'Triggered from a Python Script' +} + +response = requests.post( + f'https://cloud.getdbt.com/api/v2/accounts/{account_id}/jobs/{job_id}/run', + headers=headers, + json=json_data +) + +print(response) \ No newline at end of file diff --git a/_samples/node_selection_syntax/model_selection_syntax.md b/_samples/node_selection_syntax/model_selection_syntax.md new file mode 100644 index 0000000..52c9599 --- /dev/null +++ b/_samples/node_selection_syntax/model_selection_syntax.md @@ -0,0 +1,73 @@ +## Some examples of helpful syntax for building models + +1. Build dim_customers and all children + +``` +dbt build --select dim_customers+ +``` + +2. Build dim_customers and all parents + +``` +dbt build --select +dim_customers +``` + +3. Build dim_customers and all parents and children + +``` +dbt build --select +dim_customers+ +``` + +4. Build dim_customers, it's first degree parents, and it's first degree children + +``` +dbt build --select 1+dim_customers+1 +``` + +5. Build dim_customers, it's children, and all of it's children's parents + +``` +dbt build --select @dim_customers +``` + +6. Build all models in the dbt_artifacts package + +``` +dbt build --select dbt_artifacts +``` + +7. Build all models in the marts/finance directory + +``` +dbt build --select marts.finance +``` + +8. Build all models in the marts/finance directory except fct_orders and it's children + +``` +dbt build --select marts.finance --exclude fct_orders+ +``` + +9. Build all incremental models + +``` +dbt build --select config.materialized.incremental +``` + +10. Build all models (and seeds and snapshots) in the selector all_models_and_tagged_snapshots + +``` +dbt build --selector all_models_and_tagged_snapshots +``` + +11. Build version 2 (latest version) of a model called example_private_finance_model which has an alias of example_private_finance_model + +``` +dbt build --select example_private_finance_model +``` + +12. Build version 1 of a model called example_private_finance_model + +``` +dbt build --select example_private_finance_model.v1 +``` \ No newline at end of file diff --git a/_samples/node_selection_syntax/seed_selection_syntax.md b/_samples/node_selection_syntax/seed_selection_syntax.md new file mode 100644 index 0000000..2ef18ee --- /dev/null +++ b/_samples/node_selection_syntax/seed_selection_syntax.md @@ -0,0 +1,20 @@ +## Some examples of helpful syntax for using seeds + +1. Create all seeds in your warehouse without testing any + +``` +dbt seed +``` + +2. Build a seed named alphabet_grouping in your warehouse including testing it + +``` +dbt build --select alphabet_grouping +``` + +3. Full refresh your seed so that dbt will drop and recreate the object instead of truncating and inserting. +This is useful when you add or remove columns from a seed + +``` +dbt build --select alphabet_grouping --full-refresh +``` \ No newline at end of file diff --git a/_samples/node_selection_syntax/source_selection_syntax.md b/_samples/node_selection_syntax/source_selection_syntax.md new file mode 100644 index 0000000..e03a4e7 --- /dev/null +++ b/_samples/node_selection_syntax/source_selection_syntax.md @@ -0,0 +1,54 @@ +## Some examples of helpful syntax for using sources + +### Testing Sources + +1. Test all of your sources + +``` +dbt test --select source:* +``` + +2. Test all tables in your source named jaffle_shop + +``` +dbt test --select source:jaffle_shop +``` + +3. Test a specific source table named orders within your jaffle_shop source + +``` +dbt test --select source:jaffle_shop.orders +``` + +### Source Freshness +1. Run source freshness on all of your sources + +``` +dbt source freshness --select source:* +``` + +2. Run source freshness on all tables in your source named jaffle_shop + +``` +dbt source freshness --select source:jaffle_shop +``` + +3. Run source freshness on a specific source table named orders within your jaffle_shop source + +``` +dbt source freshness --select source:jaffle_shop.orders +``` + +### Building (running and testing) downstream sources + +1. Build (run and test) all models downstream of a source named jaffle_shop + +``` +dbt build --select source:jaffle_shop+ +``` + +2. Build (run and test) all models downstream specific source table named orders within your jaffle_shop source + +``` +dbt build --select source:jaffle_shop.orders+ +``` \ No newline at end of file diff --git a/_samples/node_selection_syntax/tag_selection_syntax.md b/_samples/node_selection_syntax/tag_selection_syntax.md new file mode 100644 index 0000000..6124bd4 --- /dev/null +++ b/_samples/node_selection_syntax/tag_selection_syntax.md @@ -0,0 +1,25 @@ +## Some examples of helpful syntax for using tags + +1. Build all models that have the hourly tag + +``` +dbt build --select tag:hourly +``` + +2. Build all models and their downstream dependencies that have the hourly tag + +``` +dbt build --select tag:hourly+ +``` + +3. Build all models that have both the finance tag AND the orders tag + +``` +dbt build --select tag:hourly,tag:orders +``` + +4. Build all models that have both the finance tag OR the orders tag + +``` +dbt build --select tag:hourly tag:orders +``` diff --git a/_samples/node_selection_syntax/test_selection_syntax.md b/_samples/node_selection_syntax/test_selection_syntax.md new file mode 100644 index 0000000..cd48c8b --- /dev/null +++ b/_samples/node_selection_syntax/test_selection_syntax.md @@ -0,0 +1,19 @@ +## Some examples of helpful syntax for testing specific tests + +1. Test all generic tests + +``` +dbt test --select test_type:generic +``` + +2. Test all tests named unique + +``` +dbt test --select test_name:unique +``` + +3. Test a source named jaffle_shop + +``` +dbt test --select source:jaffle_shop +``` \ No newline at end of file diff --git a/_samples/tags/tag_selection_syntax.md b/_samples/tags/tag_selection_syntax.md index 441ab35..6124bd4 100644 --- a/_samples/tags/tag_selection_syntax.md +++ b/_samples/tags/tag_selection_syntax.md @@ -1,7 +1,5 @@ ## Some examples of helpful syntax for using tags -### Testing Sources - 1. Build all models that have the hourly tag ``` diff --git a/models/marts/marketing/dim_customers_legacy.sql b/analysis/legacy_tpch/dim_customers_legacy.sql similarity index 100% rename from models/marts/marketing/dim_customers_legacy.sql rename to analysis/legacy_tpch/dim_customers_legacy.sql diff --git a/dbt_project.yml b/dbt_project.yml index 68dce95..0801a25 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -27,9 +27,12 @@ clean-targets: # directories to be removed by `dbt clean` vars: example_variable: foo example_target_snapshot_schema: "{{ target.schema if target.name == 'dev' else 'prod_snapshot' }}" + incremental_lookback_value: 3 + incremental_lookback_period: 'hour' + future_proof_date: '9999-12-31' -# on-run-end: - # - "{% if target.name == 'prod' %}{{ dbt_artifacts.upload_results(results) }}{% endif %}" +on-run-end: + - "{% if target.name == 'prod' %}{{ dbt_artifacts.upload_results(results) }}{% endif %}" # Configuring models @@ -39,11 +42,17 @@ models: marts: +materialized: table +tags: ['tag'] + finance: + +group: finance + marketing: + +group: marketing + operations: + +group: operations intermediate: +materialized: view staging: +materialized: view - examples: + _samples: +materialized: table staging: jaffle_shop: @@ -57,7 +66,6 @@ models: tests: rapid_onboarding_exemplar: - examples: + _samples: staging: +store_failures: "{{ true if target.name == 'prod' else false }}" - diff --git a/macros/examples/audit_helper/audit_dim_customers.sql b/macros/_samples/audit_helper/audit_dim_customers.sql similarity index 100% rename from macros/examples/audit_helper/audit_dim_customers.sql rename to macros/_samples/audit_helper/audit_dim_customers.sql diff --git a/models/_samples/incremental/example_incremental_model.sql b/models/_samples/incremental/example_incremental_model.sql new file mode 100644 index 0000000..cf8cb7b --- /dev/null +++ b/models/_samples/incremental/example_incremental_model.sql @@ -0,0 +1,17 @@ +{{ + config( + materialized='incremental', + on_schema_change='sync_all_columns' + ) +}} + +with source as ( + select * from {{ ref('example_source_for_incremental') }} + {% if is_incremental() %} + -- this filter will only be applied on an incremental run + where _etl_loaded_at > (select max(_etl_loaded_at) from {{ this }}) + {% endif %} +) + +select * +from source \ No newline at end of file diff --git a/models/_samples/incremental/example_incremental_model_no_timestamp.sql b/models/_samples/incremental/example_incremental_model_no_timestamp.sql new file mode 100644 index 0000000..9e0bdf7 --- /dev/null +++ b/models/_samples/incremental/example_incremental_model_no_timestamp.sql @@ -0,0 +1,17 @@ +{{ + config( + materialized='incremental', + on_schema_change='sync_all_columns' + ) +}} + +with source as ( + select * from {{ ref('example_source_for_incremental') }} + {% if is_incremental() %} + -- this filter will only be applied on an incremental run + where event_id not in (select event_id from {{ this }}) + {% endif %} +) + +select * +from source \ No newline at end of file diff --git a/models/_samples/incremental/example_incremental_model_relative_start_and_end_dates.sql b/models/_samples/incremental/example_incremental_model_relative_start_and_end_dates.sql new file mode 100644 index 0000000..a108873 --- /dev/null +++ b/models/_samples/incremental/example_incremental_model_relative_start_and_end_dates.sql @@ -0,0 +1,24 @@ +{{ + config( + materialized='incremental', + unique_key='event_id', + on_schema_change='sync_all_columns' + ) +}} + + + +with source as ( + select * from {{ ref('example_source_for_incremental') }} + {% if is_incremental() %} + -- this filter will only be applied on an incremental run + where _etl_loaded_at >= dateadd(day, -2, current_date) + and _etl_loaded_at <= dateadd(day, -1, current_date) + + {% endif %} + + +) + +select * +from source \ No newline at end of file diff --git a/models/_samples/incremental/example_incremental_model_using_project_variable_lookback_window.sql b/models/_samples/incremental/example_incremental_model_using_project_variable_lookback_window.sql new file mode 100644 index 0000000..532c878 --- /dev/null +++ b/models/_samples/incremental/example_incremental_model_using_project_variable_lookback_window.sql @@ -0,0 +1,30 @@ +{{ + config( + materialized='incremental', + unique_key='event_id', + on_schema_change='sync_all_columns' + ) +}} + +with source as ( + select * from {{ ref('example_source_for_incremental') }} + {% if is_incremental() %} + -- this filter will only be applied on an incremental run + where _etl_loaded_at > ( + + select {{ dbt.dateadd( + var('incremental_lookback_period'), + -var('incremental_lookback_value'), + "max(_etl_loaded_at)") }} + + from {{ this }} + ) + + + {% endif %} + + +) + +select * +from source \ No newline at end of file diff --git a/models/_samples/incremental/example_incremental_model_with_unique_key.sql b/models/_samples/incremental/example_incremental_model_with_unique_key.sql new file mode 100644 index 0000000..c359fac --- /dev/null +++ b/models/_samples/incremental/example_incremental_model_with_unique_key.sql @@ -0,0 +1,20 @@ +{{ + config( + materialized='incremental', + unique_key='event_id', + on_schema_change='sync_all_columns' + ) +}} + +with source as ( + select * from {{ ref('example_source_for_incremental') }} + {% if is_incremental() %} + -- this filter will only be applied on an incremental run + where _etl_loaded_at > (select {{ dbt.dateadd("hour", -3, "max(_etl_loaded_at)") }} from {{ this }}) + + + {% endif %} +) + +select * +from source \ No newline at end of file diff --git a/models/_samples/incremental/example_source_for_incremental.sql b/models/_samples/incremental/example_source_for_incremental.sql new file mode 100644 index 0000000..1cbe674 --- /dev/null +++ b/models/_samples/incremental/example_source_for_incremental.sql @@ -0,0 +1,95 @@ + +{{ + config( + materialized='view' + ) +}} + +{% set country_options=['US', 'Mexico', 'Canada', 'Germany', 'England', 'France', 'Ireland'] %} +{% set page_url_options=['https://www.getdbt.com/', 'https://docs.getdbt.com/', 'https://docs.getdbt.com/quickstarts'] %} +{% set page_url_title_options=['dbt Labs | Transform Data in Your Warehouse', 'What is dbt? | dbt Developer Hub', 'Quickstarts | dbt Developer Hub'] %} + +with series as ( + +{{ dbt_utils.generate_series(3000000) }} + +), + +seconds_this_month as ( + +select + generated_number as event_id, + + -- events occur every second in the month + {{ dbt.dateadd("second", "event_id", dbt.date_trunc('month', 'current_timestamp')) }} as event_timestamp + + +from series + +where event_timestamp <= current_timestamp + +), + +final as ( + + select + event_id, + round(1 + date_part('second', event_timestamp)::int / 10.0) as user_id, + + case + when date_part('second', event_timestamp)::int % 3 < 2 + then 'page_view' + else 'page_ping' + end as event, + + event_timestamp, + + case + when date_part('second', event_timestamp)::int % 3 = 0 + then '{{ page_url_options[0] }}' + when date_part('second', event_timestamp)::int % 3 = 1 + then '{{ page_url_options[1] }}' + else '{{ page_url_options[2] }}' + end as page_url, + + case + when date_part('second', event_timestamp)::int % 3 = 0 + then '{{ page_url_title_options[0] }}' + when date_part('second', event_timestamp)::int % 3 = 1 + then '{{ page_url_title_options[1] }}' + else '{{ page_url_title_options[2] }}' + end as page_title, + + case + when round(1 + date_part('second', event_timestamp)::int / 10.0) % 7 < 4 + then 'computer' + else 'mobile' + end as device_type, + + case + when round(1 + date_part('second', event_timestamp)::int / 10.0)::int = 1 + then '{{ country_options[0] }}' + when round(1 + date_part('second', event_timestamp)::int / 10.0)::int = 2 + then '{{ country_options[1] }}' + when round(1 + date_part('second', event_timestamp)::int / 10.0)::int = 3 + then '{{ country_options[2] }}' + when round(1 + date_part('second', event_timestamp)::int / 10.0)::int = 4 + then '{{ country_options[3] }}' + when round(1 + date_part('second', event_timestamp)::int / 10.0)::int = 5 + then '{{ country_options[4] }}' + when round(1 + date_part('second', event_timestamp)::int / 10.0)::int = 6 + then '{{ country_options[5] }}' + else '{{ country_options[6] }}' + end as geo, + + -- rounded event timestamp to mimic data being loaded every 5 seconds + {{ dbt.dateadd("second", "- ((event_id % 60) % 5)", "event_timestamp" ) }} as _etl_loaded_at + + + + from seconds_this_month + +) + +select * +from final diff --git a/models/examples/marts/dim_jaffle_shop_customers.sql b/models/_samples/marts/dim_jaffle_shop_customers.sql similarity index 100% rename from models/examples/marts/dim_jaffle_shop_customers.sql rename to models/_samples/marts/dim_jaffle_shop_customers.sql diff --git a/models/_samples/model_governance/_model_governance__models.yml b/models/_samples/model_governance/_model_governance__models.yml new file mode 100644 index 0000000..6c83505 --- /dev/null +++ b/models/_samples/model_governance/_model_governance__models.yml @@ -0,0 +1,59 @@ +version: 2 + +models: + - name: example_private_finance_model + access: private + group: finance + latest_version: 2 + config: + contract: + enforced: true + columns: + - name: order_id + data_type: int + constraints: + # not null constraint is enforced in snowflake + - type: not_null + # unique constraint is NOT enforced in snowflake + - type: unique + warn_unenforced: false + tests: + - unique + - name: order_date + data_type: date + - name: customer_id + data_type: int + - name: order_status_code + data_type: varchar + - name: priority_code + data_type: varchar + - name: clerk_name + data_type: varchar + - name: ship_priority + data_type: int + - name: order_count + data_type: int + - name: gross_item_sales_amount + data_type: number + - name: item_discount_amount + data_type: number + - name: item_tax_amount + data_type: number + - name: net_item_sales_amount + data_type: number + versions: + - v: 2 + config: + alias: example_private_finance_model + columns: + - include: all + exclude: + - priority_code + - v: 1 + + - name: example_selecting_from_private_model + # change this to marketing if you want to see the error from not having access to the private finance model + group: finance + + - name: example_selecting_from_old_version_of_private_model + group: finance \ No newline at end of file diff --git a/models/_samples/model_governance/example_private_finance_model_v1.sql b/models/_samples/model_governance/example_private_finance_model_v1.sql new file mode 100644 index 0000000..39a619b --- /dev/null +++ b/models/_samples/model_governance/example_private_finance_model_v1.sql @@ -0,0 +1,2 @@ +select * +from {{ ref('fct_orders') }} diff --git a/models/_samples/model_governance/example_private_finance_model_v2.sql b/models/_samples/model_governance/example_private_finance_model_v2.sql new file mode 100644 index 0000000..e069595 --- /dev/null +++ b/models/_samples/model_governance/example_private_finance_model_v2.sql @@ -0,0 +1,3 @@ +select + * exclude priority_code +from {{ ref('fct_orders') }} diff --git a/models/_samples/model_governance/example_selecting_from_old_version_of_private_model.sql b/models/_samples/model_governance/example_selecting_from_old_version_of_private_model.sql new file mode 100644 index 0000000..e9f49ce --- /dev/null +++ b/models/_samples/model_governance/example_selecting_from_old_version_of_private_model.sql @@ -0,0 +1,7 @@ + +select * + +-- specifying old version of model +from {{ ref('example_private_finance_model', v=1) }} + + diff --git a/models/_samples/model_governance/example_selecting_from_private_model.sql b/models/_samples/model_governance/example_selecting_from_private_model.sql new file mode 100644 index 0000000..6f6d84d --- /dev/null +++ b/models/_samples/model_governance/example_selecting_from_private_model.sql @@ -0,0 +1,7 @@ + +select * + +-- defaults to the latest version because didnt specify the version in the ref +from {{ ref('example_private_finance_model') }} + + diff --git a/models/examples/python/_config.yml b/models/_samples/python/_python__models.yml similarity index 100% rename from models/examples/python/_config.yml rename to models/_samples/python/_python__models.yml diff --git a/models/examples/python/py01__python_builtins__describe.py b/models/_samples/python/py01__python_builtins__describe.py similarity index 100% rename from models/examples/python/py01__python_builtins__describe.py rename to models/_samples/python/py01__python_builtins__describe.py diff --git a/models/examples/python/py02__use_variables__customers_limit_10.py b/models/_samples/python/py02__use_variables__customers_limit_10.py similarity index 100% rename from models/examples/python/py02__use_variables__customers_limit_10.py rename to models/_samples/python/py02__use_variables__customers_limit_10.py diff --git a/models/examples/python/py03__define_function__payment_glitch.py b/models/_samples/python/py03__define_function__payment_glitch.py similarity index 100% rename from models/examples/python/py03__define_function__payment_glitch.py rename to models/_samples/python/py03__define_function__payment_glitch.py diff --git a/models/examples/python/py04__add_udf_function__payment_glitch.py b/models/_samples/python/py04__add_udf_function__payment_glitch.py similarity index 100% rename from models/examples/python/py04__add_udf_function__payment_glitch.py rename to models/_samples/python/py04__add_udf_function__payment_glitch.py diff --git a/models/_samples/snapshot/_snapshot__models.yml b/models/_samples/snapshot/_snapshot__models.yml new file mode 100644 index 0000000..a6786f4 --- /dev/null +++ b/models/_samples/snapshot/_snapshot__models.yml @@ -0,0 +1,9 @@ +version: 2 + +models: + - name: example_join_snapshots + columns: + - name: surrogate_key + tests: + - unique + - not_null \ No newline at end of file diff --git a/models/_samples/snapshot/example_join_snapshots.sql b/models/_samples/snapshot/example_join_snapshots.sql new file mode 100644 index 0000000..0281922 --- /dev/null +++ b/models/_samples/snapshot/example_join_snapshots.sql @@ -0,0 +1,49 @@ +with order_snapshot as ( + select + * exclude dbt_valid_to, + coalesce(dbt_valid_to, cast('{{ var("future_proof_date") }}' as timestamp)) as dbt_valid_to + from {{ ref('example_orders_snapshot') }} +), + +orders_line_items_snapshot as ( + select + * exclude dbt_valid_to, + coalesce(dbt_valid_to, cast('{{ var("future_proof_date") }}' as timestamp)) as dbt_valid_to + from {{ ref('example_orders_line_items_snapshot') }} +), + +joined as ( + +select + orders_line_items_snapshot.id as line_item_id, + order_snapshot.order_id, + order_snapshot.payment_method, + order_snapshot.status, + orders_line_items_snapshot.amount, + greatest(order_snapshot.dbt_valid_from, + orders_line_items_snapshot.dbt_valid_from) as valid_from, + least(order_snapshot.dbt_valid_to, + orders_line_items_snapshot.dbt_valid_to) as valid_to, + min(least(order_snapshot.order_created_at, orders_line_items_snapshot.order_created_at)) as order_created_at, + max(least(order_snapshot.order_updated_at, orders_line_items_snapshot.order_updated_at)) as order_updated_at + + +from order_snapshot + +left join orders_line_items_snapshot +on order_snapshot.order_id = orders_line_items_snapshot.order_id +and order_snapshot.dbt_valid_from <= orders_line_items_snapshot.dbt_valid_to +and order_snapshot.dbt_valid_to >= orders_line_items_snapshot.dbt_valid_from + +group by 1,2,3,4,5,6,7 + +), + +final as ( + select + {{ dbt_utils.generate_surrogate_key(['line_item_id', 'valid_from', 'valid_to']) }} as surrogate_key, + * + from joined +) + +select * from final \ No newline at end of file diff --git a/models/_samples/snapshot/example_orders_line_items_source_for_snapshot.sql b/models/_samples/snapshot/example_orders_line_items_source_for_snapshot.sql new file mode 100644 index 0000000..75c4889 --- /dev/null +++ b/models/_samples/snapshot/example_orders_line_items_source_for_snapshot.sql @@ -0,0 +1,108 @@ +{{ + config( + materialized='view' + ) +}} + + +with id_series as ( + +{{ dbt_utils.generate_series(100000) }} + +), + +adding_order_id as ( + + +select + generated_number as id, + + -- wacky way to get some semblance of randomness for which ids are tied to which orders + dense_rank() over ( + order by + case + when id % 5 = 0 + then id - 1 + when id % 5 = 1 + then id - 2 + when id % 9 = 0 + then id - 1 + else id + end + ) + as order_id + + +from id_series + +), + +distinct_order_ids as ( + select + distinct order_id + + from adding_order_id +), + +adding_order_created_at as ( + select + order_id, + {{ dbt.dateadd( + 'month', + -1, + dbt.dateadd("second", "order_id", dbt.date_trunc('day', 'current_timestamp'))) + }} as order_created_at + + from distinct_order_ids +), + +adding_order_details as ( + select + order_id, + -- every 5th order gets an update + order_id % 5 = 0 as needs_update, + + order_created_at, + + case + -- every 5th order gets randomly updated + when order_id % 5 = 0 + then current_timestamp + else order_created_at + end as order_updated_at + + + from adding_order_created_at +), + +final as ( + + select + adding_order_id.id, + adding_order_id.order_id, + + case + when adding_order_details.needs_update + then uniform(5, 35, random())::float - .01 + when id % 3 = 0 + then 9.99 + when id % 3 = 1 + then 5.99 + else 19.99 + end as amount, + + adding_order_details.order_created_at, + + adding_order_details.order_updated_at + + + from adding_order_id + + join adding_order_details + using (order_id) + +) + +select * +from final +order by 1, 2 \ No newline at end of file diff --git a/models/_samples/snapshot/example_orders_source_for_snapshot.sql b/models/_samples/snapshot/example_orders_source_for_snapshot.sql new file mode 100644 index 0000000..385d419 --- /dev/null +++ b/models/_samples/snapshot/example_orders_source_for_snapshot.sql @@ -0,0 +1,120 @@ +{{ + config( + materialized='view' + ) +}} + + +{% set statuses=['returned', 'completed', 'return_pending', 'shipped', 'placed'] %} + + +with id_series as ( + +{{ dbt_utils.generate_series(100000) }} + +), + +adding_order_id as ( + + +select + generated_number as id, + + -- wacky way to get some semblance of randomness for which ids are tied to which orders + dense_rank() over ( + order by + case + when id % 5 = 0 + then id - 1 + when id % 5 = 1 + then id - 2 + when id % 9 = 0 + then id - 1 + else id + end + ) + as order_id + + +from id_series + +), + +distinct_order_ids as ( + select + distinct order_id + + from adding_order_id +), + +adding_order_created_at as ( + select + order_id, + {{ dbt.dateadd( + 'month', + -1, + dbt.dateadd("second", "order_id", dbt.date_trunc('day', 'current_timestamp'))) + }} as order_created_at + + from distinct_order_ids +), + +final as ( + select + order_id, + -- every 5th order gets an update + order_id % 5 = 0 as needs_update, + + case + when adding_order_created_at.order_id % 10 <= 6 + then 'credit' + when adding_order_created_at.order_id % 10 <= 8 + then 'debit' + else 'cash' + end as payment_method, + + case + -- every 5th order gets randomly updated + when order_id % 5 = 0 + then + case + when uniform(0, 5, random()) = 0 + then '{{ statuses[0] }}' + when uniform(0, 5, random()) = 1 + then '{{ statuses[1] }}' + when uniform(0, 5, random()) = 2 + then '{{ statuses[2] }}' + when uniform(0, 5, random()) = 3 + then '{{ statuses[3] }}' + else '{{ statuses[4] }}' + end + when order_id % 3 = 0 + then 'placed' + when order_id % 3 = 1 + then 'shipped' + else 'returned' + end as status, + + order_created_at, + + case + -- every 5th order gets randomly updated + when order_id % 5 = 0 + then current_timestamp + else order_created_at + end as order_updated_at + + + from adding_order_created_at +) + +select + order_id, + status, + payment_method, + order_created_at, + order_updated_at + +from final + +order by 1 \ No newline at end of file diff --git a/models/examples/staging/jaffle_shop/_jaffle_shop__models.yml b/models/_samples/staging/jaffle_shop/_jaffle_shop__models.yml similarity index 100% rename from models/examples/staging/jaffle_shop/_jaffle_shop__models.yml rename to models/_samples/staging/jaffle_shop/_jaffle_shop__models.yml diff --git a/models/examples/staging/jaffle_shop/_jaffle_shop__sources.yml b/models/_samples/staging/jaffle_shop/_jaffle_shop__sources.yml similarity index 100% rename from models/examples/staging/jaffle_shop/_jaffle_shop__sources.yml rename to models/_samples/staging/jaffle_shop/_jaffle_shop__sources.yml diff --git a/models/examples/staging/jaffle_shop/stg_jaffle_shop__customers.sql b/models/_samples/staging/jaffle_shop/stg_jaffle_shop__customers.sql similarity index 100% rename from models/examples/staging/jaffle_shop/stg_jaffle_shop__customers.sql rename to models/_samples/staging/jaffle_shop/stg_jaffle_shop__customers.sql diff --git a/models/examples/staging/jaffle_shop/stg_jaffle_shop__orders.sql b/models/_samples/staging/jaffle_shop/stg_jaffle_shop__orders.sql similarity index 100% rename from models/examples/staging/jaffle_shop/stg_jaffle_shop__orders.sql rename to models/_samples/staging/jaffle_shop/stg_jaffle_shop__orders.sql diff --git a/models/examples/staging/my_old_unorganized_model.sql b/models/_samples/staging/my_old_unorganized_model.sql similarity index 100% rename from models/examples/staging/my_old_unorganized_model.sql rename to models/_samples/staging/my_old_unorganized_model.sql diff --git a/models/examples/staging/my_unorganized_model.sql b/models/_samples/staging/my_unorganized_model.sql similarity index 100% rename from models/examples/staging/my_unorganized_model.sql rename to models/_samples/staging/my_unorganized_model.sql diff --git a/models/marts/finance/_finance__models.yml b/models/marts/finance/_finance__models.yml index 370a00c..25d6f0d 100644 --- a/models/marts/finance/_finance__models.yml +++ b/models/marts/finance/_finance__models.yml @@ -1,5 +1,11 @@ version: 2 +groups: + - name: finance + owner: + email: finance@jaffleshop.com + name: Finance Person + models: - name: fct_order_items description: order items fact table diff --git a/models/marts/marketing/_marketing__models.yml b/models/marts/marketing/_marketing__models.yml index 56dc5e7..a0d3eda 100644 --- a/models/marts/marketing/_marketing__models.yml +++ b/models/marts/marketing/_marketing__models.yml @@ -1,5 +1,11 @@ version: 2 +groups: + - name: marketing + owner: + email: marketing@jaffleshop.com + name: Marketing Person + models: - name: dim_customers description: Customer dimensions table @@ -26,18 +32,4 @@ models: - name: account_balance description: '{{ doc("account_balance") }}' - name: market_segment - description: market segment of the customer - - name: dim_customers_legacy - description: Customer dimensions table - columns: - - name: customer_id - description: Primary id on the customers table - tests: - - unique - - not_null - - name: region - description: region name - tests: - - accepted_values: - values: ['AFRICA','MIDDLE EAST','ASIA','EUROPE','AMERICA'] # extra "!" from model will throw a warning - severity: warn + description: market segment of the customer \ No newline at end of file diff --git a/models/marts/operations/_operations__models.yml b/models/marts/operations/_operations__models.yml index 1197f35..1d46d11 100644 --- a/models/marts/operations/_operations__models.yml +++ b/models/marts/operations/_operations__models.yml @@ -1,5 +1,11 @@ version: 2 +groups: + - name: operations + owner: + email: operations@jaffleshop.com + name: Operations Person + models: - name: dim_parts description: Parts dimensions table diff --git a/packages.yml b/packages.yml index 66a5163..1b69596 100644 --- a/packages.yml +++ b/packages.yml @@ -8,7 +8,7 @@ packages: - package: dbt-labs/metrics version: 1.5.0 - package: brooklyn-data/dbt_artifacts - version: 2.3.0 + version: 2.4.2 - package: dbt-labs/dbt_project_evaluator version: 0.6.0 - package: calogica/dbt_expectations diff --git a/seeds/examples/schema.yml b/seeds/samples/_seeds.yml similarity index 100% rename from seeds/examples/schema.yml rename to seeds/samples/_seeds.yml diff --git a/seeds/examples/alphabet_grouping.csv b/seeds/samples/alphabet_grouping.csv similarity index 100% rename from seeds/examples/alphabet_grouping.csv rename to seeds/samples/alphabet_grouping.csv diff --git a/selectors.yml b/selectors.yml new file mode 100644 index 0000000..094962a --- /dev/null +++ b/selectors.yml @@ -0,0 +1,10 @@ +selectors: + - name: all_models_and_tagged_snapshots + description: includes all models, but only snapshots tagged daily + definition: + union: + - 'models/*' + - 'seeds/*' + - intersection: + - 'config.materialized:snapshot' + - 'tag:daily' \ No newline at end of file diff --git a/snapshots/_samples/_example__snapshots.yml b/snapshots/_samples/_example__snapshots.yml new file mode 100644 index 0000000..0b03612 --- /dev/null +++ b/snapshots/_samples/_example__snapshots.yml @@ -0,0 +1,25 @@ +version: 2 + +snapshots: + - name: example_orders_line_items_snapshot + columns: + - name: id + tests: + - relationships: + to: ref('example_orders_line_items_source_for_snapshot') + field: id + - name: dbt_scd_id + tests: + - unique + - not_null + - name: example_check_snapshot + columns: + - name: id + tests: + - relationships: + to: ref('example_orders_line_items_source_for_snapshot') + field: id + - name: dbt_scd_id + tests: + - unique + - not_null \ No newline at end of file diff --git a/snapshots/_samples/example_check_snapshot.sql b/snapshots/_samples/example_check_snapshot.sql new file mode 100644 index 0000000..7fd0573 --- /dev/null +++ b/snapshots/_samples/example_check_snapshot.sql @@ -0,0 +1,20 @@ +{% snapshot example_check_snapshot %} + {{ + config( + target_database='analytics', + target_schema='snapshots', + unique_key='id', + strategy='check', + check_cols=['check_cols'], + invalidate_hard_deletes=True + ) + }} + + {% set column_names = dbt_utils.get_filtered_columns_in_relation(from=ref('example_orders_line_items_source_for_snapshot'), except=["id"]) %} + + + select + *, + {{ dbt_utils.generate_surrogate_key(column_names) }} as check_cols + from {{ ref('example_orders_line_items_source_for_snapshot') }} + {% endsnapshot %} \ No newline at end of file diff --git a/snapshots/_samples/example_generate_schema_snapshot.sql b/snapshots/_samples/example_generate_schema_snapshot.sql new file mode 100644 index 0000000..2174d7b --- /dev/null +++ b/snapshots/_samples/example_generate_schema_snapshot.sql @@ -0,0 +1,14 @@ +{% snapshot example_generate_schema_snapshot %} + {{ + config( + target_database='analytics', + target_schema=generate_schema_name('snapshots'), + unique_key='id', + strategy='timestamp', + updated_at='order_updated_at', + invalidate_hard_deletes=True + ) + }} + + select * from {{ ref('example_orders_line_items_source_for_snapshot') }} + {% endsnapshot %} \ No newline at end of file diff --git a/snapshots/_samples/example_orders_line_items_snapshot.sql b/snapshots/_samples/example_orders_line_items_snapshot.sql new file mode 100644 index 0000000..63603f5 --- /dev/null +++ b/snapshots/_samples/example_orders_line_items_snapshot.sql @@ -0,0 +1,14 @@ +{% snapshot example_orders_line_items_snapshot %} + {{ + config( + target_database='analytics', + target_schema='snapshots', + unique_key='id', + strategy='timestamp', + updated_at='order_updated_at', + invalidate_hard_deletes=True + ) + }} + + select * from {{ ref('example_orders_line_items_source_for_snapshot') }} + {% endsnapshot %} \ No newline at end of file diff --git a/snapshots/_samples/example_orders_snapshot.sql b/snapshots/_samples/example_orders_snapshot.sql new file mode 100644 index 0000000..e4e7ad6 --- /dev/null +++ b/snapshots/_samples/example_orders_snapshot.sql @@ -0,0 +1,14 @@ +{% snapshot example_orders_snapshot %} + {{ + config( + target_database='analytics', + target_schema='snapshots', + unique_key='order_id', + strategy='timestamp', + updated_at='order_updated_at', + invalidate_hard_deletes=True + ) + }} + + select * from {{ ref('example_orders_source_for_snapshot') }} + {% endsnapshot %} \ No newline at end of file diff --git a/tests/examples/audit_helper_compare_all_columns__stg_tpch__orders.sql b/tests/_samples/audit_helper_compare_all_columns__stg_tpch__orders.sql similarity index 100% rename from tests/examples/audit_helper_compare_all_columns__stg_tpch__orders.sql rename to tests/_samples/audit_helper_compare_all_columns__stg_tpch__orders.sql diff --git a/tests/examples/example_singular_test_multiple_models.sql b/tests/_samples/example_singular_test_multiple_models.sql similarity index 100% rename from tests/examples/example_singular_test_multiple_models.sql rename to tests/_samples/example_singular_test_multiple_models.sql