From 069d71ce1d6a041e201ce11b349ed38f34fe1cdd Mon Sep 17 00:00:00 2001 From: Doug Guthrie Date: Fri, 29 Sep 2023 08:37:26 -0600 Subject: [PATCH] Add postgres support (#368) * Add macros for postgres * Add steps for postgres * Add postgres entry * Add a postgres service to action * Just test postgres for now * Fix duplicate section * Install correct package * Update host to localhost * Update profiles.yml to env vars * Add postgres env vars to step * Try again * Remove parse_json func * Another fix * Another fix * Alias the subquery * Add back other CDW [skip-ci] * Add postgres as supported DW [skip ci] * Test removing env vars * Another test * Add back DWs * Add missing CI tests * Formatting * Update to include new features --------- Co-authored-by: Gemma Down --- .github/workflows/ci_test_package.yml | 30 ++++++++- .github/workflows/main_test_package.yml | 15 ++++- README.md | 1 + integration_test_project/profiles.yml | 13 +++- .../tests/singular_test.sql | 2 +- .../upload_exposures.sql | 34 ++++++++++ .../upload_invocations.sql | 62 +++++++++++++++++++ .../upload_model_executions.sql | 41 ++++++++++++ .../upload_models.sql | 35 +++++++++++ .../upload_seed_executions.sql | 57 +++++++++++++++++ .../upload_seeds.sql | 31 ++++++++++ .../upload_snapshot_executions.sql | 57 +++++++++++++++++ .../upload_snapshots.sql | 33 ++++++++++ .../upload_sources.sql | 31 ++++++++++ .../upload_test_executions.sql | 55 ++++++++++++++++ .../upload_tests.sql | 28 +++++++++ .../insert_into_metadata_table.sql | 14 +++++ tox.ini | 40 ++++++++++++ 18 files changed, 573 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci_test_package.yml b/.github/workflows/ci_test_package.yml index edd8cdc9..b02997ee 100644 --- a/.github/workflows/ci_test_package.yml +++ b/.github/workflows/ci_test_package.yml @@ -37,7 +37,7 @@ jobs: strategy: fail-fast: false # Don't fail one DWH if the others fail matrix: - warehouse: ["snowflake", "bigquery"] + warehouse: ["snowflake", "bigquery", "postgres"] runs-on: ubuntu-latest environment: name: Approve Integration Tests @@ -45,6 +45,19 @@ jobs: contents: "read" id-token: "write" + services: + postgres: + image: postgres + env: + POSTGRES_PASSWORD: postgres + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + steps: - name: Get latest release uses: rez0n/actions-github-release@main @@ -100,7 +113,7 @@ jobs: strategy: fail-fast: false # Don't fail one DWH if the others fail matrix: - warehouse: ["snowflake", "bigquery"] + warehouse: ["snowflake", "bigquery", "postgres"] # When supporting a new version, update the list here version: ["1_3_0", "1_4_0", "1_5_0", "1_6_0"] runs-on: ubuntu-latest @@ -110,6 +123,19 @@ jobs: contents: "read" id-token: "write" + services: + postgres: + image: postgres + env: + POSTGRES_PASSWORD: postgres + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + steps: - uses: actions/setup-python@v4 with: diff --git a/.github/workflows/main_test_package.yml b/.github/workflows/main_test_package.yml index 9aa70ae3..4398c459 100644 --- a/.github/workflows/main_test_package.yml +++ b/.github/workflows/main_test_package.yml @@ -34,13 +34,26 @@ jobs: integration: strategy: matrix: - warehouse: ["snowflake", "bigquery"] + warehouse: ["snowflake", "bigquery", "postgres"] version: ["1_3_0", "1_4_0", "1_5_0", "1_6_0"] runs-on: ubuntu-latest permissions: contents: "read" id-token: "write" + services: + postgres: + image: postgres + env: + POSTGRES_PASSWORD: postgres + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 5432:5432 + steps: - name: Checkout uses: actions/checkout@v3 diff --git a/README.md b/README.md index 3f0818f7..17688961 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ The package currently supports - Spark :white_check_mark: - Snowflake :white_check_mark: - Google BigQuery :white_check_mark: +- Postgres :white_check_mark: Models included: diff --git a/integration_test_project/profiles.yml b/integration_test_project/profiles.yml index e8548e3f..b24ad80d 100644 --- a/integration_test_project/profiles.yml +++ b/integration_test_project/profiles.yml @@ -2,8 +2,8 @@ # You should __NEVER__ check credentials into version control. Thanks for reading :) config: - send_anonymous_usage_stats: False - use_colors: True + send_anonymous_usage_stats: False + use_colors: True dbt_artifacts: target: snowflake @@ -43,3 +43,12 @@ dbt_artifacts: timeout_seconds: 300 priority: interactive retries: 1 + postgres: + type: postgres + host: localhost + user: postgres + password: postgres + port: 5432 + dbname: postgres + schema: public + threads: 8 diff --git a/integration_test_project/tests/singular_test.sql b/integration_test_project/tests/singular_test.sql index 67a2508d..a1e49e8c 100644 --- a/integration_test_project/tests/singular_test.sql +++ b/integration_test_project/tests/singular_test.sql @@ -1 +1 @@ -select 1 as failures from (select 2) where 1 = 2 +select 1 as failures from (select 2) as foo where 1 = 2 diff --git a/macros/upload_individual_datasets/upload_exposures.sql b/macros/upload_individual_datasets/upload_exposures.sql index 488f9ffd..9f0ec5d3 100644 --- a/macros/upload_individual_datasets/upload_exposures.sql +++ b/macros/upload_individual_datasets/upload_exposures.sql @@ -84,3 +84,37 @@ {{ return("") }} {% endif %} {%- endmacro %} + +{% macro postgres__get_exposures_dml_sql(exposures) -%} + {% if exposures != [] %} + + {% set exposure_values %} + {% for exposure in exposures -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + $${{ exposure.unique_id }}$$, {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + $${{ exposure.name }}$$, {# name #} + '{{ exposure.type }}', {# type #} + $${{ tojson(exposure.owner) }}$$, {# owner #} + '{{ exposure.maturity }}', {# maturity #} + $${{ exposure.original_file_path }}$$, {# path #} + $${{ exposure.description }}$$, {# description #} + '{{ exposure.url }}', {# url #} + '{{ exposure.package_name }}', {# package_name #} + $${{ tojson(exposure.depends_on.nodes) }}$$, {# depends_on_nodes #} + $${{ tojson(exposure.tags) }}$$, {# tags #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + $${{ tojson(exposure) }}$$ {# all_results #} + {% endif %} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + {% endset %} + {{ exposure_values }} + {% else %} + {{ return("") }} + {% endif %} +{%- endmacro %} diff --git a/macros/upload_individual_datasets/upload_invocations.sql b/macros/upload_individual_datasets/upload_invocations.sql index f3dcdd6b..21c5574c 100644 --- a/macros/upload_individual_datasets/upload_invocations.sql +++ b/macros/upload_individual_datasets/upload_invocations.sql @@ -159,3 +159,65 @@ {{ invocation_values }} {% endmacro -%} + +{% macro postgres__get_invocations_dml_sql() -%} + {% set invocation_values %} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ dbt_version }}', {# dbt_version #} + '{{ project_name }}', {# project_name #} + '{{ run_started_at }}', {# run_started_at #} + '{{ flags.WHICH }}', {# dbt_command #} + {{ flags.FULL_REFRESH }}, {# full_refresh_flag #} + '{{ target.profile_name }}', {# target_profile_name #} + '{{ target.name }}', {# target_name #} + '{{ target.schema }}', {# target_schema #} + {{ target.threads }}, {# target_threads #} + + '{{ env_var("DBT_CLOUD_PROJECT_ID", "") }}', {# dbt_cloud_project_id #} + '{{ env_var("DBT_CLOUD_JOB_ID", "") }}', {# dbt_cloud_job_id #} + '{{ env_var("DBT_CLOUD_RUN_ID", "") }}', {# dbt_cloud_run_id #} + '{{ env_var("DBT_CLOUD_RUN_REASON_CATEGORY", "") }}', {# dbt_cloud_run_reason_category #} + $${{ env_var('DBT_CLOUD_RUN_REASON', '') }}$$, {# dbt_cloud_run_reason #} + + {% if var('env_vars', none) %} + {% set env_vars_dict = {} %} + {% for env_variable in var('env_vars') %} + {% do env_vars_dict.update({env_variable: (env_var(env_variable, ''))}) %} + {% endfor %} + $${{ tojson(env_vars_dict) }}$$, {# env_vars #} + {% else %} + null, {# env_vars #} + {% endif %} + + {% if var('dbt_vars', none) %} + {% set dbt_vars_dict = {} %} + {% for dbt_var in var('dbt_vars') %} + {% do dbt_vars_dict.update({dbt_var: (var(dbt_var, ''))}) %} + {% endfor %} + $${{ tojson(dbt_vars_dict) }}$$, {# dbt_vars #} + {% else %} + null, {# dbt_vars #} + {% endif %} + + {% if invocation_args_dict.vars %} + {# vars - different format for pre v1.5 (yaml vs list) #} + {% if invocation_args_dict.vars is string %} + {# BigQuery does not handle the yaml-string from "--vars" well, when passed to "parse_json". Workaround is to parse the string, and then "tojson" will properly format the dict as a json-object. #} + {% set parsed_inv_args_vars = fromyaml(invocation_args_dict.vars) %} + {% do invocation_args_dict.update({'vars': parsed_inv_args_vars}) %} + {% endif %} + {% endif %} + + $${{ tojson(invocation_args_dict) }}$$, {# invocation_args #} + + {% set metadata_env = {} %} + {% for key, value in dbt_metadata_envs.items() %} + {% do metadata_env.update({key: value}) %} + {% endfor %} + $${{ tojson(metadata_env) }}$$ {# dbt_custom_envs #} + ) + {% endset %} + {{ invocation_values }} + +{% endmacro -%} diff --git a/macros/upload_individual_datasets/upload_model_executions.sql b/macros/upload_individual_datasets/upload_model_executions.sql index fca386b2..bca26fea 100644 --- a/macros/upload_individual_datasets/upload_model_executions.sql +++ b/macros/upload_individual_datasets/upload_model_executions.sql @@ -162,3 +162,44 @@ {{ return("") }} {% endif %} {% endmacro -%} + +{% macro postgres__get_model_executions_dml_sql(models) -%} + {% if models != [] %} + {% set model_execution_values %} + {% for model in models -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ model.node.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + + {% set config_full_refresh = model.node.config.full_refresh %} + {% if config_full_refresh is none %} + {% set config_full_refresh = flags.FULL_REFRESH %} + {% endif %} + {{ config_full_refresh }}, {# was_full_refresh #} + + '{{ model.thread_id }}', {# thread_id #} + '{{ model.status }}', {# status #} + + {% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %} + {% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #} + {% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %} + {% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #} + + {{ model.execution_time }}, {# total_node_runtime #} + null, {# rows_affected #} + '{{ model.node.config.materialized }}', {# materialization #} + '{{ model.node.schema }}', {# schema #} + '{{ model.node.name }}', {# name #} + '{{ model.node.alias }}', {# alias #} + $${{ model.message }}$$, {# message #} + $${{ tojson(model.adapter_response) }}$$ {# adapter_response #} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + {% endset %} + {{ model_execution_values }} + {% else %} + {{ return("") }} + {% endif %} +{%- endmacro %} diff --git a/macros/upload_individual_datasets/upload_models.sql b/macros/upload_individual_datasets/upload_models.sql index f0748876..d8dc49fd 100644 --- a/macros/upload_individual_datasets/upload_models.sql +++ b/macros/upload_individual_datasets/upload_models.sql @@ -89,3 +89,38 @@ {{ return("") }} {% endif %} {%- endmacro %} + +{% macro postgres__get_models_dml_sql(models) -%} + {% if models != [] %} + {% set model_values %} + {% for model in models -%} + {% do model.pop('raw_code', None) %} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ model.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + '{{ model.database }}', {# database #} + '{{ model.schema }}', {# schema #} + '{{ model.name }}', {# name #} + '{{ tojson(model.depends_on.nodes) }}', {# depends_on_nodes #} + '{{ model.package_name }}', {# package_name #} + $${{ model.original_file_path | replace('\\', '\\\\') }}$$, {# path #} + '{{ model.checksum.checksum }}', {# checksum #} + '{{ model.config.materialized }}', {# materialization #} + '{{ tojson(model.tags) }}', {# tags #} + $${{ model.config.meta }}$$, {# meta #} + '{{ model.alias }}', {# alias #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + $${{ tojson(model) }}$$ {# all_results #} + {% endif %} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + {% endset %} + {{ model_values }} + {% else %} + {{ return("") }} + {% endif %} +{%- endmacro %} diff --git a/macros/upload_individual_datasets/upload_seed_executions.sql b/macros/upload_individual_datasets/upload_seed_executions.sql index 69c4540c..ca947ac2 100644 --- a/macros/upload_individual_datasets/upload_seed_executions.sql +++ b/macros/upload_individual_datasets/upload_seed_executions.sql @@ -208,3 +208,60 @@ {{ return("") }} {% endif %} {% endmacro -%} + +{% macro postgres__get_seed_executions_dml_sql(seeds) -%} + {% if seeds != [] %} + {% set seed_execution_values %} + {% for model in seeds -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ model.node.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + + {% set config_full_refresh = model.node.config.full_refresh %} + {% if config_full_refresh is none %} + {% set config_full_refresh = flags.FULL_REFRESH %} + {% endif %} + {{ config_full_refresh }}, {# was_full_refresh #} + + '{{ model.thread_id }}', {# thread_id #} + '{{ model.status }}', {# status #} + + {% if model.timing != [] %} + {% for stage in model.timing if stage.name == "compile" %} + {% if loop.length == 0 %} + null, {# compile_started_at #} + {% else %} + '{{ stage.started_at }}', {# compile_started_at #} + {% endif %} + {% endfor %} + + {% for stage in model.timing if stage.name == "execute" %} + {% if loop.length == 0 %} + null, {# query_completed_at #} + {% else %} + '{{ stage.completed_at }}', {# query_completed_at #} + {% endif %} + {% endfor %} + {% else %} + null, {# compile_started_at #} + null, {# query_completed_at #} + {% endif %} + + {{ model.execution_time }}, {# total_node_runtime #} + null, -- rows_affected not available {# Databricks #} + '{{ model.node.config.materialized }}', {# materialization #} + '{{ model.node.schema }}', {# schema #} + '{{ model.node.name }}', {# name #} + '{{ model.node.alias }}', {# alias #} + $${{ model.message }}$$, {# message #} + $${{ tojson(model.adapter_response) }}$$ {# adapter_response #} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + {% endset %} + {{ seed_execution_values }} + {% else %} + {{ return("") }} + {% endif %} +{% endmacro -%} diff --git a/macros/upload_individual_datasets/upload_seeds.sql b/macros/upload_individual_datasets/upload_seeds.sql index 2f12507e..32e67383 100644 --- a/macros/upload_individual_datasets/upload_seeds.sql +++ b/macros/upload_individual_datasets/upload_seeds.sql @@ -78,3 +78,34 @@ {{ return("") }} {% endif %} {%- endmacro %} + +{% macro postgres__get_seeds_dml_sql(seeds) -%} + {% if seeds != [] %} + {% set seed_values %} + {% for seed in seeds -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ seed.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + '{{ seed.database }}', {# database #} + '{{ seed.schema }}', {# schema #} + '{{ seed.name }}', {# name #} + '{{ seed.package_name }}', {# package_name #} + '{{ seed.original_file_path | replace('\\', '\\\\') }}', {# path #} + '{{ seed.checksum.checksum }}', {# checksum #} + $${{ tojson(seed.config.meta) }}$$, {# meta #} + '{{ seed.alias }}', {# alias #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + $${{ tojson(seed) }}$$ {# all_results #} + {% endif %} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + {% endset %} + {{ seed_values }} + {% else %} + {{ return("") }} + {% endif %} +{%- endmacro %} diff --git a/macros/upload_individual_datasets/upload_snapshot_executions.sql b/macros/upload_individual_datasets/upload_snapshot_executions.sql index a9980b18..369348cd 100644 --- a/macros/upload_individual_datasets/upload_snapshot_executions.sql +++ b/macros/upload_individual_datasets/upload_snapshot_executions.sql @@ -208,3 +208,60 @@ {{ return("") }} {% endif %} {% endmacro -%} + +{% macro postgres__get_snapshot_executions_dml_sql(snapshots) -%} + {% if snapshots != [] %} + {% set snapshot_execution_values %} + {% for model in snapshots -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ model.node.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + + {% set config_full_refresh = model.node.config.full_refresh %} + {% if config_full_refresh is none %} + {% set config_full_refresh = flags.FULL_REFRESH %} + {% endif %} + {{ config_full_refresh }}, {# was_full_refresh #} + + '{{ model.thread_id }}', {# thread_id #} + '{{ model.status }}', {# status #} + + {% if model.timing != [] %} + {% for stage in model.timing if stage.name == "compile" %} + {% if loop.length == 0 %} + null, {# compile_started_at #} + {% else %} + '{{ stage.started_at }}', {# compile_started_at #} + {% endif %} + {% endfor %} + + {% for stage in model.timing if stage.name == "execute" %} + {% if loop.length == 0 %} + null, {# query_completed_at #} + {% else %} + '{{ stage.completed_at }}', {# query_completed_at #} + {% endif %} + {% endfor %} + {% else %} + null, {# compile_started_at #} + null, {# query_completed_at #} + {% endif %} + + {{ model.execution_time }}, {# total_node_runtime #} + null, {# rows_affected #} + '{{ model.node.config.materialized }}', {# materialization #} + '{{ model.node.schema }}', {# schema #} + '{{ model.node.name }}', {# name #} + '{{ model.node.alias }}', {# alias #} + $${{ model.message }}$$, {# message #} + $${{ tojson(model.adapter_response) }}$$ {# adapter_response #} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + {% endset %} + {{ snapshot_execution_values }} + {% else %} + {{ return("") }} + {% endif %} +{% endmacro -%} diff --git a/macros/upload_individual_datasets/upload_snapshots.sql b/macros/upload_individual_datasets/upload_snapshots.sql index 82789e3a..0be6759b 100644 --- a/macros/upload_individual_datasets/upload_snapshots.sql +++ b/macros/upload_individual_datasets/upload_snapshots.sql @@ -86,3 +86,36 @@ {{ return("") }} {% endif %} {%- endmacro %} + +{% macro postgres__get_snapshots_dml_sql(snapshots) -%} + {% if snapshots != [] %} + {% set snapshot_values %} + {% for snapshot in snapshots -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ snapshot.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + '{{ snapshot.database }}', {# database #} + '{{ snapshot.schema }}', {# schema #} + '{{ snapshot.name }}', {# name #} + $${{ tojson(snapshot.depends_on.nodes) }}$$, {# depends_on_nodes #} + '{{ snapshot.package_name }}', {# package_name #} + '{{ snapshot.original_file_path | replace('\\', '\\\\') }}', {# path #} + '{{ snapshot.checksum.checksum }}', {# checksum #} + '{{ snapshot.config.strategy }}', {# strategy #} + $${{ tojson(snapshot.config.meta) }}$$, {# meta #} + '{{ snapshot.alias }}', {# alias #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + $${{ tojson(snapshot) }}$$ {# all_results #} + {% endif %} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + {% endset %} + {{ snapshot_values }} + {% else %} + {{ return("") }} + {% endif %} +{%- endmacro %} diff --git a/macros/upload_individual_datasets/upload_sources.sql b/macros/upload_individual_datasets/upload_sources.sql index 23208d6d..dbcb49e4 100644 --- a/macros/upload_individual_datasets/upload_sources.sql +++ b/macros/upload_individual_datasets/upload_sources.sql @@ -78,3 +78,34 @@ {{ return("") }} {% endif %} {%- endmacro %} + +{% macro postgres__get_sources_dml_sql(sources) -%} + {% if sources != [] %} + {% set source_values %} + {% for source in sources -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ source.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + '{{ source.database }}', {# database #} + '{{ source.schema }}', {# schema #} + '{{ source.source_name }}', {# source_name #} + '{{ source.loader }}', {# loader #} + '{{ source.name }}', {# name #} + '{{ source.identifier }}', {# identifier #} + $${{ source.loaded_at_field }}$$, {# loaded_at_field #} + $${{ tojson(source.freshness) }}$$, {# freshness #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + $${{ tojson(source) }}$$ {# all_results #} + {% endif %} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + {% endset %} + {{ source_values }} + {% else %} + {{ return("") }} + {% endif %} +{%- endmacro %} diff --git a/macros/upload_individual_datasets/upload_test_executions.sql b/macros/upload_individual_datasets/upload_test_executions.sql index 60986658..a42b03ba 100644 --- a/macros/upload_individual_datasets/upload_test_executions.sql +++ b/macros/upload_individual_datasets/upload_test_executions.sql @@ -125,3 +125,58 @@ {{ return("") }} {% endif %} {% endmacro -%} + +{% macro postgres__get_test_executions_dml_sql(tests) -%} + {% if tests != [] %} + {% set test_execution_values %} + {% for test in tests -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ test.node.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + + {% set config_full_refresh = test.node.config.full_refresh %} + {% if config_full_refresh is none %} + {% set config_full_refresh = flags.FULL_REFRESH %} + {% endif %} + {{ config_full_refresh }}, {# was_full_refresh #} + + '{{ test.thread_id }}', {# thread_id #} + '{{ test.status }}', {# status #} + + {% if test.timing != [] %} + {% for stage in test.timing if stage.name == "compile" %} + {% if loop.length == 0 %} + null, {# compile_started_at #} + {% else %} + '{{ stage.started_at }}', {# compile_started_at #} + {% endif %} + {% endfor %} + + {% for stage in test.timing if stage.name == "execute" %} + {% if loop.length == 0 %} + null, {# query_completed_at #} + {% else %} + '{{ stage.completed_at }}', {# query_completed_at #} + {% endif %} + {% endfor %} + {% else %} + null, {# compile_started_at #} + null, {# query_completed_at #} + {% endif %} + + {{ test.execution_time }}, {# total_node_runtime #} + null, {# rows_affected not available in Databricks #} + {{ 'null' if test.failures is none else test.failures }}, {# failures #} + $${{ test.message }}$$, {# message #} + $${{ tojson(test.adapter_response) }}$$ {# adapter_response #} + ) + {%- if not loop.last %},{%- endif %} + + {%- endfor %} + {% endset %} + {{ test_execution_values }} + {% else %} + {{ return("") }} + {% endif %} +{% endmacro -%} diff --git a/macros/upload_individual_datasets/upload_tests.sql b/macros/upload_individual_datasets/upload_tests.sql index d29341ff..cf75e9ba 100644 --- a/macros/upload_individual_datasets/upload_tests.sql +++ b/macros/upload_individual_datasets/upload_tests.sql @@ -69,3 +69,31 @@ {{ return("") }} {% endif %} {%- endmacro %} + +{% macro postgres__get_tests_dml_sql(tests) -%} + {% if tests != [] %} + {% set test_values %} + {% for test in tests -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ test.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + '{{ test.name }}', {# name #} + $${{ tojson(test.depends_on.nodes) }}$$, {# depends_on_nodes #} + '{{ test.package_name }}', {# package_name #} + '{{ test.original_file_path | replace('\\', '\\\\') }}', {# test_path #} + $${{ tojson(test.tags) }}$$, {# tags #} + {% if var('dbt_artifacts_exclude_all_results', false) %} + null + {% else %} + $${{ tojson(test) }}$$ {# all_results #} + {% endif %} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + {% endset %} + {{ test_values }} + {% else %} + {{ return("") }} + {% endif %} +{%- endmacro %} diff --git a/macros/upload_results/insert_into_metadata_table.sql b/macros/upload_results/insert_into_metadata_table.sql index 2ec4d5d5..24f1eb77 100644 --- a/macros/upload_results/insert_into_metadata_table.sql +++ b/macros/upload_results/insert_into_metadata_table.sql @@ -45,5 +45,19 @@ {%- endmacro %} +{%- endmacro %} + +{% macro postgres__insert_into_metadata_table(relation, fields, content) -%} + + {% set insert_into_table_query %} + insert into {{ relation }} {{ fields }} + values + {{ content }} + {% endset %} + + {% do run_query(insert_into_table_query) %} + +{%- endmacro %} + {% macro default__insert_into_metadata_table(relation, fields, content) -%} {%- endmacro %} diff --git a/tox.ini b/tox.ini index fbff0176..8add76ba 100644 --- a/tox.ini +++ b/tox.ini @@ -250,3 +250,43 @@ commands = dbt deps dbt build --exclude snapshot --target spark +[testenv:integration_postgres] +changedir = integration_test_project +deps = dbt-postgres~=1.6.0 +commands = + dbt clean + dbt deps + dbt build --target postgres + +[testenv:integration_postgres_1_3_0] +changedir = integration_test_project +deps = dbt-postgres~=1.3.0 +commands = + dbt clean + dbt deps + dbt build --target postgres + +[testenv:integration_postgres_1_4_0] +changedir = integration_test_project +deps = dbt-postgres~=1.4.0 +commands = + dbt clean + dbt deps + dbt build --target postgres + +[testenv:integration_postgres_1_5_0] +changedir = integration_test_project +deps = dbt-postgres~=1.5.0 +commands = + dbt clean + dbt deps + dbt build --target postgres + +[testenv:integration_postgres_1_6_0] +changedir = integration_test_project +deps = dbt-postgres~=1.6.0 +commands = + dbt clean + dbt deps + dbt build --target postgres +