From 7d187c4dbe6aa159b7a28472af869204ef96cb94 Mon Sep 17 00:00:00 2001 From: FishtownBuildBot <77737458+FishtownBuildBot@users.noreply.github.com> Date: Fri, 11 Oct 2024 19:35:24 -0400 Subject: [PATCH 01/12] [Automated] Update pre-commit hooks (#297) Co-authored-by: Mike Alfare Co-authored-by: Mike Alfare <13974384+mikealfare@users.noreply.github.com> --- .pre-commit-config.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0f2a03f7..aa942447 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,7 +18,7 @@ repos: - id: dbt-core-in-adapters-check - repo: https://github.com/psf/black - rev: 24.4.0 + rev: 24.8.0 hooks: - id: black args: @@ -30,7 +30,7 @@ repos: - --force-exclude=dbt/adapters/events/adapter_types_pb2.py - repo: https://github.com/pycqa/flake8 - rev: 7.0.0 + rev: 7.1.1 hooks: - id: flake8 exclude: dbt/adapters/events/adapter_types_pb2.py|tests/functional/ @@ -41,7 +41,7 @@ repos: - --per-file-ignores=*/__init__.py:F401,*/conftest.py:F401 - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.9.0 + rev: v1.11.2 hooks: - id: mypy exclude: dbt/adapters/events/adapter_types_pb2.py|dbt-tests-adapter/dbt/__init__.py From 3b6ad1f93e731b5046651ddecb635113a5c1cbe7 Mon Sep 17 00:00:00 2001 From: Github Build Bot Date: Tue, 15 Oct 2024 23:03:35 +0000 Subject: [PATCH 02/12] Bumping version to 1.7.1 and generate changelog --- .changes/1.7.1.md | 5 +++++ .changes/unreleased/Features-20240927-134248.yaml | 6 ------ CHANGELOG.md | 10 ++++++++-- dbt/adapters/__about__.py | 2 +- 4 files changed, 14 insertions(+), 9 deletions(-) create mode 100644 .changes/1.7.1.md delete mode 100644 .changes/unreleased/Features-20240927-134248.yaml diff --git a/.changes/1.7.1.md b/.changes/1.7.1.md new file mode 100644 index 00000000..4acda7f8 --- /dev/null +++ b/.changes/1.7.1.md @@ -0,0 +1,5 @@ +## dbt-adapters 1.7.1 - October 15, 2024 + +### Features + +- Enable setting current value of dbt_valid_to ([#320](https://github.com/dbt-labs/dbt-adapters/issues/320)) diff --git a/.changes/unreleased/Features-20240927-134248.yaml b/.changes/unreleased/Features-20240927-134248.yaml deleted file mode 100644 index 0b456244..00000000 --- a/.changes/unreleased/Features-20240927-134248.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Enable setting current value of dbt_valid_to -time: 2024-09-27T13:42:48.654556-04:00 -custom: - Author: gshank - Issue: "320" diff --git a/CHANGELOG.md b/CHANGELOG.md index dfb37ad5..8e9f64fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,12 +11,18 @@ and is generated by [Changie](https://github.com/miniscruff/changie). - dbt-tests-adapters: Add required begin to microbatch model config to BaseMicrobatch test ([#315](https://github.com/dbt-labs/dbt-adapters/issues/315)) - - ## dbt-adapters 1.10.1 - September 16, 2024 ## dbt-adapters 1.10.0 - September 12, 2024 +## dbt-adapters 1.7.1 - October 15, 2024 + +### Features + +- Enable setting current value of dbt_valid_to ([#320](https://github.com/dbt-labs/dbt-adapters/issues/320)) + + + ## dbt-adapters 1.7.0 - September 19, 2024 ### Features diff --git a/dbt/adapters/__about__.py b/dbt/adapters/__about__.py index a55413d1..116d5667 100644 --- a/dbt/adapters/__about__.py +++ b/dbt/adapters/__about__.py @@ -1 +1 @@ -version = "1.7.0" +version = "1.7.1" From 1029b2d5c55a840fcba86eb0b1d3819b26dc7e57 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Thu, 17 Oct 2024 15:06:34 -0400 Subject: [PATCH 03/12] Drop support for Python 3.8 (#332) --- .changes/unreleased/Breaking Changes-20241016-180629.yaml | 6 ++++++ .github/actions/publish-results/action.yml | 2 +- .github/workflows/unit-tests.yml | 2 +- .pre-commit-config.yaml | 2 +- dbt-tests-adapter/pyproject.toml | 3 +-- pyproject.toml | 3 +-- 6 files changed, 11 insertions(+), 7 deletions(-) create mode 100644 .changes/unreleased/Breaking Changes-20241016-180629.yaml diff --git a/.changes/unreleased/Breaking Changes-20241016-180629.yaml b/.changes/unreleased/Breaking Changes-20241016-180629.yaml new file mode 100644 index 00000000..40f103c0 --- /dev/null +++ b/.changes/unreleased/Breaking Changes-20241016-180629.yaml @@ -0,0 +1,6 @@ +kind: Breaking Changes +body: Drop support for Python 3.8 +time: 2024-10-16T18:06:29.535761-04:00 +custom: + Author: mikealfare + Issue: "332" diff --git a/.github/actions/publish-results/action.yml b/.github/actions/publish-results/action.yml index 0d5cb7e6..7c73a94f 100644 --- a/.github/actions/publish-results/action.yml +++ b/.github/actions/publish-results/action.yml @@ -5,7 +5,7 @@ inputs: description: File type for file name stub (e.g. "unit-tests") required: true python-version: - description: Python version for the file name stub (e.g. "3.8") + description: Python version for the file name stub (e.g. "3.9") required: true source-file: description: File to be uploaded diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index b61c83d7..b4ac615d 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -23,7 +23,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.9", "3.10", "3.11", "3.12"] steps: - name: Check out repository diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index aa942447..b7835274 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -23,10 +23,10 @@ repos: - id: black args: - --line-length=99 - - --target-version=py38 - --target-version=py39 - --target-version=py310 - --target-version=py311 + - --target-version=py312 - --force-exclude=dbt/adapters/events/adapter_types_pb2.py - repo: https://github.com/pycqa/flake8 diff --git a/dbt-tests-adapter/pyproject.toml b/dbt-tests-adapter/pyproject.toml index c9082d43..d2f732b7 100644 --- a/dbt-tests-adapter/pyproject.toml +++ b/dbt-tests-adapter/pyproject.toml @@ -4,7 +4,7 @@ name = "dbt-tests-adapter" description = "The set of reusable tests and test fixtures used to test common functionality" readme = "README.md" keywords = ["dbt", "adapter", "adapters", "database", "elt", "dbt-core", "dbt Core", "dbt Cloud", "dbt Labs"] -requires-python = ">=3.8.0" +requires-python = ">=3.9.0" authors = [ { name = "dbt Labs", email = "info@dbtlabs.com" }, ] @@ -17,7 +17,6 @@ classifiers = [ "Operating System :: MacOS :: MacOS X", "Operating System :: Microsoft :: Windows", "Operating System :: POSIX :: Linux", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", diff --git a/pyproject.toml b/pyproject.toml index 76ca3dee..52550fb5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ name = "dbt-adapters" description = "The set of adapter protocols and base functionality that supports integration with dbt-core" readme = "README.md" keywords = ["dbt", "adapter", "adapters", "database", "elt", "dbt-core", "dbt Core", "dbt Cloud", "dbt Labs"] -requires-python = ">=3.8.0" +requires-python = ">=3.9.0" authors = [ { name = "dbt Labs", email = "info@dbtlabs.com" }, ] @@ -17,7 +17,6 @@ classifiers = [ "Operating System :: MacOS :: MacOS X", "Operating System :: Microsoft :: Windows", "Operating System :: POSIX :: Linux", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", From 937c8c7d257055ed6cd3eaf7f7e5e01026d7b7d7 Mon Sep 17 00:00:00 2001 From: Quigley Malcolm Date: Thu, 17 Oct 2024 20:45:23 -0700 Subject: [PATCH 04/12] Always ensure valid incremental strategy (#331) Co-authored-by: Mike Alfare <13974384+mikealfare@users.noreply.github.com> --- .../unreleased/Fixes-20241016-160412.yaml | 6 +++ .../dbt/tests/adapter/basic/files.py | 7 ++++ .../tests/adapter/basic/test_incremental.py | 39 +++++++++++++++++++ .../models/incremental/incremental.sql | 5 ++- 4 files changed, 55 insertions(+), 2 deletions(-) create mode 100644 .changes/unreleased/Fixes-20241016-160412.yaml diff --git a/.changes/unreleased/Fixes-20241016-160412.yaml b/.changes/unreleased/Fixes-20241016-160412.yaml new file mode 100644 index 00000000..00a1982c --- /dev/null +++ b/.changes/unreleased/Fixes-20241016-160412.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Always validate an incremental model's `incremental_strategy` +time: 2024-10-16T16:04:12.58581-05:00 +custom: + Author: QMalcolm + Issue: "330" diff --git a/dbt-tests-adapter/dbt/tests/adapter/basic/files.py b/dbt-tests-adapter/dbt/tests/adapter/basic/files.py index 751b01a0..d0253a53 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/basic/files.py +++ b/dbt-tests-adapter/dbt/tests/adapter/basic/files.py @@ -186,6 +186,10 @@ {{ config(materialized="incremental") }} """ +config_materialized_incremental_invalid_strategy = """ + {{ config(materialized="incremental", incremental_strategy="bad_strategy") }} +""" + config_materialized_var = """ {{ config(materialized=var("materialized_var", "table"))}} """ @@ -217,3 +221,6 @@ ephemeral_view_sql = config_materialized_view + model_ephemeral ephemeral_table_sql = config_materialized_table + model_ephemeral incremental_sql = config_materialized_incremental + model_incremental +incremental_invalid_strategy_sql = ( + config_materialized_incremental_invalid_strategy + model_incremental +) diff --git a/dbt-tests-adapter/dbt/tests/adapter/basic/test_incremental.py b/dbt-tests-adapter/dbt/tests/adapter/basic/test_incremental.py index fe04a5a1..57cc4db9 100644 --- a/dbt-tests-adapter/dbt/tests/adapter/basic/test_incremental.py +++ b/dbt-tests-adapter/dbt/tests/adapter/basic/test_incremental.py @@ -86,6 +86,45 @@ def test_incremental_not_schema_change(self, project): assert run_result == RunStatus.Success +class BaseIncrementalBadStrategy: + @pytest.fixture(scope="class") + def project_config_update(self): + return {"name": "incremental"} + + @pytest.fixture(scope="class") + def models(self): + return { + "incremental.sql": files.incremental_invalid_strategy_sql, + "schema.yml": files.schema_base_yml, + } + + @pytest.fixture(scope="class") + def seeds(self): + return {"base.csv": files.seeds_base_csv, "added.csv": files.seeds_added_csv} + + @pytest.fixture(autouse=True) + def clean_up(self, project): + yield + with project.adapter.connection_named("__test"): + relation = project.adapter.Relation.create( + database=project.database, schema=project.test_schema + ) + project.adapter.drop_schema(relation) + + def test_incremental_invalid_strategy(self, project): + # seed command + results = run_dbt(["seed"]) + assert len(results) == 2 + + # try to run the incremental model, it should fail on the first attempt + results = run_dbt(["run"], expect_pass=False) + assert len(results.results) == 1 + assert ( + 'dbt could not find an incremental strategy macro with the name "get_incremental_bad_strategy_sql"' + in results.results[0].message + ) + + class Testincremental(BaseIncremental): pass diff --git a/dbt/include/global_project/macros/materializations/models/incremental/incremental.sql b/dbt/include/global_project/macros/materializations/models/incremental/incremental.sql index f932751a..41d2de26 100644 --- a/dbt/include/global_project/macros/materializations/models/incremental/incremental.sql +++ b/dbt/include/global_project/macros/materializations/models/incremental/incremental.sql @@ -32,6 +32,9 @@ {% set to_drop = [] %} + {% set incremental_strategy = config.get('incremental_strategy') or 'default' %} + {% set strategy_sql_macro_func = adapter.get_incremental_strategy_macro(context, incremental_strategy) %} + {% if existing_relation is none %} {% set build_sql = get_create_table_as_sql(False, target_relation, sql) %} {% elif full_refresh_mode %} @@ -52,9 +55,7 @@ {% endif %} {#-- Get the incremental_strategy, the macro to use for the strategy, and build the sql --#} - {% set incremental_strategy = config.get('incremental_strategy') or 'default' %} {% set incremental_predicates = config.get('predicates', none) or config.get('incremental_predicates', none) %} - {% set strategy_sql_macro_func = adapter.get_incremental_strategy_macro(context, incremental_strategy) %} {% set strategy_arg_dict = ({'target_relation': target_relation, 'temp_relation': temp_relation, 'unique_key': unique_key, 'dest_columns': dest_columns, 'incremental_predicates': incremental_predicates }) %} {% set build_sql = strategy_sql_macro_func(strategy_arg_dict) %} From 5fd5467da1df8c568423869136356ea192a95498 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Fri, 18 Oct 2024 20:31:17 -0400 Subject: [PATCH 05/12] Allow use of multiple column unique keys in snapshots (#326) Co-authored-by: Antonio Papa Co-authored-by: Colin Rogers <111200756+colin-rogers-dbt@users.noreply.github.com> --- .../unreleased/Features-20240422-081302.yaml | 6 ++ dbt/adapters/base/relation.py | 11 +++ .../materializations/snapshots/helpers.sql | 79 +++++++++++++------ .../materializations/snapshots/snapshot.sql | 18 +++-- .../materializations/snapshots/strategies.sql | 6 +- pyproject.toml | 2 +- 6 files changed, 89 insertions(+), 33 deletions(-) create mode 100644 .changes/unreleased/Features-20240422-081302.yaml diff --git a/.changes/unreleased/Features-20240422-081302.yaml b/.changes/unreleased/Features-20240422-081302.yaml new file mode 100644 index 00000000..c58e471e --- /dev/null +++ b/.changes/unreleased/Features-20240422-081302.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Allows unique_key for snapshots to take a list +time: 2024-04-22T08:13:02.937534-04:00 +custom: + Author: agpapa + Issue: "181" diff --git a/dbt/adapters/base/relation.py b/dbt/adapters/base/relation.py index 80dbd34b..ecd87345 100644 --- a/dbt/adapters/base/relation.py +++ b/dbt/adapters/base/relation.py @@ -6,6 +6,7 @@ Dict, FrozenSet, Iterator, + List, Optional, Set, Tuple, @@ -341,6 +342,16 @@ def create( ) return cls.from_dict(kwargs) + @classmethod + def scd_args(cls: Type[Self], primary_key: Union[str, List[str]], updated_at) -> List[str]: + scd_args = [] + if isinstance(primary_key, list): + scd_args.extend(primary_key) + else: + scd_args.append(primary_key) + scd_args.append(updated_at) + return scd_args + @property def can_be_renamed(self) -> bool: return self.type in self.renameable_relations diff --git a/dbt/include/global_project/macros/materializations/snapshots/helpers.sql b/dbt/include/global_project/macros/materializations/snapshots/helpers.sql index 52fdb9bf..5c2bad99 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/helpers.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/helpers.sql @@ -49,9 +49,7 @@ snapshotted_data as ( - select *, - {{ strategy.unique_key }} as dbt_unique_key - + select *, {{ unique_key_fields(strategy.unique_key) }} from {{ target_relation }} where {% if config.get('dbt_valid_to_current') %} @@ -65,9 +63,7 @@ insertions_source_data as ( - select - *, - {{ strategy.unique_key }} as dbt_unique_key, + select *, {{ unique_key_fields(strategy.unique_key) }}, {{ strategy.updated_at }} as {{ columns.dbt_updated_at }}, {{ strategy.updated_at }} as {{ columns.dbt_valid_from }}, {{ get_dbt_valid_to_current(strategy, columns) }}, @@ -78,9 +74,7 @@ updates_source_data as ( - select - *, - {{ strategy.unique_key }} as dbt_unique_key, + select *, {{ unique_key_fields(strategy.unique_key) }}, {{ strategy.updated_at }} as {{ columns.dbt_updated_at }}, {{ strategy.updated_at }} as {{ columns.dbt_valid_from }}, {{ strategy.updated_at }} as {{ columns.dbt_valid_to }} @@ -92,9 +86,7 @@ deletes_source_data as ( - select - *, - {{ strategy.unique_key }} as dbt_unique_key + select *, {{ unique_key_fields(strategy.unique_key) }} from snapshot_query ), {% endif %} @@ -106,13 +98,11 @@ source_data.* from insertions_source_data as source_data - left outer join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key - where snapshotted_data.dbt_unique_key is null - or ( - snapshotted_data.dbt_unique_key is not null - and ( - {{ strategy.row_changed }} - ) + left outer join snapshotted_data + on {{ unique_key_join_on(strategy.unique_key, "snapshotted_data", "source_data") }} + where {{ unique_key_is_null(strategy.unique_key, "snapshotted_data") }} + or ({{ unique_key_is_not_null(strategy.unique_key, "snapshotted_data") }} and {{ strategy.row_changed }}) + ) ), @@ -125,7 +115,8 @@ snapshotted_data.{{ columns.dbt_scd_id }} from updates_source_data as source_data - join snapshotted_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key + join snapshotted_data + on {{ unique_key_join_on(strategy.unique_key, "snapshotted_data", "source_data") }} where ( {{ strategy.row_changed }} ) @@ -145,8 +136,9 @@ snapshotted_data.{{ columns.dbt_scd_id }} from snapshotted_data - left join deletes_source_data as source_data on snapshotted_data.dbt_unique_key = source_data.dbt_unique_key - where source_data.dbt_unique_key is null + left join deletes_source_data as source_data + on {{ unique_key_join_on(strategy.unique_key, "snapshotted_data", "source_data") }} + where {{ unique_key_is_null(strategy.unique_key, "source_data") }} ) {%- endif %} @@ -217,8 +209,51 @@ {% endif %} {% endmacro %} + {% macro get_dbt_valid_to_current(strategy, columns) %} {% set dbt_valid_to_current = config.get('dbt_valid_to_current') or "null" %} coalesce(nullif({{ strategy.updated_at }}, {{ strategy.updated_at }}), {{dbt_valid_to_current}}) as {{ columns.dbt_valid_to }} {% endmacro %} + + +{% macro unique_key_fields(unique_key) %} + {% if unique_key | is_list %} + {% for key in unique_key %} + {{ key }} as dbt_unique_key_{{ loop.index }} + {%- if not loop.last %} , {%- endif %} + {% endfor %} + {% else %} + {{ unique_key }} as dbt_unique_key + {% endif %} +{% endmacro %} + + +{% macro unique_key_join_on(unique_key, identifier, from_identifier) %} + {% if strategy.unique_key | is_list %} + {% for key in strategy.unique_key %} + {{ identifier }}.dbt_unique_key_{{ loop.index }} = {{ from_identifier }}.dbt_unique_key_{{ loop.index }} + {%- if not loop.last %} and {%- endif %} + {% endfor %} + {% else %} + {{ identifier }}.dbt_unique_key = {{ from_identifier }}.dbt_unique_key + {% endif %} +{% endmacro %} + + +{% macro unique_key_is_null(unique_key, identifier) %} + {% if unique_key | is_list %} + {{ identifier }}.dbt_unique_key_1 is null + {% else %} + {{ identifer }}.dbt_unique_key is null + {% endif %} +{% endmacro %} + + +{% macro unique_key_is_not_null(unique_key, identifier) %} + {% if unique_key | is_list %} + {{ identifier }}.dbt_unique_key_1 is not null + {% else %} + {{ identifer }}.dbt_unique_key is not null + {% endif %} +{% endmacro %} diff --git a/dbt/include/global_project/macros/materializations/snapshots/snapshot.sql b/dbt/include/global_project/macros/materializations/snapshots/snapshot.sql index 5daead4c..0c9590b6 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/snapshot.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/snapshot.sql @@ -46,20 +46,22 @@ {% do adapter.expand_target_column_types(from_relation=staging_table, to_relation=target_relation) %} + {% set remove_columns = ['dbt_change_type', 'DBT_CHANGE_TYPE', 'dbt_unique_key', 'DBT_UNIQUE_KEY'] %} + {% if unique_key | is_list %} + {% for key in strategy.unique_key %} + {{ remove_columns.append('dbt_unique_key_' + loop.index|string) }} + {{ remove_columns.append('DBT_UNIQUE_KEY_' + loop.index|string) }} + {% endfor %} + {% endif %} + {% set missing_columns = adapter.get_missing_columns(staging_table, target_relation) - | rejectattr('name', 'equalto', 'dbt_change_type') - | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE') - | rejectattr('name', 'equalto', 'dbt_unique_key') - | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY') + | rejectattr('name', 'in', remove_columns) | list %} {% do create_columns(target_relation, missing_columns) %} {% set source_columns = adapter.get_columns_in_relation(staging_table) - | rejectattr('name', 'equalto', 'dbt_change_type') - | rejectattr('name', 'equalto', 'DBT_CHANGE_TYPE') - | rejectattr('name', 'equalto', 'dbt_unique_key') - | rejectattr('name', 'equalto', 'DBT_UNIQUE_KEY') + | rejectattr('name', 'in', remove_columns) | list %} {% set quoted_source_columns = [] %} diff --git a/dbt/include/global_project/macros/materializations/snapshots/strategies.sql b/dbt/include/global_project/macros/materializations/snapshots/strategies.sql index 8c086182..f9f5afbd 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/strategies.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/strategies.sql @@ -70,7 +70,8 @@ ({{ snapshotted_rel }}.{{ columns.dbt_valid_from }} < {{ current_rel }}.{{ updated_at }}) {%- endset %} - {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %} + {% set scd_args = api.Relation.scd_args(primary_key, updated_at) %} + {% set scd_id_expr = snapshot_hash_arguments(scd_args) %} {% do return({ "unique_key": primary_key, @@ -166,7 +167,8 @@ ) {%- endset %} - {% set scd_id_expr = snapshot_hash_arguments([primary_key, updated_at]) %} + {% set scd_args = api.Relation.scd_args(primary_key, updated_at) %} + {% set scd_id_expr = snapshot_hash_arguments(scd_args) %} {% do return({ "unique_key": primary_key, diff --git a/pyproject.toml b/pyproject.toml index 52550fb5..7a8d1a50 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,7 @@ classifiers = [ "Programming Language :: Python :: 3.12", ] dependencies = [ - "dbt-common>=1.10,<2.0", + "dbt-common>=1.11,<2.0", "pytz>=2015.7", # installed via dbt-common but used directly "agate>=1.0,<2.0", From 2bf38087bc7225dd2ffadc09fcfdc7fc781287fa Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Mon, 21 Oct 2024 10:26:55 -0400 Subject: [PATCH 06/12] Fix some typos in multiple unique_key (#335) --- .../macros/materializations/snapshots/helpers.sql | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dbt/include/global_project/macros/materializations/snapshots/helpers.sql b/dbt/include/global_project/macros/materializations/snapshots/helpers.sql index 5c2bad99..b4cd7c14 100644 --- a/dbt/include/global_project/macros/materializations/snapshots/helpers.sql +++ b/dbt/include/global_project/macros/materializations/snapshots/helpers.sql @@ -101,7 +101,7 @@ left outer join snapshotted_data on {{ unique_key_join_on(strategy.unique_key, "snapshotted_data", "source_data") }} where {{ unique_key_is_null(strategy.unique_key, "snapshotted_data") }} - or ({{ unique_key_is_not_null(strategy.unique_key, "snapshotted_data") }} and {{ strategy.row_changed }}) + or ({{ unique_key_is_not_null(strategy.unique_key, "snapshotted_data") }} and ({{ strategy.row_changed }}) ) @@ -230,8 +230,8 @@ {% macro unique_key_join_on(unique_key, identifier, from_identifier) %} - {% if strategy.unique_key | is_list %} - {% for key in strategy.unique_key %} + {% if unique_key | is_list %} + {% for key in unique_key %} {{ identifier }}.dbt_unique_key_{{ loop.index }} = {{ from_identifier }}.dbt_unique_key_{{ loop.index }} {%- if not loop.last %} and {%- endif %} {% endfor %} @@ -245,7 +245,7 @@ {% if unique_key | is_list %} {{ identifier }}.dbt_unique_key_1 is null {% else %} - {{ identifer }}.dbt_unique_key is null + {{ identifier }}.dbt_unique_key is null {% endif %} {% endmacro %} @@ -254,6 +254,6 @@ {% if unique_key | is_list %} {{ identifier }}.dbt_unique_key_1 is not null {% else %} - {{ identifer }}.dbt_unique_key is not null + {{ identifier }}.dbt_unique_key is not null {% endif %} {% endmacro %} From d6a00475936eaeb4951df9975dcff7aa3ae544a6 Mon Sep 17 00:00:00 2001 From: Github Build Bot Date: Mon, 21 Oct 2024 23:02:12 +0000 Subject: [PATCH 07/12] Bumping version to 1.7.2 and generate changelog --- .changes/1.7.2.md | 16 +++++++++++++++ .../Breaking Changes-20241016-180629.yaml | 6 ------ .../unreleased/Features-20240422-081302.yaml | 6 ------ .../unreleased/Fixes-20241016-160412.yaml | 6 ------ CHANGELOG.md | 20 +++++++++++++++++-- dbt/adapters/__about__.py | 2 +- 6 files changed, 35 insertions(+), 21 deletions(-) create mode 100644 .changes/1.7.2.md delete mode 100644 .changes/unreleased/Breaking Changes-20241016-180629.yaml delete mode 100644 .changes/unreleased/Features-20240422-081302.yaml delete mode 100644 .changes/unreleased/Fixes-20241016-160412.yaml diff --git a/.changes/1.7.2.md b/.changes/1.7.2.md new file mode 100644 index 00000000..b17bf217 --- /dev/null +++ b/.changes/1.7.2.md @@ -0,0 +1,16 @@ +## dbt-adapters 1.7.2 - October 21, 2024 + +### Breaking Changes + +- Drop support for Python 3.8 ([#332](https://github.com/dbt-labs/dbt-adapters/issues/332)) + +### Features + +- Allows unique_key for snapshots to take a list ([#181](https://github.com/dbt-labs/dbt-adapters/issues/181)) + +### Fixes + +- Always validate an incremental model's `incremental_strategy` ([#330](https://github.com/dbt-labs/dbt-adapters/issues/330)) + +### Contributors +- [@agpapa](https://github.com/agpapa) ([#181](https://github.com/dbt-labs/dbt-adapters/issues/181)) diff --git a/.changes/unreleased/Breaking Changes-20241016-180629.yaml b/.changes/unreleased/Breaking Changes-20241016-180629.yaml deleted file mode 100644 index 40f103c0..00000000 --- a/.changes/unreleased/Breaking Changes-20241016-180629.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Breaking Changes -body: Drop support for Python 3.8 -time: 2024-10-16T18:06:29.535761-04:00 -custom: - Author: mikealfare - Issue: "332" diff --git a/.changes/unreleased/Features-20240422-081302.yaml b/.changes/unreleased/Features-20240422-081302.yaml deleted file mode 100644 index c58e471e..00000000 --- a/.changes/unreleased/Features-20240422-081302.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Features -body: Allows unique_key for snapshots to take a list -time: 2024-04-22T08:13:02.937534-04:00 -custom: - Author: agpapa - Issue: "181" diff --git a/.changes/unreleased/Fixes-20241016-160412.yaml b/.changes/unreleased/Fixes-20241016-160412.yaml deleted file mode 100644 index 00a1982c..00000000 --- a/.changes/unreleased/Fixes-20241016-160412.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: Always validate an incremental model's `incremental_strategy` -time: 2024-10-16T16:04:12.58581-05:00 -custom: - Author: QMalcolm - Issue: "330" diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e9f64fc..102d50b6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,13 +15,29 @@ and is generated by [Changie](https://github.com/miniscruff/changie). ## dbt-adapters 1.10.0 - September 12, 2024 -## dbt-adapters 1.7.1 - October 15, 2024 +## dbt-adapters 1.7.2 - October 21, 2024 + +### Breaking Changes + +- Drop support for Python 3.8 ([#332](https://github.com/dbt-labs/dbt-adapters/issues/332)) ### Features -- Enable setting current value of dbt_valid_to ([#320](https://github.com/dbt-labs/dbt-adapters/issues/320)) +- Allows unique_key for snapshots to take a list ([#181](https://github.com/dbt-labs/dbt-adapters/issues/181)) + +### Fixes + +- Always validate an incremental model's `incremental_strategy` ([#330](https://github.com/dbt-labs/dbt-adapters/issues/330)) +### Contributors +- [@agpapa](https://github.com/agpapa) ([#181](https://github.com/dbt-labs/dbt-adapters/issues/181)) + + +## dbt-adapters 1.7.1 - October 15, 2024 +### Features + +- Enable setting current value of dbt_valid_to ([#320](https://github.com/dbt-labs/dbt-adapters/issues/320)) ## dbt-adapters 1.7.0 - September 19, 2024 diff --git a/dbt/adapters/__about__.py b/dbt/adapters/__about__.py index 116d5667..2196826f 100644 --- a/dbt/adapters/__about__.py +++ b/dbt/adapters/__about__.py @@ -1 +1 @@ -version = "1.7.1" +version = "1.7.2" From 951d98d02ea8ed373f06874854fb27775e2aac7f Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Mon, 28 Oct 2024 16:32:36 -0400 Subject: [PATCH 08/12] [Bug] Behavior flags should take the default value when the project file is not loaded (#338) --- .../unreleased/Fixes-20241024-154518.yaml | 6 ++ dbt/adapters/base/impl.py | 13 +-- tests/unit/behavior_flag_tests/__init__.py | 0 .../test_behavior_flags.py | 0 .../behavior_flag_tests/test_empty_project.py | 87 +++++++++++++++++++ 5 files changed, 100 insertions(+), 6 deletions(-) create mode 100644 .changes/unreleased/Fixes-20241024-154518.yaml create mode 100644 tests/unit/behavior_flag_tests/__init__.py rename tests/unit/{ => behavior_flag_tests}/test_behavior_flags.py (100%) create mode 100644 tests/unit/behavior_flag_tests/test_empty_project.py diff --git a/.changes/unreleased/Fixes-20241024-154518.yaml b/.changes/unreleased/Fixes-20241024-154518.yaml new file mode 100644 index 00000000..f1871d03 --- /dev/null +++ b/.changes/unreleased/Fixes-20241024-154518.yaml @@ -0,0 +1,6 @@ +kind: Fixes +body: Always make behavior flags available for evaluation +time: 2024-10-24T15:45:18.114936-04:00 +custom: + Author: mikealfare + Issue: "338" diff --git a/dbt/adapters/base/impl.py b/dbt/adapters/base/impl.py index f3788fe3..6bd8fc70 100644 --- a/dbt/adapters/base/impl.py +++ b/dbt/adapters/base/impl.py @@ -300,12 +300,13 @@ def behavior(self) -> Behavior: @behavior.setter # type: ignore def behavior(self, flags: List[BehaviorFlag]) -> None: flags.extend(self._behavior_flags) - try: - # we don't always get project flags, for example during `dbt debug` - self._behavior = Behavior(flags, self.config.flags) - except AttributeError: - # in that case, don't load any behavior to avoid unexpected defaults - self._behavior = Behavior([], {}) + + # we don't always get project flags, for example, the project file is not loaded during `dbt debug` + # in that case, load the default values for behavior flags to avoid compilation errors + # this mimics not loading a project file, or not specifying flags in a project file + user_overrides = getattr(self.config, "flags", {}) + + self._behavior = Behavior(flags, user_overrides) @property def _behavior_flags(self) -> List[BehaviorFlag]: diff --git a/tests/unit/behavior_flag_tests/__init__.py b/tests/unit/behavior_flag_tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/test_behavior_flags.py b/tests/unit/behavior_flag_tests/test_behavior_flags.py similarity index 100% rename from tests/unit/test_behavior_flags.py rename to tests/unit/behavior_flag_tests/test_behavior_flags.py diff --git a/tests/unit/behavior_flag_tests/test_empty_project.py b/tests/unit/behavior_flag_tests/test_empty_project.py new file mode 100644 index 00000000..f9fd7a76 --- /dev/null +++ b/tests/unit/behavior_flag_tests/test_empty_project.py @@ -0,0 +1,87 @@ +from types import SimpleNamespace +from typing import Any, Dict, List + +from dbt_common.behavior_flags import BehaviorFlag +from dbt_common.exceptions import DbtBaseException +import pytest + +from dbt.adapters.contracts.connection import AdapterRequiredConfig, QueryComment + +from tests.unit.fixtures.credentials import CredentialsStub + + +@pytest.fixture +def flags() -> Dict[str, Any]: + return { + "unregistered_flag": True, + "default_false_user_false_flag": False, + "default_false_user_true_flag": True, + "default_true_user_false_flag": False, + "default_true_user_true_flag": True, + } + + +@pytest.fixture +def config(flags) -> AdapterRequiredConfig: + raw_config = { + "credentials": CredentialsStub("test_database", "test_schema"), + "profile_name": "test_profile", + "target_name": "test_target", + "threads": 4, + "project_name": "test_project", + "query_comment": QueryComment(), + "cli_vars": {}, + "target_path": "path/to/nowhere", + "log_cache_events": False, + } + return SimpleNamespace(**raw_config) + + +@pytest.fixture +def behavior_flags() -> List[BehaviorFlag]: + return [ + { + "name": "default_false_user_false_flag", + "default": False, + "docs_url": "https://docs.com", + }, + { + "name": "default_false_user_true_flag", + "default": False, + "description": "This is a false flag.", + }, + { + "name": "default_false_user_skip_flag", + "default": False, + "description": "This is a true flag.", + }, + { + "name": "default_true_user_false_flag", + "default": True, + "description": "This is fake news.", + }, + { + "name": "default_true_user_true_flag", + "default": True, + "docs_url": "https://moar.docs.com", + }, + { + "name": "default_true_user_skip_flag", + "default": True, + "description": "This is a true flag.", + }, + ] + + +def test_register_behavior_flags(adapter): + # make sure that users cannot add arbitrary flags to this collection + with pytest.raises(DbtBaseException): + assert adapter.behavior.unregistered_flag + + # check the values of the valid behavior flags + assert not adapter.behavior.default_false_user_false_flag + assert not adapter.behavior.default_false_user_true_flag + assert not adapter.behavior.default_false_user_skip_flag + assert adapter.behavior.default_true_user_false_flag + assert adapter.behavior.default_true_user_true_flag + assert adapter.behavior.default_true_user_skip_flag From d83314dcf3fb70e4dbe49e9a10d81edb726f856e Mon Sep 17 00:00:00 2001 From: Mila Page <67295367+VersusFacit@users.noreply.github.com> Date: Mon, 28 Oct 2024 14:20:35 -0700 Subject: [PATCH 09/12] Add adapter telemetry method for core model run snowplow event. (#328) --- .../Under the Hood-20241016-033159.yaml | 6 +++++ dbt/adapters/base/__init__.py | 1 + dbt/adapters/base/impl.py | 27 +++++++++++++++++++ dbt/adapters/base/relation.py | 8 ++++++ tests/unit/test_adapter_telemetry.py | 15 +++++++++++ 5 files changed, 57 insertions(+) create mode 100644 .changes/unreleased/Under the Hood-20241016-033159.yaml create mode 100644 tests/unit/test_adapter_telemetry.py diff --git a/.changes/unreleased/Under the Hood-20241016-033159.yaml b/.changes/unreleased/Under the Hood-20241016-033159.yaml new file mode 100644 index 00000000..27e6730f --- /dev/null +++ b/.changes/unreleased/Under the Hood-20241016-033159.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Add adapter telemetry. +time: 2024-10-16T03:31:59.334011-07:00 +custom: + Author: versusfacit + Issue: "301" diff --git a/dbt/adapters/base/__init__.py b/dbt/adapters/base/__init__.py index ade1af3d..c30dd01f 100644 --- a/dbt/adapters/base/__init__.py +++ b/dbt/adapters/base/__init__.py @@ -12,4 +12,5 @@ BaseRelation, RelationType, SchemaSearchMap, + AdapterTrackingRelationInfo, ) diff --git a/dbt/adapters/base/impl.py b/dbt/adapters/base/impl.py index 6bd8fc70..41481535 100644 --- a/dbt/adapters/base/impl.py +++ b/dbt/adapters/base/impl.py @@ -4,6 +4,7 @@ from contextlib import contextmanager from datetime import datetime from enum import Enum +from importlib import import_module from multiprocessing.context import SpawnContext from typing import ( Any, @@ -61,12 +62,14 @@ ComponentName, InformationSchema, SchemaSearchMap, + AdapterTrackingRelationInfo, ) from dbt.adapters.cache import RelationsCache, _make_ref_key_dict from dbt.adapters.capability import Capability, CapabilityDict from dbt.adapters.contracts.connection import Credentials from dbt.adapters.contracts.macros import MacroResolverProtocol from dbt.adapters.contracts.relation import RelationConfig + from dbt.adapters.events.types import ( CacheMiss, CatalogGenerationError, @@ -1744,6 +1747,30 @@ def capabilities(cls) -> CapabilityDict: def supports(cls, capability: Capability) -> bool: return bool(cls.capabilities()[capability]) + @classmethod + def get_adapter_run_info(cls, config: RelationConfig) -> AdapterTrackingRelationInfo: + adapter_class_name, *_ = cls.__name__.split("Adapter") + adapter_name = adapter_class_name.lower() + + if adapter_name == "base": + adapter_version = "" + else: + adapter_version = import_module(f"dbt.adapters.{adapter_name}.__version__").version + + return AdapterTrackingRelationInfo( + adapter_name=adapter_name, + base_adapter_version=import_module("dbt.adapters.__about__").version, + adapter_version=adapter_version, + model_adapter_details=cls._get_adapter_specific_run_info(config), + ) + + @classmethod + def _get_adapter_specific_run_info(cls, config) -> Dict[str, Any]: + """ + Adapter maintainers should overwrite this method to return any run metadata that should be captured during a run. + """ + return {} + COLUMNS_EQUAL_SQL = """ with diff_count as ( diff --git a/dbt/adapters/base/relation.py b/dbt/adapters/base/relation.py index ecd87345..7d4888e4 100644 --- a/dbt/adapters/base/relation.py +++ b/dbt/adapters/base/relation.py @@ -542,3 +542,11 @@ def flatten(self, allow_multiple_databases: bool = False) -> "SchemaSearchMap": ) return new + + +@dataclass(frozen=True, eq=False, repr=False) +class AdapterTrackingRelationInfo(FakeAPIObject, Hashable): + adapter_name: str + base_adapter_version: str + adapter_version: str + model_adapter_details: Any diff --git a/tests/unit/test_adapter_telemetry.py b/tests/unit/test_adapter_telemetry.py new file mode 100644 index 00000000..1d5c4911 --- /dev/null +++ b/tests/unit/test_adapter_telemetry.py @@ -0,0 +1,15 @@ +import dbt.adapters.__about__ + +from dbt.adapters.base.impl import BaseAdapter +from dbt.adapters.base.relation import AdapterTrackingRelationInfo + + +def test_telemetry_returns(): + res = BaseAdapter.get_adapter_run_info({}) + + assert res.adapter_name == "base" + assert res.base_adapter_version == dbt.adapters.__about__.version + assert res.adapter_version == "" + assert res.model_adapter_details == {} + + assert type(res) is AdapterTrackingRelationInfo From 4d7c9a11b40a1e3d1830ab765f1efd112e02cf50 Mon Sep 17 00:00:00 2001 From: Github Build Bot Date: Tue, 29 Oct 2024 18:07:56 +0000 Subject: [PATCH 10/12] Bumping version to 1.8.0 and generate changelog --- .changes/1.8.0.md | 9 +++++++++ .changes/unreleased/Fixes-20241024-154518.yaml | 6 ------ .../unreleased/Under the Hood-20241016-033159.yaml | 6 ------ CHANGELOG.md | 13 ++++++++++++- dbt/adapters/__about__.py | 2 +- 5 files changed, 22 insertions(+), 14 deletions(-) create mode 100644 .changes/1.8.0.md delete mode 100644 .changes/unreleased/Fixes-20241024-154518.yaml delete mode 100644 .changes/unreleased/Under the Hood-20241016-033159.yaml diff --git a/.changes/1.8.0.md b/.changes/1.8.0.md new file mode 100644 index 00000000..f73a0300 --- /dev/null +++ b/.changes/1.8.0.md @@ -0,0 +1,9 @@ +## dbt-adapters 1.8.0 - October 29, 2024 + +### Fixes + +- Always make behavior flags available for evaluation ([#338](https://github.com/dbt-labs/dbt-adapters/issues/338)) + +### Under the Hood + +- Add adapter telemetry. ([#301](https://github.com/dbt-labs/dbt-adapters/issues/301)) diff --git a/.changes/unreleased/Fixes-20241024-154518.yaml b/.changes/unreleased/Fixes-20241024-154518.yaml deleted file mode 100644 index f1871d03..00000000 --- a/.changes/unreleased/Fixes-20241024-154518.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Fixes -body: Always make behavior flags available for evaluation -time: 2024-10-24T15:45:18.114936-04:00 -custom: - Author: mikealfare - Issue: "338" diff --git a/.changes/unreleased/Under the Hood-20241016-033159.yaml b/.changes/unreleased/Under the Hood-20241016-033159.yaml deleted file mode 100644 index 27e6730f..00000000 --- a/.changes/unreleased/Under the Hood-20241016-033159.yaml +++ /dev/null @@ -1,6 +0,0 @@ -kind: Under the Hood -body: Add adapter telemetry. -time: 2024-10-16T03:31:59.334011-07:00 -custom: - Author: versusfacit - Issue: "301" diff --git a/CHANGELOG.md b/CHANGELOG.md index 102d50b6..63f9e59b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,18 @@ and is generated by [Changie](https://github.com/miniscruff/changie). ## dbt-adapters 1.10.0 - September 12, 2024 +## dbt-adapters 1.8.0 - October 29, 2024 + +### Fixes + +- Always make behavior flags available for evaluation ([#338](https://github.com/dbt-labs/dbt-adapters/issues/338)) + +### Under the Hood + +- Add adapter telemetry. ([#301](https://github.com/dbt-labs/dbt-adapters/issues/301)) + + + ## dbt-adapters 1.7.2 - October 21, 2024 ### Breaking Changes @@ -32,7 +44,6 @@ and is generated by [Changie](https://github.com/miniscruff/changie). ### Contributors - [@agpapa](https://github.com/agpapa) ([#181](https://github.com/dbt-labs/dbt-adapters/issues/181)) - ## dbt-adapters 1.7.1 - October 15, 2024 ### Features diff --git a/dbt/adapters/__about__.py b/dbt/adapters/__about__.py index 2196826f..6aaa73b8 100644 --- a/dbt/adapters/__about__.py +++ b/dbt/adapters/__about__.py @@ -1 +1 @@ -version = "1.7.2" +version = "1.8.0" From 05f5fb055152d751a1df6464499a40623235d584 Mon Sep 17 00:00:00 2001 From: Github Build Bot Date: Tue, 29 Oct 2024 18:17:03 +0000 Subject: [PATCH 11/12] Bumping version to 1.10.3 and generate changelog --- .changes/1.10.3.md | 1 + CHANGELOG.md | 6 ++++-- dbt-tests-adapter/dbt/tests/__about__.py | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) create mode 100644 .changes/1.10.3.md diff --git a/.changes/1.10.3.md b/.changes/1.10.3.md new file mode 100644 index 00000000..29844ce2 --- /dev/null +++ b/.changes/1.10.3.md @@ -0,0 +1 @@ +## dbt-adapters 1.10.3 - October 29, 2024 diff --git a/CHANGELOG.md b/CHANGELOG.md index 63f9e59b..9971e5ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html), and is generated by [Changie](https://github.com/miniscruff/changie). +## dbt-adapters 1.10.3 - October 29, 2024 + + + ## dbt-adapters 1.10.2 - October 01, 2024 ### Under the Hood @@ -25,8 +29,6 @@ and is generated by [Changie](https://github.com/miniscruff/changie). - Add adapter telemetry. ([#301](https://github.com/dbt-labs/dbt-adapters/issues/301)) - - ## dbt-adapters 1.7.2 - October 21, 2024 ### Breaking Changes diff --git a/dbt-tests-adapter/dbt/tests/__about__.py b/dbt-tests-adapter/dbt/tests/__about__.py index 8c657eec..977620c3 100644 --- a/dbt-tests-adapter/dbt/tests/__about__.py +++ b/dbt-tests-adapter/dbt/tests/__about__.py @@ -1 +1 @@ -version = "1.10.2" +version = "1.10.3" From 5812a2ea6b1d9d538108102f2722fa7a161729a0 Mon Sep 17 00:00:00 2001 From: Colin Date: Thu, 31 Oct 2024 16:10:45 -0700 Subject: [PATCH 12/12] merge main --- dbt/adapters/contracts/connection.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dbt/adapters/contracts/connection.py b/dbt/adapters/contracts/connection.py index e3baf284..f3c4647a 100644 --- a/dbt/adapters/contracts/connection.py +++ b/dbt/adapters/contracts/connection.py @@ -226,3 +226,4 @@ class AdapterRequiredConfig(HasCredentials, Protocol): cli_vars: Dict[str, Any] target_path: str log_cache_events: bool + catalogs = Optional[ExternalCatalogConfig]