Null-safe equality for unique_key within delete+insert and merge incremental strategies

.changes/unreleased/Fixes-20240227105926.yaml

@@ -0,0 +1,7 @@
+kind: Fixes
+body: Fix incremental merge to use an `is null` check to allow for handling of nullable
+  fields in the unique_key
+time: 2024-02-27T10:59:59.202000+00:00
+custom:
+  Author: amardatar
+  Issue: "7597"

dbt-tests-adapter/dbt/tests/adapter/incremental/test_incremental_unique_id.py

@@ -240,6 +240,8 @@
 select 'NY','New York','Manhattan','2021-04-01'
 union all
 select 'PA','Philadelphia','Philadelphia','2021-05-21'
+union all
+select 'CO','Denver',null,'2021-06-18'
 
 """
 
@@ -265,6 +267,8 @@
 select 'NY','New York','Manhattan','2021-04-01'
 union all
 select 'PA','Philadelphia','Philadelphia','2021-05-21'
+union all
+select 'CO','Denver',null,'2021-06-18'
 
 """
 
@@ -288,6 +292,7 @@
 NY,Kings,Brooklyn,2021-04-02
 NY,New York,Manhattan,2021-04-01
 PA,Philadelphia,Philadelphia,2021-05-21
+CO,Denver,,2021-06-18
 """
 
 seeds__add_new_rows_sql = """
@@ -439,7 +444,7 @@ def fail_to_build_inc_missing_unique_key_column(self, incremental_model_name):
     def test__no_unique_keys(self, project):
         """with no unique keys, seed and model should match"""
 
-        expected_fields = self.get_expected_fields(relation="seed", seed_rows=8)
+        expected_fields = self.get_expected_fields(relation="seed", seed_rows=9)
         test_case_fields = self.get_test_fields(
             project, seed="seed", incremental_model="no_unique_key", update_sql_file="add_new_rows"
         )
@@ -449,7 +454,7 @@ def test__no_unique_keys(self, project):
     def test__empty_str_unique_key(self, project):
         """with empty string for unique key, seed and model should match"""
 
-        expected_fields = self.get_expected_fields(relation="seed", seed_rows=8)
+        expected_fields = self.get_expected_fields(relation="seed", seed_rows=9)
         test_case_fields = self.get_test_fields(
             project,
             seed="seed",
@@ -462,7 +467,7 @@ def test__one_unique_key(self, project):
         """with one unique key, model will overwrite existing row"""
 
         expected_fields = self.get_expected_fields(
-            relation="one_str__overwrite", seed_rows=7, opt_model_count=1
+            relation="one_str__overwrite", seed_rows=8, opt_model_count=1
         )
         test_case_fields = self.get_test_fields(
             project,
@@ -487,7 +492,7 @@ def test__bad_unique_key(self, project):
     def test__empty_unique_key_list(self, project):
         """with no unique keys, seed and model should match"""
 
-        expected_fields = self.get_expected_fields(relation="seed", seed_rows=8)
+        expected_fields = self.get_expected_fields(relation="seed", seed_rows=9)
         test_case_fields = self.get_test_fields(
             project,
             seed="seed",
@@ -500,7 +505,7 @@ def test__unary_unique_key_list(self, project):
         """with one unique key, model will overwrite existing row"""
 
         expected_fields = self.get_expected_fields(
-            relation="unique_key_list__inplace_overwrite", seed_rows=7, opt_model_count=1
+            relation="unique_key_list__inplace_overwrite", seed_rows=8, opt_model_count=1
         )
         test_case_fields = self.get_test_fields(
             project,
@@ -515,7 +520,7 @@ def test__duplicated_unary_unique_key_list(self, project):
         """with two of the same unique key, model will overwrite existing row"""
 
         expected_fields = self.get_expected_fields(
-            relation="unique_key_list__inplace_overwrite", seed_rows=7, opt_model_count=1
+            relation="unique_key_list__inplace_overwrite", seed_rows=8, opt_model_count=1
         )
         test_case_fields = self.get_test_fields(
             project,
@@ -530,7 +535,7 @@ def test__trinary_unique_key_list(self, project):
         """with three unique keys, model will overwrite existing row"""
 
         expected_fields = self.get_expected_fields(
-            relation="unique_key_list__inplace_overwrite", seed_rows=7, opt_model_count=1
+            relation="unique_key_list__inplace_overwrite", seed_rows=8, opt_model_count=1
         )
         test_case_fields = self.get_test_fields(
             project,
@@ -545,7 +550,7 @@ def test__trinary_unique_key_list_no_update(self, project):
         """even with three unique keys, adding distinct rows to seed does not
         cause seed and model to diverge"""
 
-        expected_fields = self.get_expected_fields(relation="seed", seed_rows=8)
+        expected_fields = self.get_expected_fields(relation="seed", seed_rows=9)
         test_case_fields = self.get_test_fields(
             project,
             seed="seed",

dbt/include/global_project/macros/materializations/models/incremental/merge.sql

@@ -16,13 +16,13 @@
         {% if unique_key is sequence and unique_key is not mapping and unique_key is not string %}
             {% for key in unique_key %}
                 {% set this_key_match %}
-                    DBT_INTERNAL_SOURCE.{{ key }} = DBT_INTERNAL_DEST.{{ key }}
+                    DBT_INTERNAL_SOURCE.{{ key }} = DBT_INTERNAL_DEST.{{ key }} or (DBT_INTERNAL_SOURCE.{{ key }} is null and DBT_INTERNAL_DEST.{{ key }} is null)
                 {% endset %}
                 {% do predicates.append(this_key_match) %}
             {% endfor %}
         {% else %}
             {% set unique_key_match %}
-                DBT_INTERNAL_SOURCE.{{ unique_key }} = DBT_INTERNAL_DEST.{{ unique_key }}
+                DBT_INTERNAL_SOURCE.{{ unique_key }} = DBT_INTERNAL_DEST.{{ unique_key }} or (DBT_INTERNAL_SOURCE.{{ unique_key }} is null and DBT_INTERNAL_DEST.{{ unique_key }} is null)
             {% endset %}
             {% do predicates.append(unique_key_match) %}
         {% endif %}
@@ -62,12 +62,12 @@
 
     {% if unique_key %}
         {% if unique_key is sequence and unique_key is not string %}
-            delete from {{target }}
+            delete from {{ target }}
             using {{ source }}
             where (
                 {% for key in unique_key %}
-                    {{ source }}.{{ key }} = {{ target }}.{{ key }}
-                    {{ "and " if not loop.last}}
+                    ({{ source }}.{{ key }} = {{ target }}.{{ key }} or ({{ source }}.{{ key }} is null and {{ target }}.{{ key }} is null))
+                    {{ "and " if not loop.last }}
                 {% endfor %}
                 {% if incremental_predicates %}
                     {% for predicate in incremental_predicates %}