From 9f7565e21e25f5650b5e31a563c9e9cda7b06bbf Mon Sep 17 00:00:00 2001 From: Cynthia Yin Date: Tue, 19 Sep 2023 14:16:46 -0700 Subject: [PATCH 1/2] clean up --- .../bigquery/BigQueryReservedKeywords.java | 113 ------------------ .../snowflake/SnowflakeReservedKeywords.java | 113 ------------------ .../SnowflakeSqlGenerator.java | 14 ++- 3 files changed, 12 insertions(+), 228 deletions(-) delete mode 100644 airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryReservedKeywords.java delete mode 100644 airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeReservedKeywords.java diff --git a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryReservedKeywords.java b/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryReservedKeywords.java deleted file mode 100644 index 308a002244b7..000000000000 --- a/airbyte-integrations/connectors/destination-bigquery/src/main/java/io/airbyte/integrations/destination/bigquery/BigQueryReservedKeywords.java +++ /dev/null @@ -1,113 +0,0 @@ -package io.airbyte.integrations.destination.bigquery; - -import com.google.common.collect.ImmutableList; -import java.util.List; - -/** - * NOTE: This class is not used, but is created for completeness. - * See https://cloud.google.com/bigquery/docs/reference/standard-sql/lexical#reserved_keywords - * Copied from https://github.com/airbytehq/airbyte/blob/f226503bd1d4cd9c7412b04d47de584523988443/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/reserved_keywords.py - */ -public class BigQueryReservedKeywords { - - public static final List RESERVED_KEYWORDS = ImmutableList.of( - "ALL", - "AND", - "ANY", - "ARRAY", - "AS", - "ASC", - "ASSERT_ROWS_MODIFIED", - "AT", - "BETWEEN", - "BY", - "CASE", - "CAST", - "COLLATE", - "CONTAINS", - "CREATE", - "CROSS", - "CUBE", - "CURRENT", - "CURRENT_DATE", - "CURRENT_TIME", - "CURRENT_TIMESTAMP", - "DEFAULT", - "DEFINE", - "DESC", - "DISTINCT", - "ELSE", - "END", - "ENUM", - "ESCAPE", - "EXCEPT", - "EXCLUDE", - "EXISTS", - "EXTRACT", - "FALSE", - "FETCH", - "FOLLOWING", - "FOR", - "FROM", - "FULL", - "GROUP", - "GROUPING", - "GROUPS", - "HASH", - "HAVING", - "IF", - "IGNORE", - "IN", - "INNER", - "INTERSECT", - "INTERVAL", - "INTO", - "IS", - "JOIN", - "LATERAL", - "LEFT", - "LIKE", - "LIMIT", - "LOOKUP", - "MERGE", - "NATURAL", - "NEW", - "NO", - "NOT", - "NULL", - "NULLS", - "OF", - "ON", - "OR", - "ORDER", - "OUTER", - "OVER", - "PARTITION", - "PRECEDING", - "PROTO", - "RANGE", - "RECURSIVE", - "RESPECT", - "RIGHT", - "ROLLUP", - "ROWS", - "SELECT", - "SET", - "SOME", - "STRUCT", - "TABLESAMPLE", - "THEN", - "TO", - "TREAT", - "TRUE", - "UNBOUNDED", - "UNION", - "UNNEST", - "USING", - "WHEN", - "WHERE", - "WINDOW", - "WITH", - "WITHIN"); - -} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeReservedKeywords.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeReservedKeywords.java deleted file mode 100644 index 85ff6851b373..000000000000 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/SnowflakeReservedKeywords.java +++ /dev/null @@ -1,113 +0,0 @@ -package io.airbyte.integrations.destination.snowflake; - -import com.google.common.collect.ImmutableList; -import java.util.List; - -/** - * See https://docs.snowflake.com/en/sql-reference/reserved-keywords.html - * Copied from https://github.com/airbytehq/airbyte/blob/f226503bd1d4cd9c7412b04d47de584523988443/airbyte-integrations/bases/base-normalization/normalization/transform_catalog/reserved_keywords.py - */ -public class SnowflakeReservedKeywords { - - public static final List RESERVED_KEYWORDS = ImmutableList.of( - "ALL", - "ALTER", - "AND", - "ANY", - "AS", - "BETWEEN", - "BY", - "CASE", - "CAST", - "CHECK", - "COLUMN", - "CONNECT", - "CONNECTION", - "CONSTRAINT", - "CREATE", - "CROSS", - "CURRENT", - "CURRENT_DATE", - "CURRENT_TIME", - "CURRENT_TIMESTAMP", - "CURRENT_USER", - "DATABASE", - "DEFAULT", - "DELETE", - "DISTINCT", - "DROP", - "ELSE", - "EXISTS", - "FALSE", - "FOLLOWING", - "FOR", - "FROM", - "FULL", - "GRANT", - "GROUP", - "GSCLUSTER", - "HAVING", - "ILIKE", - "IN", - "INCREMENT", - "INNER", - "INSERT", - "INTERSECT", - "INTO", - "IS", - "ISSUE", - "JOIN", - "LATERAL", - "LEFT", - "LIKE", - "LOCALTIME", - "LOCALTIMESTAMP", - "MINUS", - "NATURAL", - "NOT", - "NULL", - "OF", - "ON", - "OR", - "ORDER", - "ORGANIZATION", - "QUALIFY", - "REGEXP", - "REVOKE", - "RIGHT", - "RLIKE", - "ROW", - "ROWS", - "SAMPLE", - "SCHEMA", - "SELECT", - "SET", - "SOME", - "START", - "TABLE", - "TABLESAMPLE", - "THEN", - "TO", - "TRIGGER", - "TRUE", - "TRY_CAST", - "UNION", - "UNIQUE", - "UPDATE", - "USING", - "VALUES", - "VIEW", - "WHEN", - "WHENEVER", - "WHERE", - "WITH"); - - public static final List RESERVED_COLUMN_NAMES = ImmutableList.of( - "CURRENT_DATE", - "CURRENT_TIME", - "CURRENT_TIMESTAMP", - "CURRENT_USER", - "LOCALTIME", - "LOCALTIMESTAMP"); - -} diff --git a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGenerator.java b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGenerator.java index 57ca3db3a7f4..7138ce1abb4d 100644 --- a/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGenerator.java +++ b/airbyte-integrations/connectors/destination-snowflake/src/main/java/io/airbyte/integrations/destination/snowflake/typing_deduping/SnowflakeSqlGenerator.java @@ -7,6 +7,7 @@ import static java.util.stream.Collectors.joining; import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; import io.airbyte.integrations.base.JavaBaseConstants; import io.airbyte.integrations.base.destination.typing_deduping.AirbyteProtocolType; import io.airbyte.integrations.base.destination.typing_deduping.AirbyteType; @@ -19,7 +20,6 @@ import io.airbyte.integrations.base.destination.typing_deduping.TableNotMigratedException; import io.airbyte.integrations.base.destination.typing_deduping.Union; import io.airbyte.integrations.base.destination.typing_deduping.UnsupportedOneOf; -import io.airbyte.integrations.destination.snowflake.SnowflakeReservedKeywords; import io.airbyte.protocol.models.v0.DestinationSyncMode; import java.util.Arrays; import java.util.LinkedHashMap; @@ -35,6 +35,16 @@ public class SnowflakeSqlGenerator implements SqlGenerator RESERVED_COLUMN_NAMES = ImmutableList.of( + "CURRENT_DATE", + "CURRENT_TIME", + "CURRENT_TIMESTAMP", + "CURRENT_USER", + "LOCALTIME", + "LOCALTIMESTAMP"); + @Override public StreamId buildStreamId(final String namespace, final String name, final String rawNamespaceOverride) { // No escaping needed, as far as I can tell. We quote all our identifier names. @@ -582,7 +592,7 @@ public static String escapeSqlIdentifier(String identifier) { } private static String prefixReservedColumnName(final String columnName) { - return SnowflakeReservedKeywords.RESERVED_COLUMN_NAMES.stream().anyMatch(k -> k.equalsIgnoreCase(columnName)) ? + return RESERVED_COLUMN_NAMES.stream().anyMatch(k -> k.equalsIgnoreCase(columnName)) ? "_" + columnName : columnName; } From 67707224e811df106def9b6fc91305355f051847 Mon Sep 17 00:00:00 2001 From: Cynthia Yin Date: Tue, 19 Sep 2023 14:26:42 -0700 Subject: [PATCH 2/2] version bump --- airbyte-integrations/connectors/destination-bigquery/Dockerfile | 2 +- .../connectors/destination-bigquery/metadata.yaml | 2 +- .../connectors/destination-snowflake/Dockerfile | 2 +- .../connectors/destination-snowflake/metadata.yaml | 2 +- docs/integrations/destinations/bigquery.md | 1 + docs/integrations/destinations/snowflake.md | 1 + 6 files changed, 6 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/connectors/destination-bigquery/Dockerfile b/airbyte-integrations/connectors/destination-bigquery/Dockerfile index 3222e52b9371..13ba7f8fc78f 100644 --- a/airbyte-integrations/connectors/destination-bigquery/Dockerfile +++ b/airbyte-integrations/connectors/destination-bigquery/Dockerfile @@ -25,5 +25,5 @@ ENV AIRBYTE_NORMALIZATION_INTEGRATION bigquery COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=2.0.12 +LABEL io.airbyte.version=2.0.13 LABEL io.airbyte.name=airbyte/destination-bigquery diff --git a/airbyte-integrations/connectors/destination-bigquery/metadata.yaml b/airbyte-integrations/connectors/destination-bigquery/metadata.yaml index 4584e6fbacee..f95fe8e7c67b 100644 --- a/airbyte-integrations/connectors/destination-bigquery/metadata.yaml +++ b/airbyte-integrations/connectors/destination-bigquery/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: database connectorType: destination definitionId: 22f6c74f-5699-40ff-833c-4a879ea40133 - dockerImageTag: 2.0.12 + dockerImageTag: 2.0.13 dockerRepository: airbyte/destination-bigquery githubIssueLabel: destination-bigquery icon: bigquery.svg diff --git a/airbyte-integrations/connectors/destination-snowflake/Dockerfile b/airbyte-integrations/connectors/destination-snowflake/Dockerfile index c5f4e900053c..87a8e11790f7 100644 --- a/airbyte-integrations/connectors/destination-snowflake/Dockerfile +++ b/airbyte-integrations/connectors/destination-snowflake/Dockerfile @@ -29,5 +29,5 @@ RUN tar xf ${APPLICATION}.tar --strip-components=1 ENV ENABLE_SENTRY true -LABEL io.airbyte.version=3.1.10 +LABEL io.airbyte.version=3.1.11 LABEL io.airbyte.name=airbyte/destination-snowflake diff --git a/airbyte-integrations/connectors/destination-snowflake/metadata.yaml b/airbyte-integrations/connectors/destination-snowflake/metadata.yaml index a036d443e96e..328a81b7101e 100644 --- a/airbyte-integrations/connectors/destination-snowflake/metadata.yaml +++ b/airbyte-integrations/connectors/destination-snowflake/metadata.yaml @@ -2,7 +2,7 @@ data: connectorSubtype: database connectorType: destination definitionId: 424892c4-daac-4491-b35d-c6688ba547ba - dockerImageTag: 3.1.10 + dockerImageTag: 3.1.11 dockerRepository: airbyte/destination-snowflake githubIssueLabel: destination-snowflake icon: snowflake.svg diff --git a/docs/integrations/destinations/bigquery.md b/docs/integrations/destinations/bigquery.md index a87ddf628549..74e453914082 100644 --- a/docs/integrations/destinations/bigquery.md +++ b/docs/integrations/destinations/bigquery.md @@ -133,6 +133,7 @@ Now that you have set up the BigQuery destination connector, check out the follo | Version | Date | Pull Request | Subject | |:--------|:-----------|:-----------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 2.0.13 | 2023-09-19 | [\#30592](https://github.com/airbytehq/airbyte/pull/30592) | Internal code changes | | 2.0.12 | 2023-09-19 | [\#30319](https://github.com/airbytehq/airbyte/pull/30319) | Improved testing | | 2.0.11 | 2023-09-18 | [\#30551](https://github.com/airbytehq/airbyte/pull/30551) | GCS Staging is first loading method option | | 2.0.10 | 2023-09-15 | [\#30491](https://github.com/airbytehq/airbyte/pull/30491) | Improve error message display | diff --git a/docs/integrations/destinations/snowflake.md b/docs/integrations/destinations/snowflake.md index af9c3ffa6bef..e2f2fb088122 100644 --- a/docs/integrations/destinations/snowflake.md +++ b/docs/integrations/destinations/snowflake.md @@ -271,6 +271,7 @@ Otherwise, make sure to grant the role the required permissions in the desired n | Version | Date | Pull Request | Subject | |:----------------|:-----------|:-----------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 3.1.11 | 2023-09-19 | [\#30592](https://github.com/airbytehq/airbyte/pull/30592) | Internal code changes | | 3.1.10 | 2023-09-18 | [\#30546](https://github.com/airbytehq/airbyte/pull/30546) | Make sure that the async buffer are flush every 5 minutes | | 3.1.9 | 2023-09-19 | [\#30319](https://github.com/airbytehq/airbyte/pull/30319) | Support column names that are reserved | | 3.1.8 | 2023-09-18 | [\#30479](https://github.com/airbytehq/airbyte/pull/30479) | Fix async memory management |