Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

✨ ✨ Source Hubspot: unnest data #30322

Merged
merged 13 commits into from
Sep 27, 2023
Merged

✨ ✨ Source Hubspot: unnest data #30322

Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
f72be6e
unnest data
davydov-d Sep 11, 2023
f5e6880
update changelog
davydov-d Sep 11, 2023
a3184a0
fix migration guide
davydov-d Sep 12, 2023
dde64d7
fix breaking change message
davydov-d Sep 19, 2023
c8c1a71
Merge branch 'master' into ddavydov/#29345-source-hubspot-unnest-data
davydov-d Sep 19, 2023
ffacd98
update expected records
davydov-d Sep 19, 2023
055c9a5
Merge branch 'ddavydov/#29345-source-hubspot-unnest-data' of github.c…
davydov-d Sep 19, 2023
7dfc1bf
update expected records
davydov-d Sep 19, 2023
79d3f4c
Merge branch 'master' into ddavydov/#29345-source-hubspot-unnest-data
davydov-d Sep 25, 2023
9f11812
duplicate unnested data
davydov-d Sep 25, 2023
6e72ef3
add unit test and docstring
davydov-d Sep 25, 2023
f299c9b
update expected records
davydov-d Sep 25, 2023
f2452ad
update docs
davydov-d Sep 25, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion airbyte-integrations/connectors/source-hubspot/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,5 +34,5 @@ COPY source_hubspot ./source_hubspot
ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]

LABEL io.airbyte.version=1.4.1
LABEL io.airbyte.version=2.0.0
LABEL io.airbyte.name=airbyte/source-hubspot
62 changes: 32 additions & 30 deletions airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ acceptance_tests:
discovery:
tests:
- config_path: secrets/config_oauth.json
backward_compatibility_tests_config:
disable_for_version: 1.4.1
basic_read:
tests:
- config_path: secrets/config_oauth.json
Expand Down Expand Up @@ -48,64 +50,64 @@ acceptance_tests:
- name: ilsFilterBranch
bypass_reason: Floating fields order
companies:
- name: properties/hs_time_*
- name: properties_hs_time_*
bypass_reason: Hubspot time depend on current time
- name: properties/hs_was_imported
- name: properties_hs_was_imported
bypass_reason: attribute is not stable
contacts:
- name: properties/hs_time_*
- name: properties_hs_time_*
bypass_reason: Hubspot time depend on current time
- name: properties/hs_time_in_subscriber
- name: properties_hs_time_in_subscriber
bypass_reason: Hubspot time depend on current time
- name: properties/hs_latest_source_timestamp
- name: properties_hs_latest_source_timestamp
bypass_reason: Hubspot time depend on current time
- name: properties/hs_predictivescoringtier
- name: properties_hs_predictivescoringtier
bypass_reason: Hubspot prediction changes
- name: properties/lastmodifieddate
- name: properties_lastmodifieddate
bypass_reason: Hubspot time depend on current time
- name: properties/hs_time_in_lead
- name: properties_hs_time_in_lead
bypass_reason: Hubspot time depend on current time
- name: properties/hs_time_in_opportunity
- name: properties_hs_time_in_opportunity
bypass_reason: Hubspot time depend on current time
- name: properties/hs_was_imported
- name: properties_hs_was_imported
bypass_reason: attribute is not stable
- name: updatedAt
bypass_reason: Hubspot time depend on current time
deals:
- name: properties/hs_time_*
- name: properties_hs_time_*
bypass_reason: Hubspot time depend on current time
- name: properties/hs_acv
- name: properties_hs_acv
bypass_reason: value can be an integer or float
- name: properties/hs_arr
- name: properties_hs_arr
bypass_reason: value can be an integer or float
- name: properties/hs_mrr
- name: properties_hs_mrr
bypass_reason: value can be an integer or float
- name: properties/hs_tcv
- name: properties_hs_tcv
bypass_reason: value can be an integer or float
- name: properties/hs_num_of_associated_line_items
- name: properties_hs_num_of_associated_line_items
bypass_reason: value can be an integer or float
deals_archived:
- name: properties/hs_time_*
- name: properties_hs_time_*
bypass_reason: Hubspot time depend on current time
- name: properties/hs_acv
- name: properties_hs_acv
bypass_reason: value can be an integer or float
- name: properties/hs_arr
- name: properties_hs_arr
bypass_reason: value can be an integer or float
- name: properties/hs_mrr
- name: properties_hs_mrr
bypass_reason: value can be an integer or float
- name: properties/hs_tcv
- name: properties_hs_tcv
bypass_reason: value can be an integer or float
- name: properties/hs_num_of_associated_line_items
- name: properties_hs_num_of_associated_line_items
bypass_reason: value can be an integer or float
tickets:
- name: properties/hs_time_*
- name: properties_hs_time_*
bypass_reason: Hubspot time depend on current time
goals:
- name: properties/hs_time_*
- name: properties_hs_time_*
bypass_reason: Hubspot time depend on current time
- name: properties/hs_lastmodifieddate
- name: properties_hs_lastmodifieddate
bypass_reason: Hubspot time depend on current time
- name: properties/hs_kpi_value_last_calculated_at
- name: properties_hs_kpi_value_last_calculated_at
bypass_reason: Hubspot time depend on current time
- name: updatedAt
bypass_reason: field changes too often
Expand All @@ -120,16 +122,16 @@ acceptance_tests:
- name: ilsFilterBranch
bypass_reason: Floating fields order
companies:
- name: properties/hs_time_*
- name: properties_hs_time_*
bypass_reason: Hubspot time depend on current time
contacts:
- name: properties/hs_time_*
- name: properties_hs_time_*
bypass_reason: Hubspot time depend on current time
deals:
- name: properties/hs_time_*
- name: properties_hs_time_*
bypass_reason: Hubspot time depend on current time
tickets:
- name: properties/hs_time_*
- name: properties_hs_time_*
bypass_reason: Hubspot time depend on current time
incremental:
tests:
Expand Down
74 changes: 37 additions & 37 deletions airbyte-integrations/connectors/source-hubspot/integration_tests/expected_records.jsonl
View file
Open in desktop

Large diffs are not rendered by default.

8 changes: 6 additions & 2 deletions airbyte-integrations/connectors/source-hubspot/metadata.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ data:
connectorSubtype: api
connectorType: source
definitionId: 36c891d9-4bd9-43ac-bad2-10e12756272c
dockerImageTag: 1.4.1
dockerImageTag: 2.0.0
dockerRepository: airbyte/source-hubspot
githubIssueLabel: source-hubspot
icon: hubspot.svg
Expand All @@ -14,10 +14,14 @@ data:
registries:
cloud:
enabled: true
dockerImageTag: 1.4.1
oss:
enabled: true
releaseStage: generally_available
releases:
breakingChanges:
2.0.0:
message: "Version 2.0.0 changes the schema for most of the Hubspot streams. This is done to unnest deeply nested data so it is available as a column in the new Destinations V2 regarding normalization is no more supported. A schema refresh of all the streams is required as well as resetting the data to use the new version."
upgradeDeadline: "2023-10-01"
davydov-d marked this conversation as resolved.
Show resolved Hide resolved
suggestedStreams:
streams:
- contacts
Expand Down
99 changes: 47 additions & 52 deletions airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/campaigns.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,58 +11,53 @@
"contentId": {
"type": ["null", "integer"]
},
"counters": {
"type": ["null", "object"],
"properties": {
"open": {
"type": ["null", "integer"]
},
"processed": {
"type": ["null", "integer"]
},
"sent": {
"type": ["null", "integer"]
},
"deferred": {
"type": ["null", "integer"]
},
"unsubscribed": {
"type": ["null", "integer"]
},
"statuschange": {
"type": ["null", "integer"]
},
"bounce": {
"type": ["null", "integer"]
},
"mta_dropped": {
"type": ["null", "integer"]
},
"dropped": {
"type": ["null", "integer"]
},
"suppressed": {
"type": ["null", "integer"]
},
"click": {
"type": ["null", "integer"]
},
"delivered": {
"type": ["null", "integer"]
},
"forward": {
"type": ["null", "integer"]
},
"print": {
"type": ["null", "integer"]
},
"reply": {
"type": ["null", "integer"]
},
"spamreport": {
"type": ["null", "integer"]
}
}
"counters_open": {
"type": ["null", "integer"]
},
"counters_processed": {
"type": ["null", "integer"]
},
"counters_sent": {
"type": ["null", "integer"]
},
"counters_deferred": {
"type": ["null", "integer"]
},
"counters_unsubscribed": {
"type": ["null", "integer"]
},
"counters_statuschange": {
"type": ["null", "integer"]
},
"counters_bounce": {
"type": ["null", "integer"]
},
"counters_mta_dropped": {
"type": ["null", "integer"]
},
"counters_dropped": {
"type": ["null", "integer"]
},
"counters_suppressed": {
"type": ["null", "integer"]
},
"counters_click": {
"type": ["null", "integer"]
},
"counters_delivered": {
"type": ["null", "integer"]
},
"counters_forward": {
"type": ["null", "integer"]
},
"counters_print": {
"type": ["null", "integer"]
},
"counters_reply": {
"type": ["null", "integer"]
},
"counters_spamreport": {
"type": ["null", "integer"]
},
"id": {
"type": ["null", "integer"]
Expand Down
39 changes: 20 additions & 19 deletions airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/contact_lists.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,25 +5,26 @@
"parentId": {
"type": ["null", "integer"]
},
"metaData": {
"type": ["null", "object"],
"properties": {
"processing": {
"type": ["null", "string"]
},
"size": {
"type": ["null", "integer"]
},
"error": {
"type": ["null", "string"]
},
"lastProcessingStateChangeAt": {
"type": ["null", "integer"]
},
"lastSizeChangeAt": {
"type": ["null", "integer"]
}
}
"metaData_processing": {
"type": ["null", "string"]
},
"metaData_size": {
"type": ["null", "integer"]
},
"metaData_error": {
"type": ["null", "string"]
},
"metaData_lastProcessingStateChangeAt": {
"type": ["null", "integer"]
},
"metaData_lastSizeChangeAt": {
"type": ["null", "integer"]
},
"metaData_listReferencesCount": {
"type": ["null", "integer"]
},
"metaData_parentFolderId": {
"type": ["null", "integer"]
},
"dynamic": {
"type": ["null", "boolean"]
Expand Down
Loading
Loading