-
Notifications
You must be signed in to change notification settings - Fork 4.1k
195 lines (182 loc) · 8.64 KB
/
community_ci.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
name: Community CI
concurrency:
# This is the name of the concurrency group. It is used to prevent concurrent runs of the same workflow.
#
# - github.head_ref is only defined on PR runs, it makes sure that the concurrency group is unique for pull requests
# ensuring that only one run per pull request is active at a time.
group: ${{ github.workflow }}-${{ github.head_ref }}
cancel-in-progress: true
on:
# The pull request target event is triggered on PR open and synchronize events.
# The executed workflow code is the one from the target branch.
# It guarantees that the workflow logic is not altered by the PR.
pull_request_target:
types: [opened, synchronize]
jobs:
fail_on_protected_path_changes:
name: "Check fork do not change protected paths"
if: github.event.pull_request.head.repo.fork == true
runs-on: ubuntu-latest
permissions:
pull-requests: read
steps:
- name: Check for changes in protected paths
id: check_for_changes_in_protected_paths
uses: dorny/paths-filter@v2
with:
filters: |
protected_paths:
- '.github/**'
- 'airbyte-ci/**'
- name: Fail if changes in protected paths
if: steps.check_for_changes_in_protected_paths.outputs.protected_paths == 'true'
run: |
echo "The fork has changes in protected paths. This is not allowed."
exit 1
format_check:
# IMPORTANT: This name must match the require check name on the branch protection settings
name: "Check for formatting errors"
if: github.event.pull_request.head.repo.fork == true
environment: community-ci-auto
runs-on: community-tooling-test-small
needs: fail_on_protected_path_changes
timeout-minutes: 30
env:
MAIN_BRANCH_NAME: "master"
steps:
# This checkouts a fork which can contain untrusted code
# It's deemed safe as the formatter are not executing any checked out code
- name: Checkout fork
uses: actions/checkout@v4
with:
repository: ${{ github.event.pull_request.head.repo.full_name }}
ref: ${{ github.event.pull_request.head.sha }}
fetch-depth: 1
# This will sync the .github folder of the main repo with the fork
# This allows us to use up to date actions and CI logic from the main repo
- name: Pull .github folder and internal packages from main repository
id: pull_github_folder
run: |
git remote add main https://github.com/airbytehq/airbyte.git
git fetch main ${MAIN_BRANCH_NAME}
git checkout main/${MAIN_BRANCH_NAME} -- .github
git checkout main/${MAIN_BRANCH_NAME} -- airbyte-ci
- name: Run airbyte-ci format check all
# This path refers to the fork .github folder.
# We make sure its content is in sync with the main repo .github folder by pulling it in the previous step
uses: ./.github/actions/run-airbyte-ci
with:
context: "pull_request"
sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
subcommand: "format check all"
is_fork: "true"
connectors_early_ci:
name: Run connectors early CI on fork
if: github.event.pull_request.head.repo.fork == true
needs: fail_on_protected_path_changes
environment: community-ci-auto
runs-on: community-tooling-test-small
timeout-minutes: 10
env:
MAIN_BRANCH_NAME: "master"
permissions:
statuses: write
steps:
# This checkouts a fork which can contain untrusted code
# It's deemed safe as the static checks are not executing any checked out code
- name: Checkout fork
uses: actions/checkout@v4
with:
repository: ${{ github.event.pull_request.head.repo.full_name }}
ref: ${{ github.event.pull_request.head.sha }}
fetch-depth: 1
# This will sync the .github folder of the main repo with the fork
# This allows us to use up to date actions and CI logic from the main repo
- name: Pull .github folder from main repository
id: pull_github_folder
run: |
git remote add main https://github.com/airbytehq/airbyte.git
git fetch main ${MAIN_BRANCH_NAME}
git checkout main/${MAIN_BRANCH_NAME} -- .github
git checkout main/${MAIN_BRANCH_NAME} -- airbyte-ci
- name: Run airbyte-ci static checks and version increment checks on modified connectors
# This path refers to the fork .github folder.
# We make sure its content is in sync with the main repo .github folder by pulling it in the previous step
uses: ./.github/actions/run-airbyte-ci
with:
context: "pull_request"
sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
subcommand: "connectors --modified test --only-step=qa_checks --only-step=version_inc_check --global-status-check-context='Connectors early CI checks' --global-status-check-description='Running early CI checks on connectors'"
is_fork: "true"
git_repo_url: ${{ github.event.pull_request.head.repo.clone_url }}
git_branch: ${{ github.head_ref }}
git_revision: ${{ github.event.pull_request.head.sha }}
github_token: ${{ github.token }}
docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
- name: Upload pipeline reports
id: upload-artifact
uses: actions/upload-artifact@v4
with:
name: early-ci-pipeline-reports
path: /home/runner/work/airbyte/airbyte/airbyte-ci/connectors/pipelines/pipeline_reports/airbyte-ci/connectors/test/pull_request/**/output.html
retention-days: 7
connectors_full_ci:
name: Run connectors full CI on fork
if: github.event.pull_request.head.repo.fork == true
# Deployment of jobs on the community-ci environment requires manual approval
# This is something we set up in the GitHub environment settings:
# https://github.com/airbytehq/airbyte/settings/environments/2091483613/edit
# This is a safety measure to make sure the code running on our infrastructure has been reviewed before running on it
needs: fail_on_protected_path_changes
environment: community-ci
runs-on: community-connector-test-large
timeout-minutes: 180 # 3 hours
permissions:
statuses: write
env:
MAIN_BRANCH_NAME: "master"
steps:
# This checkouts a fork which can contain untrusted code
# It's deemed safe as the community-ci environment requires manual reviewer approval to run
- name: Checkout fork
uses: actions/checkout@v4
with:
repository: ${{ github.event.pull_request.head.repo.full_name }}
ref: ${{ github.event.pull_request.head.sha }}
fetch-depth: 1
# This will sync the .github folder of the main repo with the fork
# This allows us to use up to date actions and CI logic from the main repo
- name: Pull .github folder from main repository
id: pull_github_folder
run: |
git remote add main https://github.com/airbytehq/airbyte.git
git fetch main ${MAIN_BRANCH_NAME}
git checkout main/${MAIN_BRANCH_NAME} -- .github
git checkout main/${MAIN_BRANCH_NAME} -- airbyte-ci
- name: Run airbyte-ci connectors test
# This path refers to the fork .github folder.
# We make sure its content is in sync with the main repo .github folder by pulling it in the previous step
uses: ./.github/actions/run-airbyte-ci
with:
context: "pull_request"
dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_2 }}
docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}
sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
git_repo_url: ${{ github.event.pull_request.head.repo.clone_url }}
git_branch: ${{ github.head_ref }}
git_revision: ${{ github.event.pull_request.head.sha }}
github_token: ${{ github.token }}
s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
subcommand: "connectors --modified test"
is_fork: "true"
- name: Upload pipeline reports
id: upload-artifact
uses: actions/upload-artifact@v4
with:
name: pipeline-reports
path: /home/runner/work/airbyte/airbyte/airbyte-ci/connectors/pipelines/pipeline_reports/airbyte-ci/connectors/test/pull_request/**/output.html
retention-days: 7