-
Notifications
You must be signed in to change notification settings - Fork 1
178 lines (150 loc) · 6.14 KB
/
upload-and-deploy-to-prod-main.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
name: upload-and-deploy-to-prod-main
on:
push:
tags:
- "v[0-9]+.[0-9]+.[0-9]+"
env:
NAMESPACE: main
PYTHON_VERSION: 3.9
jobs:
pre-commit:
name: Run pre-commit hooks against all files
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: actions/setup-python@v5
with:
python-version: ${{ env.PYTHON_VERSION }}
- uses: pre-commit/action@v3.0.0
upload-files:
name: Upload files to S3 bucket in prod/main
runs-on: ubuntu-latest
needs: pre-commit
# These permissions are needed to interact with GitHub's OIDC Token endpoint.
permissions:
id-token: write
contents: read
environment: prod
steps:
- name: Setup code, pipenv, aws
uses: Sage-Bionetworks/action-pipenv-aws-setup@v3
with:
role_to_assume: ${{ vars.AWS_CREDENTIALS_IAM_ROLE }}
role_session_name: GitHubActions-${{ github.repository_owner }}-${{ github.event.repository.name }}-${{ github.run_id }}
python_version: ${{ env.PYTHON_VERSION }}
- name: Copy files to templates bucket
run: python src/scripts/manage_artifacts/artifacts.py --upload --namespace $NAMESPACE --cfn_bucket ${{ vars.CFN_BUCKET }}
sceptre-deploy-main:
name: Deploys to prod/main using sceptre
runs-on: ubuntu-latest
needs: [pre-commit, upload-files]
environment: prod
# These permissions are needed to interact with GitHub's OIDC Token endpoint.
permissions:
id-token: write
contents: read
steps:
- name: Setup code, pipenv, aws
uses: Sage-Bionetworks/action-pipenv-aws-setup@v3
with:
role_to_assume: ${{ vars.AWS_CREDENTIALS_IAM_ROLE }}
role_session_name: GitHubActions-${{ github.repository_owner }}-${{ github.event.repository.name }}-${{ github.run_id }}
python_version: ${{ env.PYTHON_VERSION }}
- name: Create directory for remote sceptre templates
run: mkdir -p templates/remote/
- name: Deploy sceptre stacks to prod/main
run: pipenv run sceptre --var "namespace=${{ env.NAMESPACE }}" launch prod --yes
- name: Delete preexisting S3 event notification for this namespace
uses: gagoar/invoke-aws-lambda@v3
with:
AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }}
AWS_SESSION_TOKEN: ${{ env.AWS_SESSION_TOKEN }}
REGION: ${{ env.AWS_REGION }}
FunctionName: ${{ env.NAMESPACE }}-S3EventConfig
Payload: '{"RequestType": "Delete"}'
LogType: Tail
- name: Create S3 event notification for this namespace
uses: gagoar/invoke-aws-lambda@v3
with:
AWS_ACCESS_KEY_ID: ${{ env.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ env.AWS_SECRET_ACCESS_KEY }}
AWS_SESSION_TOKEN: ${{ env.AWS_SESSION_TOKEN }}
REGION: ${{ env.AWS_REGION }}
FunctionName: ${{ env.NAMESPACE }}-S3EventConfig
Payload: '{"RequestType": "Create"}'
LogType: Tail
deploy-snowflake-main:
name: Deploy Snowflake resources
needs: sceptre-deploy-main
runs-on: ubuntu-latest
env:
PRIVATE_KEY_PASSPHRASE: ${{ secrets.SNOWFLAKE_PRIVATE_KEY_PASSPHRASE }}
steps:
- uses: actions/checkout@v3
- name: Configure Snowflake connection
run: |
# Create temporary files for config.toml and our private key
config_file=$(mktemp)
private_key_file=$(mktemp)
# Write to the private key file
echo "${{ secrets.SNOWFLAKE_PRIVATE_KEY }}" > $private_key_file
# Write to config.toml file
echo 'default_connection_name = "recover"' >> $config_file
echo '[connections.recover]' >> $config_file
echo "account = \"${{ vars.SNOWFLAKE_ACCOUNT }}\"" >> $config_file
echo "user = \"${{ vars.SNOWFLAKE_USER }}\"" >> $config_file
echo "role = \"${{ vars.SNOWFLAKE_ROLE }}\"" >> $config_file
echo 'warehouse = "RECOVER_XSMALL"' >> $config_file
echo 'authenticator = "SNOWFLAKE_JWT"' >> $config_file
echo "private_key_path = \"$private_key_file\"" >> $config_file
# Write config.toml path to global environment
echo "SNOWFLAKE_CONFIG_PATH=$config_file" >> $GITHUB_ENV
- name: Configuration file information
run: |
echo "Snowflake configuration is located at $SNOWFLAKE_CONFIG_PATH"
cat $SNOWFLAKE_CONFIG_PATH
- name: Install Snowflake CLI
uses: Snowflake-Labs/snowflake-cli-action@v1.5
with:
default-config-file-path: ${{ env.SNOWFLAKE_CONFIG_PATH }}
- name: Test Snowflake connection
run: |
snow --version
snow connection test
- name: Deploy Snowflake objects
run: |
snow sql \
-D "git_branch=main" \
-D "environment=main" \
-f snowflake/objects/deploy.sql
sts-access-test:
name: Runs STS access tests on prod synapse folders
runs-on: ubuntu-latest
needs: sceptre-deploy-main
environment: prod
# These permissions are needed to interact with GitHub's OIDC Token endpoint.
permissions:
id-token: write
contents: read
steps:
- name: Setup code, pipenv, aws
uses: Sage-Bionetworks/action-pipenv-aws-setup@v3
with:
role_to_assume: ${{ vars.AWS_CREDENTIALS_IAM_ROLE }}
role_session_name: ${{ github.event.repository.name }}-${{ github.run_id }}-nonglue-unit-tests
python_version: ${{ env.PYTHON_VERSION }}
- name: Test prod synapse folders for STS access with pytest
run: >
pipenv run python -m pytest tests/test_setup_external_storage.py
--test-bucket recover-input-data
--test-synapse-folder-id syn51714264
--namespace $NAMESPACE
--test-sts-permission read_only
-v
pipenv run python -m pytest tests/test_setup_external_storage.py
--test-bucket recover-processed-data
--test-synapse-folder-id syn51406699
--namespace $NAMESPACE/parquet
--test-sts-permission read_write
-v