CI #663
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: | |
| - daily | |
| - push-www | |
| schedule: | |
| - cron: "0 17 3-14 * *" | |
| workflow_dispatch: | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_DEFAULT_REGION: us-east-1 | |
| jobs: | |
| csvs: | |
| uses: ./.github/workflows/step.yml | |
| with: | |
| name: Unzip+Cache CSVs | |
| run: ctbk --s3 csv create --dask | |
| secrets: inherit | |
| normalized: | |
| needs: csvs | |
| uses: ./.github/workflows/step.yml | |
| with: | |
| name: Join regions, convert to Parquet | |
| run: ctbk --s3 normalized create --dask | |
| secrets: inherit | |
| meta_hist-names: | |
| needs: normalized | |
| uses: ./.github/workflows/step.yml | |
| with: | |
| name: Compute station name histograms | |
| run: ctbk --s3 station-meta-hist -gin create --dask | |
| secrets: inherit | |
| meta_hist-latlngs: | |
| needs: normalized | |
| uses: ./.github/workflows/step.yml | |
| with: | |
| name: Compute station lat/lng histograms | |
| run: ctbk --s3 station-meta-hist -gil create --dask | |
| secrets: inherit | |
| group_counts-start: | |
| needs: normalized | |
| uses: ./.github/workflows/step.yml | |
| with: | |
| name: Aggregate counts by start station | |
| run: ctbk --s3 aggregated -gs -ac create --dask | |
| secrets: inherit | |
| group_counts-start-end: | |
| needs: normalized | |
| uses: ./.github/workflows/step.yml | |
| with: | |
| name: Aggregate counts by station-pair | |
| run: ctbk --s3 aggregated -gse -ac create --dask | |
| secrets: inherit | |
| group_counts-year-month-region-gender-type-bike-counts-durations: | |
| needs: normalized | |
| uses: ./.github/workflows/step.yml | |
| with: | |
| name: Aggregate counts by (month, region, gender, user type, rideable type) | |
| run: ctbk --s3 aggregated -gymrgtb -acd create --dask | |
| secrets: inherit | |
| modes: | |
| needs: [meta_hist-names, meta_hist-latlngs, group_counts-start] | |
| uses: ./.github/workflows/step.yml | |
| with: | |
| name: Compute canonical station names, latlngs | |
| run: ctbk --s3 station-mode create --dask | |
| secrets: inherit | |
| station_pair_jsons: | |
| needs: [modes, group_counts-start-end] | |
| uses: ./.github/workflows/step.yml | |
| with: | |
| name: Build station-pair count JSONs | |
| run: ctbk --s3 station-pair-json create --dask | |
| secrets: inherit | |
| ymrgtb_cd_json: | |
| name: Write `ymrgtb_cd.json` used by www | |
| needs: group_counts-year-month-region-gender-type-bike-counts-durations | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ env.GITHUB_REF }} | |
| fetch-depth: 0 | |
| ssh-key: ${{ secrets.WWW_DEPLOY_KEY }} | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: 3.9 | |
| cache: pip | |
| - run: pip install -e . | |
| - run: python -m ctbk.ymrgtb_cd -f | |
| - name: Update www/public/assets/last_month.json | |
| run: | | |
| last_month="$(ctbk --s3 aggregated -gymrgtb -acd urls | tail -n1 | grep -Eo '[0-9]+' | tail -n1)" | |
| echo "Last month: $last_month" | |
| path=www/public/assets/last-month.json | |
| echo "$(jq ". = \"$last_month\"" $path)" > $path | |
| - run: | | |
| python -m ctbk.gha_update -Bpush-www -Pwww -Pmain -u "GitHub Actions" -e github@actions www/public/assets/ymrgtb_cd.json www/public/assets/last-month.json |