CI #838
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: | |
| - daily | |
| - push-www | |
| schedule: | |
| - cron: "0 17 3-14 * *" | |
| workflow_dispatch: | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_DEFAULT_REGION: us-east-1 | |
| jobs: | |
| csvs: | |
| uses: ./.github/workflows/step.yml | |
| with: | |
| name: Unzip+Cache CSVs | |
| run: ctbk --s3 csv create --dask | |
| secrets: inherit | |
| normalized: | |
| needs: csvs | |
| uses: ./.github/workflows/step.yml | |
| with: | |
| name: Join regions, convert to Parquet | |
| run: ctbk --s3 normalized create --dask | |
| secrets: inherit | |
| meta_hist-names: | |
| needs: normalized | |
| uses: ./.github/workflows/step.yml | |
| with: | |
| name: Compute station name histograms | |
| run: ctbk --s3 station-meta-hist create --dask -gin | |
| secrets: inherit | |
| meta_hist-latlngs: | |
| needs: normalized | |
| uses: ./.github/workflows/step.yml | |
| with: | |
| name: Compute station lat/lng histograms | |
| run: ctbk --s3 station-meta-hist create --dask -gil | |
| secrets: inherit | |
| group_counts-start: | |
| needs: normalized | |
| uses: ./.github/workflows/step.yml | |
| with: | |
| name: Aggregate counts by start station | |
| run: ctbk --s3 aggregated create --dask -gs -ac | |
| secrets: inherit | |
| group_counts-year-month-region-gender-type-bike-counts-durations: | |
| needs: normalized | |
| uses: ./.github/workflows/step.yml | |
| with: | |
| name: Aggregate counts by (month, region, gender, user type, rideable type) | |
| run: ctbk --s3 aggregated create --dask -gymrgtb -acd | |
| secrets: inherit | |
| group_counts-start-end: | |
| needs: normalized | |
| uses: ./.github/workflows/step.yml | |
| with: | |
| name: Aggregate counts by station-pair | |
| run: ctbk --s3 aggregated create --dask -gse -ac | |
| secrets: inherit | |
| modes: | |
| needs: [meta_hist-names, meta_hist-latlngs, group_counts-start] | |
| uses: ./.github/workflows/step.yml | |
| with: | |
| name: Compute canonical station names, latlngs | |
| run: ctbk --s3 station-mode create --dask | |
| secrets: inherit | |
| station_pair_jsons: | |
| needs: [modes, group_counts-start-end] | |
| uses: ./.github/workflows/step.yml | |
| with: | |
| name: Build station-pair count JSONs | |
| run: ctbk --s3 station-pair-json create --dask | |
| secrets: inherit | |
| ymrgtb_cd_json: | |
| name: Write `ymrgtb_cd.json` used by www | |
| needs: group_counts-year-month-region-gender-type-bike-counts-durations | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ env.GITHUB_REF }} | |
| fetch-depth: 0 | |
| ssh-key: ${{ secrets.WWW_DEPLOY_KEY }} | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: 3.11.8 | |
| cache: pip | |
| - run: pip install -e . | |
| - run: python -m ctbk.ymrgtb_cd -f | |
| - name: Update www/public/assets/last_month.json | |
| run: | | |
| last_month="$(ctbk --s3 aggregated urls -gymrgtb -acd | tail -n1 | grep -Eo '[0-9]+' | tail -n1)" | |
| if [ -z "$last_month" ]; then | |
| echo "Failed to find last month" | |
| exit 1 | |
| fi | |
| echo "Last month: $last_month" | |
| path=www/public/assets/last-month.json | |
| echo "$(jq ". = \"$last_month\"" $path)" > $path | |
| - run: | | |
| python -m ctbk.gha_update -Bpush-www -Pwww -Pmain -u "GitHub Actions" -e github@actions www/public/assets/ymrgtb_cd.json www/public/assets/last-month.json |