Skip to content

CI

CI #656

Workflow file for this run

name: CI
on:
push:
branches:
- daily
- push-www
schedule:
- cron: "0 17 3-14 * *"
workflow_dispatch:
env:
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
AWS_DEFAULT_REGION: us-east-1
jobs:
csvs:
uses: ./.github/workflows/step.yml
with:
name: Unzip+Cache CSVs
run: ctbk --s3 csv create --dask
secrets: inherit
normalized:
needs: csvs
uses: ./.github/workflows/step.yml
with:
name: Join regions, convert to Parquet
run: ctbk --s3 normalized create --dask
secrets: inherit
meta_hist-names:
needs: normalized
uses: ./.github/workflows/step.yml
with:
name: Compute station name histograms
run: ctbk --s3 station-meta-hist -gin create --dask
secrets: inherit
meta_hist-latlngs:
needs: normalized
uses: ./.github/workflows/step.yml
with:
name: Compute station lat/lng histograms
run: ctbk --s3 station-meta-hist -gil create --dask
secrets: inherit
group_counts-start:
needs: normalized
uses: ./.github/workflows/step.yml
with:
name: Aggregate counts by start station
run: ctbk --s3 aggregated -gs -ac create --dask
secrets: inherit
group_counts-start-end:
needs: normalized
uses: ./.github/workflows/step.yml
with:
name: Aggregate counts by station-pair
run: ctbk --s3 aggregated -gse -ac create --dask
secrets: inherit
group_counts-year-month-region-gender-type-bike-counts-durations:
needs: normalized
uses: ./.github/workflows/step.yml
with:
name: Aggregate counts by (month, region, gender, user type, rideable type)
run: ctbk --s3 aggregated -gymrgtb -acd create --dask
secrets: inherit
modes:
needs: [meta_hist-names, meta_hist-latlngs, group_counts-start]
uses: ./.github/workflows/step.yml
with:
name: Compute canonical station names, latlngs
run: ctbk --s3 station-mode create --dask
secrets: inherit
station_pair_jsons:
needs: [modes, group_counts-start-end]
uses: ./.github/workflows/step.yml
with:
name: Build station-pair count JSONs
run: ctbk --s3 station-pair-json create --dask
secrets: inherit
ymrgtb_cd_json:
name: Write `ymrgtb_cd.json` used by www
needs: group_counts-year-month-region-gender-type-bike-counts-durations
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
with:
ref: ${{ env.GITHUB_REF }}
fetch-depth: 0
ssh-key: ${{ secrets.WWW_DEPLOY_KEY }}
- uses: actions/setup-python@v4
with:
python-version: 3.9
cache: pip
- run: pip install -e .
- run: python -m ctbk.ymrgtb_cd -f
- name: Update www/public/assets/last_month.json
run: |
last_month="$(ctbk --s3 aggregated -gymrgtb -acd urls | tail -n1 | grep -Eo '[0-9]+' | tail -n1)"
echo "Last month: $last_month"
path=www/public/assets/last-month.json
echo "$(jq ". = \"$last_month\"" $path)" > $path
- run: |
python -m ctbk.gha_update -Bpush-www -Pwww -Pmain -u "GitHub Actions" -e github@actions www/public/assets/ymrgtb_cd.json www/public/assets/last-month.json