daily_elt_job.yml

name: Daily ELT Job
run-name: 🧃 Daily ELT Job 🧃

on:
  schedule:
    - cron: '0 14 * * 1-5'  # 7am PST (14:00 UTC) Monday to Friday
  workflow_dispatch:  # Allow manual triggering

jobs:
  daily_elt:
    name: Daily ELT Process
    runs-on: ubuntu-latest
    if: github.ref == 'refs/heads/main'  # Only run on the main branch
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.12'

      - name: Authenticate to Google Cloud
        uses: 'google-github-actions/auth@v2'
        with:
          credentials_json: '${{ secrets.GOOGLE_SQLMESH_CREDENTIALS }}'

      - name: Install SQLMesh Enterprise + Dependencies
        run: |
          pip install poetry keyring keyrings-google-artifactregistry-auth
          poetry install --no-root
        shell: bash

      - name: Append raw events to tcloud_raw_data.raw_events to display observer metrics
        run: |
          poetry run python demo_scripts/main.py append-rawdata
        env:
          GOOGLE_SQLMESH_CREDENTIALS: ${{ secrets.GOOGLE_SQLMESH_CREDENTIALS }}

      - name: Run SQLMesh
        run: |
          poetry run tcloud sqlmesh run
        env:
          TCLOUD_TOKEN: ${{ secrets.TCLOUD_TOKEN }}
          GOOGLE_SQLMESH_CREDENTIALS: ${{ secrets.GOOGLE_SQLMESH_CREDENTIALS }}