scrape.yml

name: Scrape

on:
  workflow_dispatch:
    inputs:
      accountsToScrape:
        default: ""
        required: false
        description: "Accounts to scrape (comma separated)"
      daysBack:
        default: "1"
        required: false
        description: "Days back to scrape"
      scrapeFromBeginningOfMonth:
        default: true
        required: false
        description: "If true, the days back will be caluclated automacitlly from the beginning of the month"
      worksheetName:
        default: "current"
        required: false
        description: "The name of the worksheet to write to"
  schedule:
    - cron: "00 06 * * *"
env:
  REGISTRY: ghcr.io
jobs:
  scrape:
    runs-on: ubuntu-latest
    timeout-minutes: 60
    steps:
      - name: Checkout
        uses: actions/checkout@v4.1.1

      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.11.4"

      - name: Check if first day of the month
        run: |
          current_date=$(date +"%Y-%m-%d")
          day=$(date -d "$current_date" +"%d")
          echo $current_date
          if [ "$day" = "01" ]; then
            sudo apt install libxslt-dev libxml2-dev python3-dev 
            printf "%s\n" "$GOOGLE_JSON" > google.json    
            python -m pip install --upgrade pip
            pip install -r requirements.txt
            python gpt.py
          else
            echo "It is not the first day of the month."
          fi
        env:
          GOOGLE_JSON: ${{ secrets.GOOGLE_JSON }}

      - id: normalize-repository-name
        run: echo "repository=$(echo ${{ github.repository }} | tr '[:upper:]' '[:lower:]')" >> $GITHUB_OUTPUT

      - name: Pull image
        run: docker pull ${{ env.REGISTRY }}/${{ steps.normalize-repository-name.outputs.repository }}:latest

      - name: Run scraper
        run: docker run --rm
          -e DEBUG
          -e DAYS_BACK
          -e TZ
          -e WORKSHEET_NAME
          -e ACCOUNTS_JSON
          -e TELEGRAM_API_KEY
          -e TELEGRAM_CHAT_ID
          -e GOOGLE_SHEET_ID
          -e GOOGLE_SERVICE_ACCOUNT_EMAIL
          -e GOOGLE_SERVICE_ACCOUNT_PRIVATE_KEY
          -e ACCOUNTS_TO_SCRAPE
          -e ADE_DATABASE_NAME
          -e ADE_TABLE_NAME
          -e ADE_INGESTION_MAPPING
          -e ADE_INGEST_URI
          -e AZURE_TENANT_ID
          -e AZURE_APP_ID
          -e AZURE_APP_KEY
          -e SCRAPE_FROM_BEGINNING_OF_MONTH
          -e YNAB_TOKEN
          -e YNAB_BUDGET_ID
          -e YNAB_ACCOUNTS
          -e BUXFER_USER_NAME
          -e BUXFER_PASSWORD
          -e BUXFER_ACCOUNTS
          -e TRANSACTION_HASH_TYPE
          -e WEB_POST_URL
          ${{ env.REGISTRY }}/${{ steps.normalize-repository-name.outputs.repository }}:latest
        env:
          DEBUG: ""
          TZ: "UTC"
          DAYS_BACK: ${{ github.event.inputs.daysBack }}
          WORKSHEET_NAME: ${{ github.event.inputs.worksheetName || 'current' }}
          SCRAPE_FROM_BEGINNING_OF_MONTH: ${{ github.event.inputs.scrapeFromBeginningOfMonth || true }}
          ACCOUNTS_TO_SCRAPE: ${{ github.event.inputs.accountsToScrape }}
          ACCOUNTS_JSON: ${{ secrets.ACCOUNTS_JSON }}
          TELEGRAM_API_KEY: ${{ secrets.TELEGRAM_API_KEY }}
          TELEGRAM_CHAT_ID: ${{ secrets.TELEGRAM_CHAT_ID }}
          GOOGLE_SHEET_ID: ${{ secrets.GOOGLE_SHEET_ID }}
          GOOGLE_SERVICE_ACCOUNT_EMAIL: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_EMAIL }}
          GOOGLE_SERVICE_ACCOUNT_PRIVATE_KEY: ${{ secrets.GOOGLE_SERVICE_ACCOUNT_PRIVATE_KEY }}
          ADE_DATABASE_NAME: ${{ secrets.ADE_DATABASE_NAME }}
          ADE_TABLE_NAME: ${{ secrets.ADE_TABLE_NAME }}
          ADE_INGESTION_MAPPING: ${{ secrets.ADE_INGESTION_MAPPING }}
          ADE_INGEST_URI: ${{ secrets.ADE_INGEST_URI }}
          AZURE_TENANT_ID: ${{ secrets.AZURE_TENANT_ID }}
          AZURE_APP_ID: ${{ secrets.AZURE_APP_ID }}
          AZURE_APP_KEY: ${{ secrets.AZURE_APP_KEY }}
          YNAB_TOKEN: ${{ secrets.YNAB_TOKEN }}
          YNAB_BUDGET_ID: ${{ secrets.YNAB_BUDGET_ID }}
          YNAB_ACCOUNTS: ${{ secrets.YNAB_ACCOUNTS }}
          BUXFER_USER_NAME: ${{ secrets.BUXFER_USER_NAME }}
          BUXFER_PASSWORD: ${{ secrets.BUXFER_PASSWORD }}
          BUXFER_ACCOUNTS: ${{ secrets.BUXFER_ACCOUNTS }}
          TRANSACTION_HASH_TYPE: ${{ secrets.TRANSACTION_HASH_TYPE }}
          WEB_POST_URL: ${{ secrets.WEB_POST_URL }}

      - name: Update categories
        run: |
          current_day=$(date +%a)
          if [ "$current_day" == "Sat" ]; then
            echo "It's Shabbat, not update categories."
          else
            sudo apt install libxslt-dev libxml2-dev python3-dev 
            printf "%s\n" "$GOOGLE_JSON" > google.json    
            python -m pip install --upgrade pip
            pip install pyTelegramBotAPI gspread oauth2client
            python categories.py
          fi
        env:
          GOOGLE_JSON: ${{ secrets.GOOGLE_JSON }}
          TOKEN: ${{ secrets.TELEGRAM_API_KEY }}
          TELEGRAM_CHAT_ID: ${{ secrets.TELEGRAM_CHAT_ID }}
          WHATSAPP_URL: ${{ secrets.WHATSAPP_URL }}

      - name: Send Update
        uses: fjogeleit/http-request-action@v1
        with:
          url: "${{ secrets.UPDATE_WEB_HOOK }}"
          method: "GET"