test_e2eshark.yml

# Copyright 2024 Advanced Micro Devices
#
# Licensed under the Apache License v2.0 with LLVM Exceptions.
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception

name: E2ESHARK Test Suite
on:
  workflow_dispatch:
  # schedule:
  # # Runs at 12:00 PM UTC, which is 5:00 AM PST
  #  - cron: '0 12 * * *'

jobs:
  e2eshark:
    runs-on: nodai-amdgpu-mi250-x86-64
    env:
      E2E_VENV_DIR: ${{ github.workspace }}/test-suite_venv
      HF_TOKEN: ${{ secrets.HF_TOKEN }}
      AZ_PRIVATE_CONNECTION: ${{ secrets.ONNXPRIVATESTORAGE_AZ_PRIVATE_CONNECTION }}
    steps:
      - name: Checkout Test Suite
        uses: actions/checkout@v2
        with:
          repository: nod-ai/SHARK-TestSuite
          path: test-suite

      - name: Checkout SHARK Turbine
        uses: actions/checkout@v2
        with:
          repository: nod-ai/SHARK-Turbine
          path: SHARK-Turbine

      - name: Checkout iree turbine
        uses: actions/checkout@v2
        with:
          repository: iree-org/iree-turbine
          path: iree-turbine

      - name: "Setup e2eshark python venv"
        run: |
          python3.11 -m venv ${E2E_VENV_DIR}
          source ${E2E_VENV_DIR}/bin/activate
          pip install --upgrade pip
          pip install -r ./e2eshark/requirements.txt
          pip uninstall -y numpy
          pip install numpy==1.26.3
          pip install --upgrade transformers
          pip install \
            --find-links https://iree.dev/pip-release-links.html \
            --upgrade \
            iree-compiler \
            iree-runtime
          pip install \
            --find-links https://github.com/llvm/torch-mlir-release/releases/expanded_assets/dev-wheels \
            --upgrade \
            torch-mlir
        working-directory: ./test-suite

      - name: Run Onnx Mode
        run: |
          source ${E2E_VENV_DIR}/bin/activate
          pip list
          cd e2eshark
          free -mh
          python3.11 ./run.py \
            -r ./test-onnx \
            --report \
            --cachedir /groups/aig_sharks/test-suite-ci-cache \
            --testsfile ./ci-list.txt \
            --mode onnx \
            -f onnx pytorch \
            -g models \
            --tolerance .01 .01 \
            --cleanup \
            --postprocess \
            --ci \
            -v
        working-directory: ./test-suite

      - name: Setup turbine python venv
        run: |
          source ${E2E_VENV_DIR}/bin/activate
          pip install -f https://iree.dev/pip-release-links.html --upgrade \
          -r ../iree-turbine/iree-requirements.txt
          pip install -e ../iree-turbine[testing]
          pip install --no-compile --pre --upgrade -e ../SHARK-Turbine/models -r ../SHARK-Turbine/models/requirements.txt
        working-directory: ./test-suite

      - name: Run Turbine Mode
        run: |
          source ${E2E_VENV_DIR}/bin/activate
          cd e2eshark
          free -mh
          HF_TOKEN=${{ secrets.HF_TOKEN }} python3.11 ./run.py \
          -r ./test-turbine \
          --report \
          --cachedir /groups/aig_sharks/test-suite-ci-cache \
          --mode turbine \
          -g models \
          --cleanup \
          --postprocess \
          --ci \
          -v
        working-directory: ./test-suite
      
      - uses: actions/upload-artifact@master
        with:
          name: ci_reports
          path: ./test-suite/e2eshark/ci_reports

  upload_artifacts:
    needs: [e2eshark]
    runs-on: nodai-amdgpu-mi250-x86-64
    steps:
      - name: Checkout repo
        uses: actions/checkout@v2
        with:
          repository: nod-ai/e2eshark-reports
          ref: 'main'
          token: ${{ secrets.E2ESHARK_GITHUB_TOKEN }}
          path: e2eshark-reports
      
      - uses: actions/download-artifact@master
        with:
          name: ci_reports
          path: ./e2eshark-reports

      - name: Push artifacts
        run: |
          date=$(date '+%Y-%m-%d')
          git config user.name "GitHub Actions Bot"
          git config user.email "<>"
          git add $date
          git commit -m "add CI reports for e2eshark"
          git push origin main
        working-directory: ./e2eshark-reports