chore: add package based flow

starkware-libs · Jul 9, 2024 · a79cb70 · a79cb70
1 parent b301a5d
commit a79cb70
Show file tree

Hide file tree

Showing 5 changed files with 120 additions and 137 deletions.
diff --git a/.github/workflows/blockifier/blockifier_ci.yml b/.github/workflows/blockifier/blockifier_ci.yml
@@ -17,65 +17,6 @@ on:
       - edited
 
 jobs:
-  commitlint:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Install commitlint
-        run: npm install --global @commitlint/cli @commitlint/config-conventional
-
-      - name: Validate PR commits with commitlint
-        if: github.event_name == 'pull_request' && !(contains(github.event.pull_request.title, '/merge-main') || contains(github.event.pull_request.title, '/merge main'))
-        env:
-          BASE_SHA: ${{ github.event.pull_request.base.sha }}
-          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
-        run: commitlint --from "$BASE_SHA" --to "$HEAD_SHA" --verbose
-
-      - name: Validate PR title with commitlint
-        if: github.event_name != 'merge_group' && github.event_name != 'push' && !(contains(github.event.pull_request.title, '/merge-main') || contains(github.event.pull_request.title, '/merge main'))
-        env:
-          TITLE: ${{ github.event.pull_request.title }}
-        run: echo "$TITLE" | commitlint --verbose
-
-  format:
-    runs-on: ubuntu-20.04
-    steps:
-      - uses: actions/checkout@v4
-      - uses: dtolnay/rust-toolchain@master
-        with:
-          components: rustfmt
-          toolchain: nightly-2024-04-29
-      - uses: Swatinem/rust-cache@v2
-        with:
-          prefix-key: "v0-rust-ubuntu-20.04"
-      - run: scripts/rust_fmt.sh --check
-
-  clippy:
-    runs-on: ubuntu-20.04
-    steps:
-      - uses: actions/checkout@v4
-      - uses: dtolnay/rust-toolchain@stable
-        with:
-          components: clippy
-      - uses: Swatinem/rust-cache@v2
-        with:
-          prefix-key: "v0-rust-ubuntu-20.04"
-
-      # Setup pypy and link to the location expected by .cargo/config.toml.
-      - uses: actions/setup-python@v5
-        id: setup-pypy
-        with:
-          python-version: 'pypy3.9'
-      - run: ln -s '${{ steps.setup-pypy.outputs.python-path }}' /usr/local/bin/pypy3.9
-      - env:
-          LD_LIBRARY_PATH: ${{ steps.setup-pypy.outputs.pythonLocation }}/bin
-        run: echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV
-
-      - run: scripts/clippy.sh
-
   featureless-build:
     runs-on: ubuntu-20.04
     steps:

diff --git a/.github/workflows/blockifier_ci.yml b/.github/workflows/blockifier_ci.yml
@@ -21,65 +21,6 @@ on:
       - 'crates/blockifier/**'
 
 jobs:
-  commitlint:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Install commitlint
-        run: npm install --global @commitlint/cli @commitlint/config-conventional
-
-      - name: Validate PR commits with commitlint
-        if: github.event_name == 'pull_request' && !(contains(github.event.pull_request.title, '/merge-main') || contains(github.event.pull_request.title, '/merge main'))
-        env:
-          BASE_SHA: ${{ github.event.pull_request.base.sha }}
-          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
-        run: commitlint --from "$BASE_SHA" --to "$HEAD_SHA" --verbose
-
-      - name: Validate PR title with commitlint
-        if: github.event_name != 'merge_group' && github.event_name != 'push' && !(contains(github.event.pull_request.title, '/merge-main') || contains(github.event.pull_request.title, '/merge main'))
-        env:
-          TITLE: ${{ github.event.pull_request.title }}
-        run: echo "$TITLE" | commitlint --verbose
-
-  format:
-    runs-on: ubuntu-20.04
-    steps:
-      - uses: actions/checkout@v4
-      - uses: dtolnay/rust-toolchain@master
-        with:
-          components: rustfmt
-          toolchain: nightly-2024-04-29
-      - uses: Swatinem/rust-cache@v2
-        with:
-          prefix-key: "v0-rust-ubuntu-20.04"
-      - run: scripts/rust_fmt.sh --check
-
-  clippy:
-    runs-on: ubuntu-20.04
-    steps:
-      - uses: actions/checkout@v4
-      - uses: dtolnay/rust-toolchain@stable
-        with:
-          components: clippy
-      - uses: Swatinem/rust-cache@v2
-        with:
-          prefix-key: "v0-rust-ubuntu-20.04"
-
-      # Setup pypy and link to the location expected by .cargo/config.toml.
-      - uses: actions/setup-python@v5
-        id: setup-pypy
-        with:
-          python-version: 'pypy3.9'
-      - run: ln -s '${{ steps.setup-pypy.outputs.python-path }}' /usr/local/bin/pypy3.9
-      - env:
-          LD_LIBRARY_PATH: ${{ steps.setup-pypy.outputs.pythonLocation }}/bin
-        run: echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV
-
-      - run: scripts/clippy.sh
-
   featureless-build:
     runs-on: ubuntu-20.04
     steps:

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -0,0 +1,110 @@
+name: Main-CI-Flow
+
+on:
+  push:
+    branches:
+      - main
+      - main-v[0-9].**
+    tags:
+      - v[0-9].**
+
+  pull_request:
+    types:
+      - opened
+      - reopened
+      - synchronize
+      - auto_merge_enabled
+      - edited
+
+jobs:
+  commitlint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Install commitlint
+        run: npm install --global @commitlint/cli @commitlint/config-conventional
+
+      - name: Validate PR commits with commitlint
+        if: github.event_name == 'pull_request' && !(contains(github.event.pull_request.title, '/merge-main') || contains(github.event.pull_request.title, '/merge main'))
+        env:
+          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+        run: commitlint --from "$BASE_SHA" --to "$HEAD_SHA" --verbose
+
+      - name: Validate PR title with commitlint
+        if: github.event_name != 'merge_group' && github.event_name != 'push' && !(contains(github.event.pull_request.title, '/merge-main') || contains(github.event.pull_request.title, '/merge main'))
+        env:
+          TITLE: ${{ github.event.pull_request.title }}
+        run: echo "$TITLE" | commitlint --verbose
+
+  format:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@master
+        with:
+          components: rustfmt
+          toolchain: nightly-2024-04-29
+      - uses: Swatinem/rust-cache@v2
+        with:
+          prefix-key: "v0-rust-ubuntu-20.04"
+      - run: scripts/rust_fmt.sh --check
+
+  clippy:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          components: clippy
+      - uses: Swatinem/rust-cache@v2
+        with:
+          prefix-key: "v0-rust-ubuntu-20.04"
+
+      # Setup pypy and link to the location expected by .cargo/config.toml.
+      - uses: actions/setup-python@v5
+        id: setup-pypy
+        with:
+          python-version: 'pypy3.9'
+      - run: ln -s '${{ steps.setup-pypy.outputs.python-path }}' /usr/local/bin/pypy3.9
+      - env:
+          LD_LIBRARY_PATH: ${{ steps.setup-pypy.outputs.pythonLocation }}/bin
+        run: echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV
+
+      - run: scripts/clippy.sh
+
+  run-tests:
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: Swatinem/rust-cache@v2
+        with:
+          prefix-key: "v0-rust-ubuntu-20.04"
+
+      # Setup pypy and link to the location expected by .cargo/config.toml.
+      - uses: actions/setup-python@v5
+        id: setup-pypy
+        with:
+          python-version: 'pypy3.9'
+      - run: ln -s '${{ steps.setup-pypy.outputs.python-path }}' /usr/local/bin/pypy3.9
+      - env:
+          LD_LIBRARY_PATH: ${{ env.Python3_ROOT_DIR }}/bin
+        run: echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV
+      - run: |
+          python3 -m venv ci
+          ci/bin/pip install -r scripts/requirements.txt
+          ci/bin/python scripts/run_tests.py --changes_only
+          ci/bin/python scripts/run_tests.py --changes_only --features concurrency
+
+  # Keep the name 'udeps' to match original action name, so we don't need to define specific branch
+  # rules on Github for specific version branches.
+  udeps:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Run Machete (detect unused dependencies)
+        uses: bnjbvr/cargo-machete@main
diff --git a/.github/workflows/papyrus_ci.yml b/.github/workflows/papyrus_ci.yml
@@ -23,23 +23,7 @@ env:
   PROTOC_VERSION: v25.1
 
 jobs:
-  commitlint:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Install commitlint
-        run: npm install --global @commitlint/cli @commitlint/config-conventional
-
-      - name: Validate PR commits with commitlint
-        if: github.event_name == 'pull_request'
-        run: commitlint --from ${{ github.event.pull_request.base.sha }} --to ${{ github.event.pull_request.head.sha }} --verbose
 
-      - name: Validate PR title with commitlint
-        if: github.event_name != 'merge_group' && github.event_name != 'push'
-        run: echo "${{ github.event.pull_request.title }}" | commitlint --verbose
 
   executable-run:
     runs-on: ubuntu-latest

diff --git a/scripts/run_tests.py b/scripts/run_tests.py
@@ -4,7 +4,7 @@
 import re
 import subprocess
 import os
-from typing import Dict, List, Set
+from typing import Dict, List, Set, Optional
 from git import Repo
 
 PATTERN = r"(\w+)\s*v([\d.]*.*)\((.*?)\)"
@@ -60,7 +60,7 @@ def get_package_dependencies(package_name: str) -> Set[str]:
     return deps
 
 
-def run_test(changes_only: bool):
+def run_test(changes_only: bool, features: Optional[str] = None):
     local_changes = get_local_changes(".")
     modified_packages = get_modified_packages(local_changes)
     args = []
@@ -74,6 +74,10 @@ def run_test(changes_only: bool):
             print("No changes detected.")
             return
     cmd = ["cargo", "test"] + args
+
+    if features is not None:
+        cmd.extend(["--features", features])
+
     print("Running tests...")
     print(cmd)
     subprocess.run(cmd)
@@ -82,12 +86,15 @@ def run_test(changes_only: bool):
 def parse_args() -> argparse.Namespace:
     parser = argparse.ArgumentParser(description="Presubmit script.")
     parser.add_argument("--changes_only", action="store_true")
+    parser.add_argument(
+        "--features", type=str, help="Which services to deploy. For multi services separate by ','."
+    )
     return parser.parse_args()
 
 
 def main():
     args = parse_args()
-    run_test(changes_only=args.changes_only)
+    run_test(changes_only=args.changes_only, features=args.features)
 
 
 if __name__ == "__main__":