增加CE编译任务专用的代码缓存workflow,支持代码库的清理和压缩， (#3021)

* 使用PaddleQACheckout1.1版本替换1.0版本,增加workflow并发度控制 * 增加CE编译任务专用的代码缓存workflow * commit代码同步workflow支持代码库清理和压缩
PaddlePaddle · Jan 13, 2025 · f123842 · f123842
1 parent 6882da3
commit f123842
Show file tree

Hide file tree

Showing 2 changed files with 33 additions and 18 deletions.
diff --git a/...hub/workflows/repo_sync_dispatch_test.yml → .github/workflows/repo_sync_buildce_use.yml b/...hub/workflows/repo_sync_dispatch_test.yml → .github/workflows/repo_sync_buildce_use.yml
@@ -1,8 +1,8 @@
-name: Paddle kits repos sync via repository_dispatch
-run-name: Paddle kits repos sync Runs on ${{ github.event_name }}
+name: Paddle Repos Sync Triggered by Commit Using Repository Dispatch
+run-name: Paddle repos sync runs triggered by commit using ${{ github.event_name }}
 on:
   repository_dispatch:
-    types: [sync-event-test]
+    types: [sync-event-commit]
 
 jobs:
   paddleqa-checkout:
@@ -25,6 +25,7 @@ jobs:
           fetch_depth: ${{ github.event.client_payload.fetch_depth }}
           path: ${{ github.event.client_payload.path }}
           commit_sha: ${{ github.event.client_payload.commit_sha }}
+          repo_compress: ${{ github.event.client_payload.repo_compress }}
       - uses: actions/setup-python@v5
         with:
           python-version: '3.10'
@@ -34,8 +35,8 @@ jobs:
           SK: ${{ secrets.BOS_SK }}
         run: |
           git log -n 3
-          current_commit=$(git rev-parse HEAD)
           tree -L 3
+          current_commit=$(git rev-parse HEAD)
           REPO_NAME=${{ github.event.client_payload.repository }}
           REPO_NAME=${REPO_NAME##*/}
           file_name=$REPO_NAME.tar
@@ -46,18 +47,18 @@ jobs:
           tar -cf $REPO_NAME.tar ${{ github.event.client_payload.path }}
           python -m pip install bce-python-sdk==0.8.74
           # 获取远端的最新提交时间
-          latest_commit_time_remote=$(curl --silent --fail https://paddle-qa.bj.bcebos.com/CodeSync/GitInfo/${{ github.event.client_payload.repository }}/${{ github.event.client_payload.ref }}/latest_commit_time) || latest_commit_time_remote=0
+          latest_commit_time_remote=$(curl --silent --fail https://paddle-qa.bj.bcebos.com/CodeSync/GitInfo/${{ github.event.client_payload.repository }}/${{ github.event.client_payload.ref }}/${{ github.event.client_payload.commit_sha }}/latest_commit_time) || latest_commit_time_remote=0
           # 判断 latest_commit_time_remote 是否晚于 current_commit_time 时间
           if [ -n "$latest_commit_time_remote" ] && [ "$latest_commit_time_remote" -lt "$current_commit_time" ]; then
             echo "Executing upload as latest commit time ($latest_commit_time_remote) is earlier than current commit time ($current_commit_time)."
-            python tools/bos_upload.py ${file_name} paddle-qa/CodeSync/${{ github.event.client_payload.ref }}/
-            echo "Bos link: https://paddle-qa.bj.bcebos.com/CodeSync/${{ github.event.client_payload.ref }}/$file_name"
+            python tools/bos_upload.py ${file_name} paddle-qa/CodeSync/${{ github.event.client_payload.ref }}/${{ github.event.client_payload.commit_sha }}
+            echo "Bos link: https://paddle-qa.bj.bcebos.com/CodeSync/${{ github.event.client_payload.ref }}/${{ github.event.client_payload.commit_sha }}/$file_name"
             echo "${current_commit_time}" >> latest_commit_time
-            python tools/bos_upload.py latest_commit_time paddle-qa/CodeSync/GitInfo/${{ github.event.client_payload.repository }}/${{ github.event.client_payload.ref }}
+            python tools/bos_upload.py latest_commit_time paddle-qa/CodeSync/GitInfo/${{ github.event.client_payload.repository }}/${{ github.event.client_payload.ref }}/${{ github.event.client_payload.commit_sha }}
             echo "${current_commit}" >> latest_commit
-            python tools/bos_upload.py latest_commit paddle-qa/CodeSync/GitInfo/${{ github.event.client_payload.repository }}/${{ github.event.client_payload.ref }}
-            echo "GitInfo link: https://paddle-qa.bj.bcebos.com/CodeSync/GitInfo/${{ github.event.client_payload.repository }}/${{ github.event.client_payload.ref }}/latest_commit_sha"
-            echo "GitInfo link: https://paddle-qa.bj.bcebos.com/CodeSync/GitInfo/${{ github.event.client_payload.repository }}/${{ github.event.client_payload.ref }}/latest_commit_time"
+            python tools/bos_upload.py latest_commit paddle-qa/CodeSync/GitInfo/${{ github.event.client_payload.repository }}/${{ github.event.client_payload.ref }}/${{ github.event.client_payload.commit_sha }}
+            echo "GitInfo link: https://paddle-qa.bj.bcebos.com/CodeSync/GitInfo/${{ github.event.client_payload.repository }}/${{ github.event.client_payload.ref }}/${{ github.event.client_payload.commit_sha }}/latest_commit_sha"
+            echo "GitInfo link: https://paddle-qa.bj.bcebos.com/CodeSync/GitInfo/${{ github.event.client_payload.repository }}/${{ github.event.client_payload.ref }}/${{ github.event.client_payload.commit_sha }}/latest_commit_time"
           else
             echo "The latest commit time is later than the current commit. Skipping the commit operation."
           fi
diff --git a/.github/workflows/repo_sync_dispatch.yml b/.github/workflows/repo_sync_dispatch.yml
@@ -8,6 +8,9 @@ jobs:
   paddleqa-checkout:
     environment: CodeSync
     runs-on: ubuntu-latest
+    concurrency: 
+      group: exclusive-group-${{ github.workflow }}-${{ github.event.client_payload.repository }}-${{ github.event.client_payload.ref }}
+      cancel-in-progress: false
     name: Repo Checkout and Upload
     steps:
       - name: PaddleTest Checkout
@@ -17,13 +20,14 @@ jobs:
             tools
       - name: Code Checkout
         id: checkout
-        uses: XieYunshen/PaddleQACheckout@v1.0.0
+        uses: XieYunshen/PaddleQACheckout@v1.1.0
         with:
           repository: ${{ github.event.client_payload.repository }}
           ref: ${{ github.event.client_payload.ref }}
           submodules: ${{ github.event.client_payload.submodules }}
           fetch_depth: ${{ github.event.client_payload.fetch_depth }}
           path: ${{ github.event.client_payload.path }}
+          commit_sha: ${{ github.event.client_payload.commit_sha }}
       - uses: actions/setup-python@v5
         with:
           python-version: '3.10'
@@ -44,9 +48,19 @@ jobs:
           cd -
           tar -cf $REPO_NAME.tar ${{ github.event.client_payload.path }}
           python -m pip install bce-python-sdk==0.8.74
-          python tools/bos_upload.py ${file_name} paddle-qa/CodeSync/${{ github.event.client_payload.ref }}/
-          echo "Bos link: https://paddle-qa.bj.bcebos.com/CodeSync/${{ github.event.client_payload.ref }}/$file_name"
-          echo "${current_commit}" >> ${current_commit}
-          python tools/bos_upload.py ${current_commit} paddle-qa/CodeSync/${{ github.event.client_payload.repository }}/${{ github.event.client_payload.ref }}
-          echo "${current_commit_time}" >> latest_commit_time
-          python tools/bos_upload.py latest_commit_time paddle-qa/CodeSync/${{ github.event.client_payload.repository }}/${{ github.event.client_payload.ref }}
+          # 获取远端的最新提交时间
+          latest_commit_time_remote=$(curl --silent --fail https://paddle-qa.bj.bcebos.com/CodeSync/GitInfo/${{ github.event.client_payload.repository }}/${{ github.event.client_payload.ref }}/latest_commit_time) || latest_commit_time_remote=0
+          # 判断 latest_commit_time_remote 是否晚于 current_commit_time 时间
+          if [ -n "$latest_commit_time_remote" ] && [ "$latest_commit_time_remote" -lt "$current_commit_time" ]; then
+            echo "Executing upload as latest commit time ($latest_commit_time_remote) is earlier than current commit time ($current_commit_time)."
+            python tools/bos_upload.py ${file_name} paddle-qa/CodeSync/${{ github.event.client_payload.ref }}/
+            echo "Bos link: https://paddle-qa.bj.bcebos.com/CodeSync/${{ github.event.client_payload.ref }}/$file_name"
+            echo "${current_commit_time}" >> latest_commit_time
+            python tools/bos_upload.py latest_commit_time paddle-qa/CodeSync/GitInfo/${{ github.event.client_payload.repository }}/${{ github.event.client_payload.ref }}
+            echo "${current_commit}" >> latest_commit
+            python tools/bos_upload.py latest_commit paddle-qa/CodeSync/GitInfo/${{ github.event.client_payload.repository }}/${{ github.event.client_payload.ref }}
+            echo "GitInfo link: https://paddle-qa.bj.bcebos.com/CodeSync/GitInfo/${{ github.event.client_payload.repository }}/${{ github.event.client_payload.ref }}/latest_commit_sha"
+            echo "GitInfo link: https://paddle-qa.bj.bcebos.com/CodeSync/GitInfo/${{ github.event.client_payload.repository }}/${{ github.event.client_payload.ref }}/latest_commit_time"
+          else
+            echo "The latest commit time is later than the current commit. Skipping the commit operation."
+          fi