diff --git a/.github/workflows/validate-aarch64-linux-binaries.yml b/.github/workflows/validate-aarch64-linux-binaries.yml
index 486e5319de..04bc95f7f7 100644
--- a/.github/workflows/validate-aarch64-linux-binaries.yml
+++ b/.github/workflows/validate-aarch64-linux-binaries.yml
@@ -113,6 +113,7 @@ jobs:
       job-name: ${{ matrix.build_name }}
       docker-image: ${{ matrix.container_image }}
       binary-matrix: ${{ toJSON(matrix) }}
+      docker-build-dir: "skip-docker-build"
       no-sudo: true
       script: |
         set -ex
@@ -154,4 +155,4 @@ jobs:
         unset LD_LIBRARY_PATH
 
         # Standard case: Validate binaries
-        source ../../.github/scripts/validate_binaries.sh
+        source ../../test-infra/.github/scripts/validate_binaries.sh
diff --git a/.github/workflows/validate-linux-binaries.yml b/.github/workflows/validate-linux-binaries.yml
index 1110a758d3..30cb39792e 100644
--- a/.github/workflows/validate-linux-binaries.yml
+++ b/.github/workflows/validate-linux-binaries.yml
@@ -155,10 +155,6 @@ jobs:
 
         export USE_FORCE_REINSTALL="true"
         export TARGET_OS="linux"
-        # Due to xpu doesn't use pytorch/conda-builder image, need to install conda firstly
-        if [[ ${{ matrix.gpu_arch_type }} == 'xpu' ]]; then
-          source /.ci/docker/common/install_conda.sh
-        fi
         eval "$(conda shell.bash hook)"
         printf '%s\n' ${{ toJson(inputs.release-matrix) }} > release_matrix.json
 
diff --git a/.github/workflows/validate-nightly-binaries.yml b/.github/workflows/validate-nightly-binaries.yml
index 5c66d4a59e..b63420fa17 100644
--- a/.github/workflows/validate-nightly-binaries.yml
+++ b/.github/workflows/validate-nightly-binaries.yml
@@ -15,6 +15,7 @@ on:
       - .github/workflows/validate-linux-binaries.yml
       - .github/workflows/validate-windows-binaries.yml
       - .github/workflows/validate-macos-arm64-binaries.yml
+      - .github/workflows/validate-aarch64-linux-binaries.yml
       - test/smoke_test/*
   pull_request:
     paths:
@@ -22,6 +23,7 @@ on:
       - .github/workflows/validate-linux-binaries.yml
       - .github/workflows/validate-windows-binaries.yml
       - .github/workflows/validate-macos-arm64-binaries.yml
+      - .github/workflows/validate-aarch64-linux-binaries.yml
       - .github/scripts/validate_binaries.sh
       - test/smoke_test/*