From 61eb5920cce3f90c891187ff0dd08c72b8c837cf Mon Sep 17 00:00:00 2001 From: Zingo Andersen Date: Thu, 14 Nov 2024 16:22:40 +0100 Subject: [PATCH] Arm backend: Test TOSA, Ethos-U55 and Ethos-U85 on github This will run some more models in the github test flow and also enable some unit tests to use Corston3x0 FVP. This structure up the Arm backend testing into separate runable scripts in the same structure as other backends. Signed-off-by: Zingo Andersen Change-Id: I5e11b1aca19460845e330b84d0696513c400c0f0 --- .ci/scripts/setup-arm-baremetal-tools.sh | 11 +++ .ci/scripts/utils.sh | 11 --- .github/workflows/pull.yml | 8 +- .github/workflows/trunk.yml | 11 +-- backends/arm/README.md | 52 +++++++++++ backends/arm/test/runner_utils.py | 4 +- backends/arm/test/setup_testing.sh | 1 - backends/arm/test/test_arm_baremetal.sh | 108 +++++++++++++++++++++++ backends/arm/test/tester/arm_tester.py | 2 +- 9 files changed, 183 insertions(+), 25 deletions(-) create mode 100755 .ci/scripts/setup-arm-baremetal-tools.sh create mode 100755 backends/arm/test/test_arm_baremetal.sh diff --git a/.ci/scripts/setup-arm-baremetal-tools.sh b/.ci/scripts/setup-arm-baremetal-tools.sh new file mode 100755 index 0000000000..454b9f336e --- /dev/null +++ b/.ci/scripts/setup-arm-baremetal-tools.sh @@ -0,0 +1,11 @@ +#!/bin/bash +# Copyright 2024 Arm Limited and/or its affiliates. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# NB: This function could be used to install Arm dependencies +# Setup arm example environment (including TOSA tools) +git config --global user.email "github_executorch@arm.com" +git config --global user.name "Github Executorch" +bash examples/arm/setup.sh --i-agree-to-the-contained-eula diff --git a/.ci/scripts/utils.sh b/.ci/scripts/utils.sh index be927a3a3a..aa8d78da4b 100644 --- a/.ci/scripts/utils.sh +++ b/.ci/scripts/utils.sh @@ -59,17 +59,6 @@ install_flatc_from_source() { popd || return } -install_arm() { - # NB: This function could be used to install Arm dependencies - # Setup arm example environment (including TOSA tools) - git config --global user.email "github_executorch@arm.com" - git config --global user.name "Github Executorch" - bash examples/arm/setup.sh --i-agree-to-the-contained-eula - - # Test tosa_reference flow - source examples/arm/ethos-u-scratch/setup_path.sh -} - build_executorch_runner_buck2() { # Build executorch runtime with retry as this step is flaky on macos CI retry buck2 build //examples/portable/executor_runner:executor_runner diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index 5941ab52e7..d1b64e7598 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -354,13 +354,11 @@ jobs: EXECUTORCH_BUILD_ARM_BAREMETAL=ON \ .ci/scripts/setup-linux.sh "${BUILD_TOOL}" - source .ci/scripts/utils.sh # Install Arm dependencies - install_arm - - # Run pytest with coverage - pytest -c /dev/null -v -n auto --cov=./ --cov-report=xml backends/arm/test + .ci/scripts/setup-arm-baremetal-tools.sh + # Run pytest without simulator + backends/arm/test/test_arm_baremetal.sh test_pytest test-llama-runner-qnn-linux: name: test-llama-runner-qnn-linux diff --git a/.github/workflows/trunk.yml b/.github/workflows/trunk.yml index 7972269e92..90bd0eb6ef 100644 --- a/.github/workflows/trunk.yml +++ b/.github/workflows/trunk.yml @@ -146,14 +146,15 @@ jobs: source .ci/scripts/utils.sh install_executorch - install_arm + .ci/scripts/setup-arm-baremetal-tools.sh # Increase number of files user can monitor to bypass buck failures. # Hopefully this is high enough for this setup. sudo sysctl fs.inotify.max_user_watches=1048576 # 1024 * 1024 # Test ethos-u delegate examples with run.sh - PYTHON_EXECUTABLE=python bash examples/arm/run.sh examples/arm/ethos-u-scratch/ + backends/arm/test/test_arm_baremetal.sh test_run_ethosu_fvp + test-arm-reference-delegation: name: test-arm-reference-delegation @@ -172,10 +173,10 @@ jobs: source .ci/scripts/utils.sh install_executorch - install_arm + .ci/scripts/setup-arm-baremetal-tools.sh - # Run arm unit tests - pytest -c /dev/null -v -n auto --cov=./ --cov-report=xml backends/arm/test + # Run arm unit tests using the simulator + backends/arm/test/test_arm_baremetal.sh test_pytest_ethosu_fvp test-coreml-delegate: name: test-coreml-delegate diff --git a/backends/arm/README.md b/backends/arm/README.md index a7458db07c..ddfc4b098f 100644 --- a/backends/arm/README.md +++ b/backends/arm/README.md @@ -39,6 +39,28 @@ Other: - `third-party/` - Dependencies on other code - in particular the TOSA serialization_lib for compiling to TOSA and the ethos-u-core-driver for the bare-metal backend supporting Ethos-U - `test/` - Unit test and test support functions +## Testing + +After a setup you can run unit tests with the test_arm_baremetal.sh script. + +To run the pytests suite run + +``` +backends/arm/test/test_arm_baremetal.sh test_pytest +``` + +To run the unit test suite with Corstone3x0 FVP simulator support use + +``` +backends/arm/test/test_arm_baremetal.sh test_pytest_ethosu_fvp +``` + +You can test to run some models with the run.sh flow + +``` +backends/arm/test/test_arm_baremetal.sh test_run_ethosu_fvp +``` + ## Unit tests This is the structure of the test directory @@ -51,6 +73,8 @@ test # Root test folder ├── tester # Arm Tester class ├── tosautil # Utility functions for TOSA artifacts ├ common.py # Common functions and definitions used by many tests +├ setup_testing.sh # Script to prepare testing for using the Corstone 3x0 FVP +├ test_arm_baremetal.sh # Help script to trigger testing ``` Some example commands to run these tests follow. Run a single test: @@ -59,6 +83,12 @@ Some example commands to run these tests follow. Run a single test: python -m unittest backends.arm.test.ops.test_add.TestSimpleAdd -k test_add2_tosa_BI ``` +or with pytest + +``` +pytest -c /dev/null -v -n auto backends/arm/test/ops/test_add.py -k test_add2_tosa_BI +``` + Or all tests in "TestSimpleAdd": ``` @@ -71,6 +101,28 @@ Or discover and run many tests: python -m unittest discover -s backends/arm/test/ops/ ``` +or with pytest + +``` +pytest -c /dev/null -v -n auto backends/arm/test/ops/ +``` + + +You can run tests using Corstone3x0 simulators to see how it would work on something more target like +first you need to build and prepare some used target libs + +``` +examples/arm/run.sh --model_name=add --build_only +backends/arm/test/setup_testing.sh +``` + +The you can run the tests with + +``` +pytest -c /dev/null -v -n auto backends/arm/test --arm_quantize_io --arm_run_corstoneFVP +``` + + ### A note on unit tests There are currently 3 ways we unit test our code. diff --git a/backends/arm/test/runner_utils.py b/backends/arm/test/runner_utils.py index 9ae1a27cf7..b9dc4e394d 100644 --- a/backends/arm/test/runner_utils.py +++ b/backends/arm/test/runner_utils.py @@ -178,7 +178,7 @@ def __init__( self.output_name: str = None self.qp_input: list[QuantizationParams] = None self.qp_output: QuantizationParams = None - self.timeout = 120 + self.timeout = 480 self.target_board: str = None self._has_init_run = False @@ -316,7 +316,7 @@ def run_corstone( result = _run_cmd(command_args[self.target_board], check=False) if result.returncode != 0: raise RuntimeError( - f"Failed to run {command_args[self.target_board]}\nError: {result.stderr.decode()}" + f"Failed to run {command_args[self.target_board]}\nOutput:\n{result.stdout.decode()}\nError: {result.stderr.decode()}" ) result_stdout = result.stdout.decode() diff --git a/backends/arm/test/setup_testing.sh b/backends/arm/test/setup_testing.sh index 0562604dd9..3681917865 100755 --- a/backends/arm/test/setup_testing.sh +++ b/backends/arm/test/setup_testing.sh @@ -14,7 +14,6 @@ ethos_u_root_dir=${et_root_dir}/examples/arm/ethos-u-scratch/ethos-u toolchain_cmake=${et_root_dir}/examples/arm/ethos-u-setup/arm-none-eabi-gcc.cmake et_build_dir=${et_root_dir}/cmake-out build_root_test_dir=${et_build_dir}/arm_semihosting_executor_runner -fvp_model=FVP_Corstone_SSE-300_Ethos-U55 # Build Arm Baremetal executor_runner in semihosting mode. # Put in backends/arm/test/res to be used by unit tests. diff --git a/backends/arm/test/test_arm_baremetal.sh b/backends/arm/test/test_arm_baremetal.sh new file mode 100755 index 0000000000..18ea4fe216 --- /dev/null +++ b/backends/arm/test/test_arm_baremetal.sh @@ -0,0 +1,108 @@ +#!/bin/bash +# Copyright 2024 Arm Limited and/or its affiliates. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +set -e + +script_dir=$(cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd) + +# Executorch root +et_root_dir=$(cd ${script_dir}/../../.. && pwd) +cd "${et_root_dir}" +pwd + + +TEST_SUITE=$1 + +help() { + echo "Usage:" + echo " $0 " + echo " where can be any of:" + # This will list all lines in this file that is starting with test_ remove () { and print it as a list. + # e,g, "test_pytest() { # Test ops and other things" -> test_pytest # Test ops and other things + echo "all # run all tests" + grep "^test_" $0 | sed 's/([^)]*)[[:space:]]*{*//g' + exit +} + +if [[ -z "${TEST_SUITE:-}" ]]; then + echo "Missing test suite name, exiting..." + help +else + echo "Run Arm baremetal test suite ${TEST_SUITE}" +fi + +TEST_SUITE_NAME="$(basename "$0") ${TEST_SUITE}" + +all() { # Run all tests + # This will list all lines in this file that is starting with test_ remove () { and add this script name in + # front of it and execute it in a sub shell + # e.g. from this file: + # + # test_pytest() { # Test ops and other things + # bla bla bla + # } + # test_pytest_ethosu_fvp() { # Same as test_pytest but ... + # bla bla bla + # } + #... + # become a small script: + # ---- + # backends/arm/test/test_arm_baremetal.sh test_pytest # Test ops and other things + # backends/arm/test/test_arm_baremetal.sh test_pytest_ethosu_fvp # Same as test_pytest but ... + # ... + # ---- + # That is executed + echo "${TEST_SUITE_NAME}: Run all tests" + grep "^test_" backends/arm/test/test_arm_baremetal.sh | sed 's/([^)]*)[[:space:]]*{*//g' | sed "s|^|$0 |" | sh +} + +test_pytest() { # Test ops and other things + echo "${TEST_SUITE_NAME}: Run pytest" + cd "${et_root_dir}" + source examples/arm/ethos-u-scratch/setup_path.sh + + # Run arm baremetal pytest tests without FVP + pytest --config-file=/dev/null --verbose --color=yes --numprocesses=auto backends/arm/test/ +} + +test_pytest_ethosu_fvp() { # Same as test_pytest but also sometime verify using Corstone FVP + echo "${TEST_SUITE_NAME}: Run pytest with fvp" + + source examples/arm/ethos-u-scratch/setup_path.sh + + # Prepare Corstone-3x0 FVP for pytest + examples/arm/run.sh --model_name=add --build_only + backends/arm/test/setup_testing.sh + + # Run arm baremetal pytest tests with FVP + pytest --config-file=/dev/null --verbose --color=yes --numprocesses=auto backends/arm/test/ --arm_quantize_io --arm_run_corstoneFVP +} + +test_run_ethosu_fvp() { # End to End model tests + echo "${TEST_SUITE_NAME}: Test ethos-u delegate examples with run.sh" + + source examples/arm/ethos-u-scratch/setup_path.sh + + # TOSA quantized + echo "${TEST_SUITE_NAME}: Test ethos-u target TOSA" + examples/arm/run.sh --target=TOSA --model_name=mv2 + examples/arm/run.sh --target=TOSA --model_name=lstm + examples/arm/run.sh --target=TOSA --model_name=esdr + examples/arm/run.sh --target=TOSA --model_name=emformer_join + examples/arm/run.sh --target=TOSA --model_name=w2l + + # Ethos-U55 + echo "${TEST_SUITE_NAME}: Test ethos-u target Ethos-U55" + examples/arm/run.sh --target=ethos-u55-128 --model_name=mv2 + examples/arm/run.sh --target=ethos-u55-128 --model_name=lstm --reorder_inputs=1,0,2 + + # Ethos-U85 + echo "${TEST_SUITE_NAME}: Test ethos-u target Ethos-U85" + examples/arm/run.sh --target=ethos-u85-128 --model_name=mv2 + examples/arm/run.sh --target=ethos-u85-128 --model_name=lstm --reorder_inputs=1,0,2 + } + +${TEST_SUITE} \ No newline at end of file diff --git a/backends/arm/test/tester/arm_tester.py b/backends/arm/test/tester/arm_tester.py index f663b16a9d..b3f5b4f05b 100644 --- a/backends/arm/test/tester/arm_tester.py +++ b/backends/arm/test/tester/arm_tester.py @@ -251,7 +251,7 @@ def to_executorch(self, to_executorch_stage: Optional[ToExecutorch] | None = Non return super().to_executorch(to_executorch_stage) def serialize( - self, serialize_stage: Optional[Serialize] = None, timeout: int = 120 + self, serialize_stage: Optional[Serialize] = None, timeout: int = 480 ): if serialize_stage is None: serialize_stage = Serialize(self.runner_util, timeout=timeout)