diff --git a/.ci/scripts/setup-arm-baremetal-tools.sh b/.ci/scripts/setup-arm-baremetal-tools.sh new file mode 100755 index 0000000000..454b9f336e --- /dev/null +++ b/.ci/scripts/setup-arm-baremetal-tools.sh @@ -0,0 +1,11 @@ +#!/bin/bash +# Copyright 2024 Arm Limited and/or its affiliates. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# NB: This function could be used to install Arm dependencies +# Setup arm example environment (including TOSA tools) +git config --global user.email "github_executorch@arm.com" +git config --global user.name "Github Executorch" +bash examples/arm/setup.sh --i-agree-to-the-contained-eula diff --git a/.ci/scripts/utils.sh b/.ci/scripts/utils.sh index be927a3a3a..aa8d78da4b 100644 --- a/.ci/scripts/utils.sh +++ b/.ci/scripts/utils.sh @@ -59,17 +59,6 @@ install_flatc_from_source() { popd || return } -install_arm() { - # NB: This function could be used to install Arm dependencies - # Setup arm example environment (including TOSA tools) - git config --global user.email "github_executorch@arm.com" - git config --global user.name "Github Executorch" - bash examples/arm/setup.sh --i-agree-to-the-contained-eula - - # Test tosa_reference flow - source examples/arm/ethos-u-scratch/setup_path.sh -} - build_executorch_runner_buck2() { # Build executorch runtime with retry as this step is flaky on macos CI retry buck2 build //examples/portable/executor_runner:executor_runner diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index 5941ab52e7..d1b64e7598 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -354,13 +354,11 @@ jobs: EXECUTORCH_BUILD_ARM_BAREMETAL=ON \ .ci/scripts/setup-linux.sh "${BUILD_TOOL}" - source .ci/scripts/utils.sh # Install Arm dependencies - install_arm - - # Run pytest with coverage - pytest -c /dev/null -v -n auto --cov=./ --cov-report=xml backends/arm/test + .ci/scripts/setup-arm-baremetal-tools.sh + # Run pytest without simulator + backends/arm/test/test_arm_baremetal.sh test_pytest test-llama-runner-qnn-linux: name: test-llama-runner-qnn-linux diff --git a/.github/workflows/trunk.yml b/.github/workflows/trunk.yml index 7972269e92..90bd0eb6ef 100644 --- a/.github/workflows/trunk.yml +++ b/.github/workflows/trunk.yml @@ -146,14 +146,15 @@ jobs: source .ci/scripts/utils.sh install_executorch - install_arm + .ci/scripts/setup-arm-baremetal-tools.sh # Increase number of files user can monitor to bypass buck failures. # Hopefully this is high enough for this setup. sudo sysctl fs.inotify.max_user_watches=1048576 # 1024 * 1024 # Test ethos-u delegate examples with run.sh - PYTHON_EXECUTABLE=python bash examples/arm/run.sh examples/arm/ethos-u-scratch/ + backends/arm/test/test_arm_baremetal.sh test_run_ethosu_fvp + test-arm-reference-delegation: name: test-arm-reference-delegation @@ -172,10 +173,10 @@ jobs: source .ci/scripts/utils.sh install_executorch - install_arm + .ci/scripts/setup-arm-baremetal-tools.sh - # Run arm unit tests - pytest -c /dev/null -v -n auto --cov=./ --cov-report=xml backends/arm/test + # Run arm unit tests using the simulator + backends/arm/test/test_arm_baremetal.sh test_pytest_ethosu_fvp test-coreml-delegate: name: test-coreml-delegate diff --git a/backends/arm/README.md b/backends/arm/README.md index a7458db07c..ddfc4b098f 100644 --- a/backends/arm/README.md +++ b/backends/arm/README.md @@ -39,6 +39,28 @@ Other: - `third-party/` - Dependencies on other code - in particular the TOSA serialization_lib for compiling to TOSA and the ethos-u-core-driver for the bare-metal backend supporting Ethos-U - `test/` - Unit test and test support functions +## Testing + +After a setup you can run unit tests with the test_arm_baremetal.sh script. + +To run the pytests suite run + +``` +backends/arm/test/test_arm_baremetal.sh test_pytest +``` + +To run the unit test suite with Corstone3x0 FVP simulator support use + +``` +backends/arm/test/test_arm_baremetal.sh test_pytest_ethosu_fvp +``` + +You can test to run some models with the run.sh flow + +``` +backends/arm/test/test_arm_baremetal.sh test_run_ethosu_fvp +``` + ## Unit tests This is the structure of the test directory @@ -51,6 +73,8 @@ test # Root test folder ├── tester # Arm Tester class ├── tosautil # Utility functions for TOSA artifacts ├ common.py # Common functions and definitions used by many tests +├ setup_testing.sh # Script to prepare testing for using the Corstone 3x0 FVP +├ test_arm_baremetal.sh # Help script to trigger testing ``` Some example commands to run these tests follow. Run a single test: @@ -59,6 +83,12 @@ Some example commands to run these tests follow. Run a single test: python -m unittest backends.arm.test.ops.test_add.TestSimpleAdd -k test_add2_tosa_BI ``` +or with pytest + +``` +pytest -c /dev/null -v -n auto backends/arm/test/ops/test_add.py -k test_add2_tosa_BI +``` + Or all tests in "TestSimpleAdd": ``` @@ -71,6 +101,28 @@ Or discover and run many tests: python -m unittest discover -s backends/arm/test/ops/ ``` +or with pytest + +``` +pytest -c /dev/null -v -n auto backends/arm/test/ops/ +``` + + +You can run tests using Corstone3x0 simulators to see how it would work on something more target like +first you need to build and prepare some used target libs + +``` +examples/arm/run.sh --model_name=add --build_only +backends/arm/test/setup_testing.sh +``` + +The you can run the tests with + +``` +pytest -c /dev/null -v -n auto backends/arm/test --arm_quantize_io --arm_run_corstoneFVP +``` + + ### A note on unit tests There are currently 3 ways we unit test our code. diff --git a/backends/arm/test/runner_utils.py b/backends/arm/test/runner_utils.py index 9ae1a27cf7..b9dc4e394d 100644 --- a/backends/arm/test/runner_utils.py +++ b/backends/arm/test/runner_utils.py @@ -178,7 +178,7 @@ def __init__( self.output_name: str = None self.qp_input: list[QuantizationParams] = None self.qp_output: QuantizationParams = None - self.timeout = 120 + self.timeout = 480 self.target_board: str = None self._has_init_run = False @@ -316,7 +316,7 @@ def run_corstone( result = _run_cmd(command_args[self.target_board], check=False) if result.returncode != 0: raise RuntimeError( - f"Failed to run {command_args[self.target_board]}\nError: {result.stderr.decode()}" + f"Failed to run {command_args[self.target_board]}\nOutput:\n{result.stdout.decode()}\nError: {result.stderr.decode()}" ) result_stdout = result.stdout.decode() diff --git a/backends/arm/test/setup_testing.sh b/backends/arm/test/setup_testing.sh index 0562604dd9..3681917865 100755 --- a/backends/arm/test/setup_testing.sh +++ b/backends/arm/test/setup_testing.sh @@ -14,7 +14,6 @@ ethos_u_root_dir=${et_root_dir}/examples/arm/ethos-u-scratch/ethos-u toolchain_cmake=${et_root_dir}/examples/arm/ethos-u-setup/arm-none-eabi-gcc.cmake et_build_dir=${et_root_dir}/cmake-out build_root_test_dir=${et_build_dir}/arm_semihosting_executor_runner -fvp_model=FVP_Corstone_SSE-300_Ethos-U55 # Build Arm Baremetal executor_runner in semihosting mode. # Put in backends/arm/test/res to be used by unit tests. diff --git a/backends/arm/test/test_arm_baremetal.sh b/backends/arm/test/test_arm_baremetal.sh new file mode 100755 index 0000000000..18ea4fe216 --- /dev/null +++ b/backends/arm/test/test_arm_baremetal.sh @@ -0,0 +1,108 @@ +#!/bin/bash +# Copyright 2024 Arm Limited and/or its affiliates. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +set -e + +script_dir=$(cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd) + +# Executorch root +et_root_dir=$(cd ${script_dir}/../../.. && pwd) +cd "${et_root_dir}" +pwd + + +TEST_SUITE=$1 + +help() { + echo "Usage:" + echo " $0 " + echo " where can be any of:" + # This will list all lines in this file that is starting with test_ remove () { and print it as a list. + # e,g, "test_pytest() { # Test ops and other things" -> test_pytest # Test ops and other things + echo "all # run all tests" + grep "^test_" $0 | sed 's/([^)]*)[[:space:]]*{*//g' + exit +} + +if [[ -z "${TEST_SUITE:-}" ]]; then + echo "Missing test suite name, exiting..." + help +else + echo "Run Arm baremetal test suite ${TEST_SUITE}" +fi + +TEST_SUITE_NAME="$(basename "$0") ${TEST_SUITE}" + +all() { # Run all tests + # This will list all lines in this file that is starting with test_ remove () { and add this script name in + # front of it and execute it in a sub shell + # e.g. from this file: + # + # test_pytest() { # Test ops and other things + # bla bla bla + # } + # test_pytest_ethosu_fvp() { # Same as test_pytest but ... + # bla bla bla + # } + #... + # become a small script: + # ---- + # backends/arm/test/test_arm_baremetal.sh test_pytest # Test ops and other things + # backends/arm/test/test_arm_baremetal.sh test_pytest_ethosu_fvp # Same as test_pytest but ... + # ... + # ---- + # That is executed + echo "${TEST_SUITE_NAME}: Run all tests" + grep "^test_" backends/arm/test/test_arm_baremetal.sh | sed 's/([^)]*)[[:space:]]*{*//g' | sed "s|^|$0 |" | sh +} + +test_pytest() { # Test ops and other things + echo "${TEST_SUITE_NAME}: Run pytest" + cd "${et_root_dir}" + source examples/arm/ethos-u-scratch/setup_path.sh + + # Run arm baremetal pytest tests without FVP + pytest --config-file=/dev/null --verbose --color=yes --numprocesses=auto backends/arm/test/ +} + +test_pytest_ethosu_fvp() { # Same as test_pytest but also sometime verify using Corstone FVP + echo "${TEST_SUITE_NAME}: Run pytest with fvp" + + source examples/arm/ethos-u-scratch/setup_path.sh + + # Prepare Corstone-3x0 FVP for pytest + examples/arm/run.sh --model_name=add --build_only + backends/arm/test/setup_testing.sh + + # Run arm baremetal pytest tests with FVP + pytest --config-file=/dev/null --verbose --color=yes --numprocesses=auto backends/arm/test/ --arm_quantize_io --arm_run_corstoneFVP +} + +test_run_ethosu_fvp() { # End to End model tests + echo "${TEST_SUITE_NAME}: Test ethos-u delegate examples with run.sh" + + source examples/arm/ethos-u-scratch/setup_path.sh + + # TOSA quantized + echo "${TEST_SUITE_NAME}: Test ethos-u target TOSA" + examples/arm/run.sh --target=TOSA --model_name=mv2 + examples/arm/run.sh --target=TOSA --model_name=lstm + examples/arm/run.sh --target=TOSA --model_name=esdr + examples/arm/run.sh --target=TOSA --model_name=emformer_join + examples/arm/run.sh --target=TOSA --model_name=w2l + + # Ethos-U55 + echo "${TEST_SUITE_NAME}: Test ethos-u target Ethos-U55" + examples/arm/run.sh --target=ethos-u55-128 --model_name=mv2 + examples/arm/run.sh --target=ethos-u55-128 --model_name=lstm --reorder_inputs=1,0,2 + + # Ethos-U85 + echo "${TEST_SUITE_NAME}: Test ethos-u target Ethos-U85" + examples/arm/run.sh --target=ethos-u85-128 --model_name=mv2 + examples/arm/run.sh --target=ethos-u85-128 --model_name=lstm --reorder_inputs=1,0,2 + } + +${TEST_SUITE} \ No newline at end of file diff --git a/backends/arm/test/tester/arm_tester.py b/backends/arm/test/tester/arm_tester.py index f663b16a9d..b3f5b4f05b 100644 --- a/backends/arm/test/tester/arm_tester.py +++ b/backends/arm/test/tester/arm_tester.py @@ -251,7 +251,7 @@ def to_executorch(self, to_executorch_stage: Optional[ToExecutorch] | None = Non return super().to_executorch(to_executorch_stage) def serialize( - self, serialize_stage: Optional[Serialize] = None, timeout: int = 120 + self, serialize_stage: Optional[Serialize] = None, timeout: int = 480 ): if serialize_stage is None: serialize_stage = Serialize(self.runner_util, timeout=timeout)