feat: Convert benchmark.sh to python and add unit tests (intel-retail#34

)

* feat: Convert benchmark.sh to python and add unit tests

Signed-off-by: Elizabeth J Lee <elizabeth.j.lee@intel.com>

benchmark-scripts/Makefile

@@ -1,3 +1,5 @@
+# Copyright © 2024 Intel Corporation. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
 
 build-all: build-benchmark build-xpu build-igt
 
@@ -17,4 +19,10 @@ run:
 	docker run -it --rm -v /var/run/docker.sock:/var/run/docker.sock -v `pwd`/results:/tmp/results --net=host --privileged benchmark:dev /bin/bash
 
 consolidate:
-	docker run -itd -v `pwd`/$(ROOT_DIRECTORY):/$(ROOT_DIRECTORY) -e ROOT_DIRECTORY=$(ROOT_DIRECTORY)--net=host --privileged benchmark:dev /bin/bash -c "python3 consolidate_multiple_run_of_metrics.py --root_directory $(ROOT_DIRECTORY)/ --output $(ROOT_DIRECTORY)/summary.csv"
+	docker run -itd -v `pwd`/$(ROOT_DIRECTORY):/$(ROOT_DIRECTORY) -e ROOT_DIRECTORY=$(ROOT_DIRECTORY)--net=host --privileged benchmark:dev /bin/bash -c "python3 consolidate_multiple_run_of_metrics.py --root_directory $(ROOT_DIRECTORY)/ --output $(ROOT_DIRECTORY)/summary.csv"
+
+python-test:
+	python -m coverage run -m unittest benchmark_test.py
+
+python-coverage:
+	coverage report -m

benchmark-scripts/benchmark.py

@@ -0,0 +1,119 @@
+'''
+* Copyright (C) 2024 Intel Corporation.
+*
+* SPDX-License-Identifier: Apache-2.0
+'''
+
+import argparse
+import os
+import shlex
+import subprocess  # nosec B404
+import time
+import traceback
+
+
+def parse_args(print=False):
+    parser = argparse.ArgumentParser(
+        prog='benchmark',
+        description='runs benchmarking using docker compose')
+    parser.add_argument('--pipelines', type=int, default=1,
+                        help='number of pipelines')
+    parser.add_argument('--target_fps', type=int, default=None,
+                        help='stream density target FPS')
+    # TODO: add variable for stream density increment when implementing
+    parser.add_argument('--results_dir',
+                        default=os.path.join(os.curdir, 'results'),
+                        help='full path to the desired directory for logs ' +
+                             'and results')
+    parser.add_argument('--duration', type=int, default=30,
+                        help='time in seconds, not needed when ' +
+                             '--stream_density is specified')
+    parser.add_argument('--init_duration', type=int, default=5,
+                        help='time in seconds')
+    # TODO: change target_device to an env variable in docker compose
+    parser.add_argument('--target_device', default='CPU',
+                        help='desired running platform [cpu|core|xeon|dgpu.x]')
+    parser.add_argument('--compose_file', default=None, action='append',
+                        help='path to docker compose files. ' +
+                             'can be used multiple times')
+    parser.add_argument('--retail_use_case_root',
+                        default=os.path.join(
+                            os.curdir, '..', '..', 'retail-use-cases'),
+                        help='full path to the retail-use-cases repo root')
+    if print:
+        parser.print_help()
+        return
+    return parser.parse_args()
+
+
+def docker_compose_containers(command, compose_files=[], compose_pre_args="",
+                              compose_post_args="",
+                              env_vars=os.environ.copy()):
+    try:
+        files = " -f ".join(compose_files)
+        compose_string = ("docker compose %s -f %s %s %s" %
+                          (compose_pre_args, files, command,
+                           compose_post_args))
+        compose_args = shlex.split(compose_string)
+
+        p = subprocess.Popen(compose_args,
+                             stdout=subprocess.PIPE,
+                             stderr=subprocess.PIPE,
+                             env=env_vars)  # nosec B404, B603
+        stdout, stderr = p.communicate()
+
+        if p.returncode and stderr:
+            print("Error bringing %s the compose files: %s" %
+                  (command, stderr))
+        return stdout.strip(), stderr, p.returncode
+    except subprocess.CalledProcessError:
+        print("Exception bringing %s the compose files: %s" %
+              (command, traceback.format_exc()))
+
+
+def main():
+    my_args = parse_args()
+
+    results_dir = os.path.abspath(my_args.results_dir)
+    if not os.path.exists(results_dir):
+        os.mkdir(results_dir)
+
+    print("Starting workload(s)")
+
+    # start the docker containers
+    # pass in necessary variables using env vars
+    compose_files = []
+    for file in my_args.compose_file:
+        compose_files.append(os.path.abspath(file))
+
+    # add the benchmark docker compose file
+    compose_files.append(os.path.abspath(os.path.join(
+        os.curdir, '..', 'docker', 'docker-compose.yaml')))
+    env_vars = os.environ.copy()
+    env_vars["log_dir"] = results_dir
+    env_vars["RESULTS_DIR"] = results_dir
+    env_vars["DEVICE"] = my_args.target_device
+    retail_use_case_root = os.path.abspath(my_args.retail_use_case_root)
+    env_vars["RETAIL_USE_CASE_ROOT"] = retail_use_case_root
+    if my_args.pipelines > 0:
+        env_vars["PIPELINE_COUNT"] = str(my_args.pipelines)
+
+    docker_compose_containers("up", compose_files=compose_files,
+                              compose_post_args="-d", env_vars=env_vars)
+    print("Waiting for init duration to complete...")
+    time.sleep(my_args.init_duration)
+
+    # use duration to sleep
+    print("Waiting for %d seconds for workload to finish" % my_args.duration)
+    time.sleep(my_args.duration)
+    # stop all containers and camera-simulator
+    docker_compose_containers("down", compose_files=compose_files,
+                              env_vars=env_vars)
+
+    # collect metrics using copy-platform-metrics
+    print("workloads finished...")
+    # TODO: implement results handling based on what pipeline is run
+
+
+if __name__ == '__main__':
+    main()

benchmark-scripts/benchmark_test.py

@@ -0,0 +1,59 @@
+'''
+* Copyright (C) 2024 Intel Corporation.
+*
+* SPDX-License-Identifier: Apache-2.0
+'''
+
+import mock
+import subprocess  # nosec B404
+import unittest
+import benchmark
+
+
+class Testing(unittest.TestCase):
+
+    class MockPopen(object):
+        def __init__(self):
+            pass
+
+        def communicate(self, input=None):
+            pass
+
+        @property
+        def returncode(self):
+            pass
+
+    def test_docker_compose_containers_success(self):
+        mock_popen = Testing.MockPopen()
+        mock_popen.communicate = mock.Mock(
+            return_value=('', '1Starting camera: rtsp://127.0.0.1:8554/' +
+                          'camera_0 from *.mp4'))
+        mock_returncode = mock.PropertyMock(return_value=0)
+        type(mock_popen).returncode = mock_returncode
+
+        setattr(subprocess, 'Popen', lambda *args, **kargs: mock_popen)
+        res = benchmark.docker_compose_containers('up')
+
+        self.assertEqual(res, ('',
+                               '1Starting camera: rtsp://127.0.0.1:8554/' +
+                               'camera_0 from *.mp4', 0))
+        mock_popen.communicate.assert_called_once_with()
+        mock_returncode.assert_called()
+
+    def test_docker_compose_containers_fail(self):
+        mock_popen = Testing.MockPopen()
+        mock_popen.communicate = mock.Mock(return_value=('',
+                                                         b'an error occurred'))
+        mock_returncode = mock.PropertyMock(return_value=1)
+        type(mock_popen).returncode = mock_returncode
+
+        setattr(subprocess, 'Popen', lambda *args, **kargs: mock_popen)
+        res = benchmark.docker_compose_containers('up')
+
+        self.assertEqual(res, ('', b'an error occurred', 1))
+        mock_popen.communicate.assert_called_once_with()
+        mock_returncode.assert_called()
+
+
+if __name__ == '__main__':
+    unittest.main()

benchmark-scripts/camera-simulator.sh

@@ -1,6 +1,6 @@
 #!/bin/bash -e
 #
-# Copyright (C) 2023 Intel Corporation.
+# Copyright (C) 2024 Intel Corporation.
 #
 # SPDX-License-Identifier: Apache-2.0
 #
@@ -21,8 +21,6 @@ if [ "${COMMAND,,}" = "start" ]; then
     if [ -z "$CAMERAS" ]; then
 	CAMERAS=${#FILES[@]}
     fi
-
-    cd $SOURCE_DIR/camera-simulator
 
     docker run --rm -t --network=host --name camera-simulator aler9/rtsp-simple-server >rtsp_simple_server.log.txt  2>&1 &
     index=0