Merge pull request #12 from brian-intel/gitIgnore

benchmark-scripts/Dockerfile.xpu

@@ -48,7 +48,7 @@ WORKDIR /tmp/work
 RUN apt-get update && \
   apt-get install -y --no-install-recommends wget gnupg2 ca-certificates && \
   wget -qO - https://repositories.intel.com/graphics/intel-graphics.key | gpg --dearmor --output /usr/share/keyrings/intel-graphics.gpg && \
-  wget https://github.com/intel/xpumanager/releases/download/V1.2.3/xpumanager_1.2.3_20230221.054746.0e2d4bfb+ubuntu22.04_amd64.deb && \
+  wget https://github.com/intel/xpumanager/releases/download/V1.2.24/xpumanager_1.2.24_20231120.070911.ddc18e8a.u22.04_amd64.deb && \
   echo 'deb [arch=amd64 signed-by=/usr/share/keyrings/intel-graphics.gpg] https://repositories.intel.com/graphics/ubuntu jammy flex' | \
   tee /etc/apt/sources.list.d/intel.gpu.jammy.list && \
   apt-get update && \

benchmark-scripts/Makefile

@@ -0,0 +1,20 @@
+
+build-all: build-benchmark build-xpu build-igt
+
+build-benchmark:
+	echo "Building benchmark container HTTPS_PROXY=${HTTPS_PROXY} HTTP_PROXY=${HTTP_PROXY}"
+	docker build --build-arg HTTPS_PROXY=${HTTPS_PROXY} --build-arg HTTP_PROXY=${HTTP_PROXY} -t benchmark:dev -f Dockerfile.benchmark .
+
+build-xpu:
+	echo "Building xpu HTTPS_PROXY=${HTTPS_PROXY} HTTP_PROXY=${HTTP_PROXY}"
+	docker build --build-arg HTTPS_PROXY=${HTTPS_PROXY} --build-arg HTTP_PROXY=${HTTP_PROXY} -t benchmark:xpu -f Dockerfile.xpu .
+
+build-igt:
+	echo "Building igt HTTPS_PROXY=${HTTPS_PROXY} HTTP_PROXY=${HTTP_PROXY}"
+	docker build --build-arg HTTPS_PROXY=${HTTPS_PROXY} --build-arg HTTP_PROXY=${HTTP_PROXY} -t benchmark:igt -f Dockerfile.igt .
+
+run:
+	docker run -it --rm -v /var/run/docker.sock:/var/run/docker.sock -v `pwd`/results:/tmp/results --net=host --privileged benchmark:dev /bin/bash
+
+consolidate:
+	docker run -itd -v `pwd`/$(ROOT_DIRECTORY):/$(ROOT_DIRECTORY) -e ROOT_DIRECTORY=$(ROOT_DIRECTORY)--net=host --privileged benchmark:dev /bin/bash -c "python3 consolidate_multiple_run_of_metrics.py --root_directory $(ROOT_DIRECTORY)/ --output $(ROOT_DIRECTORY)/summary.csv"

benchmark-scripts/benchmark.sh

@@ -12,26 +12,22 @@ error() {
 
 show_help() {
         echo "
-         usage: $0 
+         usage: PIPELINE_PROFILE=\"object_detection\" (or others from make list-profiles) sudo -E ./$0
            --performance_mode the system performance setting [powersave | performance]
-           --pipelines NUMBER_OF_PIPELINES | --stream_density TARGET_FPS 
+           --pipelines NUMBER_OF_PIPELINES | --stream_density TARGET_FPS [PIPELINE_INCREMENT]
            --logdir FULL_PATH_TO_DIRECTORY 
            --duration SECONDS (not needed when --stream_density is specified)
            --init_duration SECONDS 
            --platform core|xeon|dgpu.x 
-           --inputsrc RS_SERIAL_NUMBER|CAMERA_RTSP_URL|file:video.mp4|/dev/video0 
-           [--classification_disabled] 
-           [ --ocr_disabled | --ocr [OCR_INTERVAL OCR_DEVICE] ] 
-           [ --barcode_disabled | --barcode [BARCODE_INTERVAL] ]
-           [--realsense_enabled]
+           --inputsrc RS_SERIAL_NUMBER|CAMERA_RTSP_URL|file:video.mp4|/dev/video0
 
          Note: 
           1. dgpu.x should be replaced with targetted GPUs such as dgpu (for all GPUs), dgpu.0, dgpu.1, etc
           2. filesrc will utilize videos stored in the sample-media folder
           3. Set environment variable STREAM_DENSITY_MODE=1 for starting single container stream density testing
           4. Set environment variable RENDER_MODE=1 for displaying pipeline and overlay CV metadata
-          5. Stream density can take two parameters: first one is for target fps, a float type value, and
-             the second one is increment integer of pipelines and is optional (in which case the increments will be dynamically adjusted internally)
+          5. Stream density can take two parameters: first one TARGET_FPS is for target fps, a float type value, and
+             the second one PIPELINE_INCREMENT is increment integer of pipelines and is optional (in which case the increments will be dynamically adjusted internally)
         "
 }
 
@@ -138,14 +134,10 @@ get_options() {
 # USAGE: 
 # 1. PLATFORM: core|xeon|dgpu.x
 # 2. INPUT SOURCE: RS_SERIAL_NUMBER|CAMERA_RTSP_URL|file:video.mp4|/dev/video0
-# 3. CLASSIFICATION: enabled|disabled
-# 4. OCR: disabled|OCR_INTERVAL OCR_DEVICE
-# 5. BARCODE: disabled|BARCODE_INTERVAL
-# 6. REALSENSE: enabled|disabled
-# 7. PIPELINE_NUMBER: the number of pipelines to start or specify MAX and a stream density benchmark will be performed with a 15 fps target per pipeline
-# 8. LOG_DIRECTORY: the location to store all the log files. The consolidation script will look for directories within the top level directory and process the results in each one so the user will want to keep in mind this structure when creating the log directory. For example, for multiple videos with different number of objects, a log_directory would look like: yolov5s_6330N/object1_mixed. Whatever is meaningful for the test run.
-# 9. DURATION: the amount of time to run the data collection
-# 10 COMPLETE_INIT_DURATION: the amount of time to allow the system to settle prior to starting the data collection.
+# 3. PIPELINE_NUMBER: the number of pipelines to start or specify MAX and a stream density benchmark will be performed with a 15 fps target per pipeline
+# 4. LOG_DIRECTORY: the location to store all the log files. The consolidation script will look for directories within the top level directory and process the results in each one so the user will want to keep in mind this structure when creating the log directory. For example, for multiple videos with different number of objects, a log_directory would look like: yolov5s_6330N/object1_mixed. Whatever is meaningful for the test run.
+# 5. DURATION: the amount of time to run the data collection
+# 6 COMPLETE_INIT_DURATION: the amount of time to allow the system to settle prior to starting the data collection.
 
 # load benchmark params
 if [ -z $1 ]
@@ -154,7 +146,7 @@ then
 fi
 get_options "$@"
 
-# load docker-run params
+# load run params
 shift $OPTIONS_TO_SKIP
 # the following syntax for arguments is meant to be re-splitting for correctly used on all $DOCKER_RUN_ARGS
 # shellcheck disable=SC2068
@@ -209,15 +201,32 @@ do
     NUM_GPU=$GPU_NUM_170
   fi
 
-  # docker-run needs to run in it's directory for the file paths to work
+  # run.sh needs to run in it's directory for the file paths to work
   cd ../
+#  pwd
 
-  echo "DEBUG: docker-run.sh" "$@"
+  echo "DEBUG: run.sh" "$@"
 
-  for pipelineIdx in $( seq 0 $(($PIPELINE_COUNT - 1)) )
+  for pipelineIdx in $( seq 0 $((PIPELINE_COUNT - 1)) )
   do
-    if [ -z "$STREAM_DENSITY_FPS" ]; then 
-      #pushd ..
+    if [ -z "$STREAM_DENSITY_FPS" ]; then
+      isCapi=$(docker run --rm -v "${PWD}":/workdir mikefarah/yq '.OvmsSingleContainer' /workdir/configs/opencv-ovms/cmd_client/res/"$PIPELINE_PROFILE"/configuration.yaml)
+      if [ "$isCapi" = false ]
+      then
+        echo "multiple pipelines for non-capi case..."
+        while true
+        do
+          containerCnt=$(docker ps -aq -f name="_ovms_pl" | wc -w)
+          if [ "$containerCnt" -lt "$pipelineIdx" ]
+          then
+            echo "pipelineIdx=$pipelineIdx, containerCnt=$containerCnt"
+            echo "waiting for the previous pipeline container to start up..."
+            sleep 1
+          else
+            break
+          fi
+        done
+      fi
       echo "Starting pipeline$pipelineIdx"
       if [ "$CPU_ONLY" != 1 ] && ([ "$HAS_FLEX_140" == 1 ] || [ "$HAS_FLEX_170" == 1 ])
       then
@@ -239,13 +248,13 @@ do
                 break
               fi
             done
-            LOW_POWER=$LOW_POWER DEVICE=$DEVICE ./docker-run.sh "$@"
+            LOW_POWER=$LOW_POWER DEVICE=$DEVICE ./run.sh "$@"
           else
             echo "Error: NUM_GPU is 0, cannot run"
             exit 1
           fi
       else
-          CPU_ONLY=$CPU_ONLY LOW_POWER=$LOW_POWER DEVICE=$DEVICE ./docker-run.sh "$@"
+          CPU_ONLY=$CPU_ONLY LOW_POWER=$LOW_POWER DEVICE=$DEVICE ./run.sh "$@"
       fi
       sleep 1
       #popd
@@ -270,7 +279,7 @@ do
       # Sync sleep in stream density script and platform metrics data collection script
       CPU_ONLY=$CPU_ONLY LOW_POWER=$LOW_POWER COMPLETE_INIT_DURATION=$COMPLETE_INIT_DURATION \
       STREAM_DENSITY_FPS=$STREAM_DENSITY_FPS STREAM_DENSITY_INCREMENTS=$STREAM_DENSITY_INCREMENTS \
-      STREAM_DENSITY_MODE=1 DEVICE=$DEVICE ./docker-run.sh "$@"
+      STREAM_DENSITY_MODE=1 DEVICE=$DEVICE ./run.sh "$@"
       #popd
     fi
   done
@@ -300,61 +309,27 @@ do
   else
         echo "Waiting for workload(s) to finish..."
         waitingMsg=1
-        ovmsCase=0
-        # figure out which case we are running like either "ovms-server" or "automated-self-checkout" container
-        mapfile -t sids < <(docker ps  -f name=automated-self-checkout -f status=running -q -a)
-        stream_workload_running=$(echo "${sids[@]}" | wc -w)
-        if (( $(echo "$stream_workload_running" 0 | awk '{if ($1 == $2) print 1;}') ))
-        then
-          # if we don't find any docker container running for dlstreamer (i.e. name with automated-self-checkout)
-          # then it is ovms running case
-          echo "running ovms client case..."
-          ovmsCase=1
-        else
-          echo "running dlstreamer case..."
-        fi
 
         # keep looping through until stream density script is done
         while true
         do
-          if [ $ovmsCase -eq 1 ]
+          # stream density is running from profile-launcer so we check that executing process
+          stream_density_running=$(pgrep -fa "stream_density.sh")
+          if [ -z "$stream_density_running" ]
           then
-            stream_density_running=$(docker exec ovms-client0 bash -c 'ps -aux | grep "stream_density_framework-pipelines.sh" | grep -v grep')
-            if [ -z "$stream_density_running" ]
+            # when the stream-density is done, we should clean up the profile-launcer process
+            proifleLauncherPid=$(pgrep -f "profile-launcher")
+            if [ -n "$proifleLauncherPid" ]
             then
-              # when stream density script process is done, we need to kill the ovms-client0 container as it keeps running forever
-              echo "killing ovms-client0 docker container..."
-              docker rm ovms-client0 -f
-              break
-            else
-              if [ $waitingMsg -eq 1 ]
-              then
-                echo "stream density script is still running..."
-                waitingMsg=0
-              fi
+              pkill -P "$proifleLauncherPid"
+              echo "profile-launcher is done"
             fi
+            break
           else
-            # since there is no longer --rm automatically remove docker-run containers
-            # we want to remove those first if any:
-            exitedIds=$(docker ps  -f name=automated-self-checkout -f status=exited -q -a)
-            if [ -n "$exitedIds" ]
+            if [ $waitingMsg -eq 1 ]
             then
-              docker rm "$exitedIds"
-            fi
-
-            mapfile -t sids < <(docker ps  --filter="name=automated-self-checkout" -q -a)
-            #echo "sids: " "${sids[@]}"
-            stream_workload_running=$(echo "${sids[@]}" | wc -w)
-            #echo "stream workload_running: $stream_workload_running"
-            if (( $(echo "$stream_workload_running" 0 | awk '{if ($1 == $2) print 1;}') ))
-            then
-                  break
-            else
-              if [ $waitingMsg -eq 1 ]
-              then
-                echo "stream density script is still running..."
-                waitingMsg=0
-              fi
+              echo "stream density script is still running..."
+              waitingMsg=0
             fi
           fi
           # there are still some pipeline running containers, waiting for them to be finished...

benchmark-scripts/camera-simulator.sh

@@ -21,6 +21,8 @@
     if [ -z "$CAMERAS" ]; then
 	CAMERAS=${#FILES[@]}
     fi
+
+    cd $SOURCE_DIR/camera-simulator
 
     docker run --rm -t --network=host --name camera-simulator aler9/rtsp-simple-server >rtsp_simple_server.log.txt  2>&1 &
     index=0

benchmark-scripts/collect_platform_metrics.sh

@@ -114,7 +114,7 @@ then
 else
   if [ "$is_xeon"  == "1"  ]
   then
-    # this script is actually called by ./benchmark.sh shell script through the docker container benchmark:dev, so we don't need another docker-run here any more
+    # this script is actually called by ./benchmark.sh shell script through the docker container benchmark:dev, so we don't need another docker run here any more
     timeout "$DURATION" "$PCM_DIRECTORY"/pcm-memory 1 -silent -nc -csv="$LOG_DIRECTORY"/memory_bandwidth.csv &
   fi 
 fi

benchmark-scripts/download_sample_videos.sh

@@ -9,4 +9,5 @@
 ./format_avc_mp4.sh coca-cola-4465029.mp4 https://www.pexels.com/video/4465029/download/ "$1" "$2" "$3"
 ./format_avc_mp4.sh vehicle-bike.mp4 https://www.pexels.com/video/853908/download/ "$1" "$2" "$3"
 #./format_avc_mp4.sh grocery-items-on-the-kitchen-shelf-4983686.mp4 https://www.pexels.com/video/4983686/download/ $1 $2 $3
-#./format_avc_mp4.sh couple-paying-at-the-counter-in-the-grocery-4121754.mp4 https://www.pexels.com/video/4121754/download/
+./format_avc_mp4.sh video_of_people_walking_855564.mp4 https://www.pexels.com/video/855564/download/ "$1" "$2" "$3"
+./format_avc_mp4.sh barcode.mp4 https://github.com/antoniomtz/sample-clips/raw/main/barcode.mp4 "$1" "$2" "$3"

benchmark-scripts/format_avc_mp4.sh

@@ -16,8 +16,8 @@ show_help() {
         "
 }
 
-WIDTH=3840
-HEIGHT=2160
+WIDTH=1920
+HEIGHT=1080
 FPS=15
 
 if [ -z "$2" ]
@@ -66,19 +66,6 @@ then
 	exit 0
 fi
 
-FIND_IMAGE_SOC=$(docker images | grep "sco-soc")
-FIND_IMAGE_DGPU=$(docker images | grep "sco-dgpu")
-if [ -z "$FIND_IMAGE_SOC" ] && [ -z "$FIND_IMAGE_DGPU" ]
-then
-	echo "ERROR: Can not find docker image sco-soc or sco-dgpu, please build image first!"
-	exit 1
-elif [ ! -z "$FIND_IMAGE_DGPU" ]
-then
-	TAG=sco-dgpu:2.0
-else
-	TAG=sco-soc:2.0
-fi 
-
 if [ ! -f ../sample-media/$1 ] && [ ! -f ../sample-media/$result ]
 then	
 	wget -O ../sample-media/$1 $2
@@ -98,7 +85,7 @@ SAMPLE_MEDIA_DIR="$PWD"/../sample-media
 docker run --network host --privileged --user root --ipc=host -e VIDEO_FILE="$1" -e DISPLAY=:0 \
 	-v /tmp/.X11-unix:/tmp/.X11-unix \
 	-v "$SAMPLE_MEDIA_DIR"/:/vids \
-	-w /vids -it  --rm "$TAG" \
+	-w /vids -it  --rm intel/dlstreamer:2023.0.0-ubuntu22-gpu682-dpcpp \
 	bash -c "if [ -f /vids/$result ]; then exit 1; else gst-launch-1.0 filesrc location=/vids/$1 ! qtdemux ! h264parse ! vaapih264dec ! vaapipostproc width=$WIDTH height=$HEIGHT ! videorate ! 'video/x-raw, framerate=$FPS/1' ! vaapih264enc ! h264parse ! mp4mux ! filesink location=/vids/$result; fi"
 
 rm ../sample-media/"$1"

benchmark-scripts/requirements-xpu.txt

@@ -1,8 +1,8 @@
-Flask==2.3.3
+Flask==3.0.0
 Flask-HTTPAuth==4.8.0
 requests==2.31.0
-prometheus-client==0.17.1
-grpcio==1.58.0
-protobuf==4.24.3
+prometheus-client==0.19.0
+grpcio==1.60.2
+protobuf==4.25.1
 marshmallow==3.20.1
 gunicorn[gthread]==21.2.0

benchmark-scripts/requirements.txt

@@ -1,3 +1,3 @@
 natsort==8.4.0
-numpy==1.26.0
-pandas==2.1.1
+numpy==1.26.2
+pandas==2.1.4

benchmark-scripts/smoke_test_benchmark.sh

@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+#
+# Copyright (C) 2023 Intel Corporation.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+# initial setup
+(
+    cd ..
+    make clean-all
+    sleep 3
+    make build-dlstreamer
+)
+
+# build benchmark Docker images:
+make
+
+# Download media
+./download_sample_videos.sh
+
+PLATFORM=$1
+CPU_ONLY=$2
+
+if [ -z "$PLATFORM" ]
+then
+    PLATFORM="core"
+fi
+
+if [ -z "$CPU_ONLY" ]
+then
+    CPU_ONLY=0
+fi
+
+sudo rm -rf results || true
+sudo rm -rf platform_"$PLATFORM"_cpuonly_"$CPU_ONLY"_smoke_test_camera_simulator_gst/  || true
+sudo rm -rf platform_"$PLATFORM"_cpuonly_"$CPU_ONLY"_smoke_test_camera_simulator_capi_yolov5_ensemble/  || true
+sudo rm -rf platform_"$PLATFORM"_cpuonly_"$CPU_ONLY"_smoke_test_camera_simulator_object_detection/  || true
+sudo rm -rf platform_"$PLATFORM"_cpuonly_"$CPU_ONLY"_smoke_test_stream_density/ || true
+
+# Note: all of benchmarking pipelines are run with RENDER_MODE=0 for better performance without spending extra resources for rendering
+# Camera simulator full pipeline
+# shell check has false postive on SC2097 and SC2098 as it doesn't detect -E flag which is inhereted the envs into subprocess; hence disable here
+# shellcheck disable=SC2097,SC2098
+PIPELINE_PROFILE="gst" CPU_ONLY="$CPU_ONLY" RENDER_MODE=0 sudo -E ./benchmark.sh --pipelines 1 --logdir platform_"$PLATFORM"_cpuonly_"$CPU_ONLY"_smoke_test_camera_simulator_gst/data --init_duration 30 --duration 60 --platform "$PLATFORM" --inputsrc rtsp://127.0.0.1:8554/camera_0
+# consolidate results
+make consolidate ROOT_DIRECTORY=platform_"$PLATFORM"_cpuonly_"$CPU_ONLY"_smoke_te_smoke_test_camera_simulator_gst
+
+# Camera simulator for capi_yolov5_ensemble
+# shell check has false postive on SC2097 and SC2098 as it doesn't detect -E flag which is inhereted the envs into subprocess; hence disable here
+# shellcheck disable=SC2097,SC2098
+PIPELINE_PROFILE="capi_yolov5_ensemble" CPU_ONLY=$CPU_ONLY RENDER_MODE=0 sudo -E ./benchmark.sh --pipelines 1 --logdir platform_"$PLATFORM"_cpuonly_"$CPU_ONLY"_smoke_test_camera_simulator_capi_yolov5_ensemble/data --init_duration 30 --duration 60 --platform "$PLATFORM" --inputsrc rtsp://127.0.0.1:8554/camera_0
+# consolidate results
+make consolidate ROOT_DIRECTORY=platform_"$PLATFORM"_cpuonly_"$CPU_ONLY"_smoke_test_camera_simulator_capi_yolov5_ensemble
+
+# Camera simulator yolov5 only
+# shell check has false postive on SC2097 and SC2098 as it doesn't detect -E flag which is inhereted the envs into subprocess; hence disable here
+# shellcheck disable=SC2097,SC2098
+PIPELINE_PROFILE="object_detection" CPU_ONLY=$CPU_ONLY RENDER_MODE=0 sudo -E ./benchmark.sh --pipelines 1 --logdir platform_"$PLATFORM"_cpuonly_"$CPU_ONLY"_smoke_test_camera_simulator_object_detection/data --init_duration 30 --duration 60 --platform "$PLATFORM" --inputsrc rtsp://127.0.0.1:8554/camera_0
+# consolidate results
+make consolidate ROOT_DIRECTORY=platform_"$PLATFORM"_cpuonly_"$CPU_ONLY"_smoke_test_camera_simulator_object_detection
+
+# Stream density for object detection
+# shell check has false postive on SC2097 and SC2098 as it doesn't detect -E flag which is inhereted the envs into subprocess; hence disable here
+# shellcheck disable=SC2097,SC2098
+PIPELINE_PROFILE="object_detection" CPU_ONLY=$CPU_ONLY RENDER_MODE=0 sudo -E ./benchmark.sh --stream_density 60 --logdir platform_"$PLATFORM"_cpuonly_"$CPU_ONLY"_smoke_test_stream_density/data --init_duration 30 --duration 60 --platform "$PLATFORM" --inputsrc rtsp://127.0.0.1:8554/camera_0