Merge pull request #14 from antoniomtz/business-logic

feat: Business logic with ROI entering/leaving

.gitignore

@@ -1 +1,2 @@
-models/
+models/
+__pycache__

Makefile

@@ -22,7 +22,7 @@ download-yolov8s:
 		docker run --user 1000:1000 -e HTTPS_PROXY=${HTTPS_PROXY} -e HTTP_PROXY=${HTTPS_PROXY} --rm \
 			-e YOLO_DEBUG=1 \
 			-v $(PWD)/models/object_detection/yolov8s:/models \
-			ultralytics/ultralytics:latest-cpu \
+			ultralytics/ultralytics:8.2.101-cpu \
 			bash -c "cd /models && yolo export model=yolov8s.pt format=openvino"; \
 		mv $(PWD)/models/object_detection/yolov8s/yolov8s_openvino_model $(PWD)/models/object_detection/yolov8s/FP32; \
 	else \
@@ -48,6 +48,7 @@ update-submodules:
 
 build:
 	docker build --build-arg HTTPS_PROXY=${HTTPS_PROXY} --build-arg HTTP_PROXY=${HTTP_PROXY} --target build-default -t dlstreamer:dev -f src/Dockerfile src/
+	docker build --build-arg HTTPS_PROXY=${HTTPS_PROXY} --build-arg HTTP_PROXY=${HTTP_PROXY} -t loss-prevention:dev -f src/app/Dockerfile src/app
 
 build-realsense:
 	docker build --build-arg HTTPS_PROXY=${HTTPS_PROXY} --build-arg HTTP_PROXY=${HTTP_PROXY} --target build-realsense -t dlstreamer:realsense -f src/Dockerfile src/

src/app/Dockerfile

@@ -0,0 +1,12 @@
+#
+# Copyright (C) 2024 Intel Corporation.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+FROM python:3.9-slim
+ENV PYTHONUNBUFFERED=1
+WORKDIR /app
+COPY . .
+RUN pip install paho-mqtt==1.6.1
+CMD ["python", "loss_prevention.py"]

src/app/loss_prevention.py

@@ -0,0 +1,55 @@
+'''
+* Copyright (C) 2024 Intel Corporation.
+*
+* SPDX-License-Identifier: Apache-2.0
+'''
+
+import os
+import json
+import paho.mqtt.client as mqtt
+
+MQTT_BROKER_URL = os.getenv("MQTT_URL", "127.0.0.1")
+MQTT_PORT = int(os.getenv("MQTT_PORT", "1883"))
+MQTT_TOPIC = os.getenv("MQTT_TOPIC", "event/detection")
+ROI_NAME = os.getenv("ROI_NAME", "BASKET")
+
+current_objects_in_roi = {}
+
+def on_message(client, userdata, message):
+    global current_objects_in_roi
+
+    frame_data = json.loads(message.payload.decode("utf-8"))
+    objects_in_current_frame = {}
+
+    if "objects" in frame_data:
+        for obj in frame_data["objects"]:
+            if obj.get("roi_type") == ROI_NAME and obj["detection"].get("label") == ROI_NAME:
+                continue
+
+            obj_id = obj["id"]
+            label = obj["detection"]["label"]
+            objects_in_current_frame[obj_id] = label
+
+    entered_objects = {obj_id: label for obj_id, label in objects_in_current_frame.items() if obj_id not in current_objects_in_roi}
+    left_objects = {obj_id: label for obj_id, label in current_objects_in_roi.items() if obj_id not in objects_in_current_frame}
+
+    for obj_id, label in entered_objects.items():
+        print(f"Object {label} (ID: {obj_id}) entered the {ROI_NAME} ROI.")
+
+    for obj_id, label in left_objects.items():
+        print(f"Object {label} (ID: {obj_id}) left the {ROI_NAME} ROI.")
+
+    current_objects_in_roi = objects_in_current_frame
+
+    print(f" Current objects in ROI: {current_objects_in_roi}")
+
+def on_connect(client, userdata, flags, rc):
+    print(f"Connected with result code {rc}")
+    client.subscribe(MQTT_TOPIC)
+
+client = mqtt.Client(client_id="", clean_session=True, userdata=None, protocol=mqtt.MQTTv311, transport="tcp")
+client.on_connect = on_connect
+client.on_message = on_message
+
+client.connect(MQTT_BROKER_URL, MQTT_PORT, 60)
+client.loop_forever()

src/docker-compose.yml

@@ -59,3 +59,24 @@ services:
       - ${RETAIL_USE_CASE_ROOT:-..}/models:/home/pipeline-server/models
       - ./pipelines/:/home/pipeline-server/pipelines
       - ./extensions/:/home/pipeline-server/extensions
+
+  mosquitto:
+    image: eclipse-mosquitto:2.0
+    container_name: mosquitto
+    network_mode: "host"
+    ports:
+      - "127.0.0.1:1883:1883"
+    depends_on:
+      - OvmsClientGst    
+
+  mqtt_tracker:
+    image: loss-prevention:dev
+    network_mode: "host"
+    environment:
+      - MQTT_URL=127.0.0.1
+      - MQTT_PORT=1883
+      - MQTT_TOPIC=event/detection
+      - ROI_NAME=BASKET
+    restart: always
+    depends_on:
+      - mosquitto

src/extensions/gva_roi_metadata.py

@@ -0,0 +1,52 @@
+'''
+* Copyright (C) 2024 Intel Corporation.
+*
+* SPDX-License-Identifier: Apache-2.0
+'''
+
+from gi.repository import Gst, GObject
+import sys
+import gi
+import json
+from copy import deepcopy
+gi.require_version('Gst', '1.0')
+
+
+class RoiMetadata:
+    def __init__(self, disable=False, roi=""):
+        self.roi = roi
+
+    def process_frame(self, frame):
+
+        if self.roi == "":
+            return True
+
+        for message in frame.messages():
+            message_obj = json.loads(message)
+
+            if "objects" in message_obj:
+                frame.remove_message(message)
+                message_objects_array = deepcopy(message_obj["objects"])
+
+                roi_objects = []
+                other_objects = []
+
+                for obj in message_objects_array:
+                    if "roi_type" in obj:
+                        if obj["roi_type"] == self.roi:
+                            roi_objects.append(obj)
+                        else:
+                            other_objects.append(obj)
+
+                for obj in other_objects:
+                    for objects in roi_objects:
+                        if (obj["x"] >= objects["x"] and obj["x"] + obj["w"] <= objects["x"] + objects["w"] and
+                                obj["y"] >= objects["y"] and obj["y"] + obj["h"] <= objects["y"] + objects["h"]):
+                            obj["roi_type"] = self.roi
+
+                message_obj["objects"] = other_objects + roi_objects
+
+                frame.add_message(json.dumps(
+                    message_obj, separators=(',', ':')))
+                break
+        return True

src/pipelines/roi.json

@@ -1,30 +1,15 @@
 [
-
     {
-
         "objects": [
-
             {
-
                 "detection": {
-
-                    "label": "ROI1"
-
+                    "label": "BASKET"
                 },
-
-                "x": 0,
-
-                "y": 0,
-
-                "w": 680,
-
-                "h": 1080
-
+                "x": 400,
+                "y": 80,
+                "w": 2000,
+                "h": 2000
             }
-
-
         ]
-
     }
-
 ]

src/pipelines/yolov8s_roi.sh

@@ -9,7 +9,9 @@ PRE_PROCESS="${PRE_PROCESS:=""}" #""|pre-process-backend=vaapi-surface-sharing|p
 AGGREGATE="${AGGREGATE:="gvametaaggregate name=aggregate !"}" # Aggregate function at the end of the pipeline ex. "" | gvametaaggregate name=aggregate
 PUBLISH="${PUBLISH:="name=destination file-format=2 file-path=/tmp/results/r$cid\"_gst\".jsonl"}" # address=localhost:1883 topic=inferenceEvent method=mqtt
 
-CLASS_IDS="0" # YOLOv8 classes to be detected example "0,1,30"
+CLASS_IDS="46,39,47" # YOLOv8 classes to be detected example "0,1,30"
+MQTT_HOST="127.0.0.1:1883"
+ROI="BASKET"
 
 if [ "$RENDER_MODE" == "1" ]; then
     OUTPUT="${OUTPUT:="! videoconvert ! gvawatermark ! videoconvert ! fpsdisplaysink video-sink=ximagesink sync=true --verbose"}"
@@ -18,9 +20,16 @@ else
 fi
 
 echo "decode type $DECODE"
-echo "Run yolov8 pipeline with ROI on $DEVICE with batch size = $BATCH_SIZE"
+echo "Run YOLOv8 pipeline with ROI and Filtering on $DEVICE with batch size = $BATCH_SIZE"
+
+gstLaunchCmd="GST_DEBUG=1 GST_TRACERS=\"latency_tracer(flags=pipeline,interval=100)\" gst-launch-1.0 $inputsrc ! $DECODE ! gvaattachroi mode=1 file-path=/home/pipeline-server/pipelines/roi.json ! gvadetect batch-size=$BATCH_SIZE model-instance-id=odmodel name=detection model=models/object_detection/yolov8s/FP32/yolov8s.xml device=$DEVICE $PRE_PROCESS inference-region=1 object-class=$ROI threshold=0.5 ! \
+gvapython module=/home/pipeline-server/extensions/object_filter.py class=ObjectDetectionFilter kwarg=\"{\\\"class_ids\\\": \\\"$CLASS_IDS\\\", \\\"rois\\\": \\\"$ROI\\\"}\" !  gvatrack ! \
+$AGGREGATE gvametaconvert name=metaconvert add-empty-results=true ! \
+gvapython module=/home/pipeline-server/extensions/gva_roi_metadata.py class=RoiMetadata kwarg=\"{\\\"roi\\\": \\\"$ROI\\\"}\" ! \
+gvametapublish method=mqtt file-format=2 address="$MQTT_HOST" mqtt-client-id=yolov8 topic=event/detection ! \
+queue ! \
+gvametapublish name=destination file-format=2 file-path=/tmp/results/r$cid\"_gst\".jsonl $OUTPUT 2>&1 | tee >/tmp/results/gst-launch_$cid\"_gst\".log >(stdbuf -oL sed -n -e 's/^.*current: //p' | stdbuf -oL cut -d , -f 1 > /tmp/results/pipeline$cid\"_gst\".log)"
 
-gstLaunchCmd="GST_DEBUG=\"GST_TRACER:7\" GST_TRACERS=\"latency_tracer(flags=pipeline,interval=100)\" gst-launch-1.0 $inputsrc ! $DECODE ! gvaattachroi mode=1 file-path=/home/pipeline-server/pipelines/roi.json ! gvadetect batch-size=$BATCH_SIZE model-instance-id=odmodel name=detection model=models/object_detection/yolov8s/FP32/yolov8s.xml device=$DEVICE $PRE_PROCESS inference-region=1 object-class=ROI1 threshold=0.5 ! gvapython module=/home/pipeline-server/extensions/object_filter.py class=ObjectDetectionFilter kwarg=\"{\\\"class_ids\\\": \\\"$CLASS_IDS\\\", \\\"rois\\\": \\\"ROI1\\\"}\" !  gvatrack ! $AGGREGATE gvametaconvert name=metaconvert add-empty-results=true ! gvametapublish name=destination file-format=2 file-path=/tmp/results/r$cid\"_gst\".jsonl $OUTPUT 2>&1 | tee >/tmp/results/gst-launch_$cid\"_gst\".log >(stdbuf -oL sed -n -e 's/^.*current: //p' | stdbuf -oL cut -d , -f 1 > /tmp/results/pipeline$cid\"_gst\".log)"
 
 echo "$gstLaunchCmd"