testing an improved low-light mode

README.md

@@ -1,15 +1,17 @@
-# Rein
+# [Rein]carnation
 
-A experimental vtuber renderer using super-easy-to-make svg models.
+A experimental vtuber renderer/app using super-easy-to-make[^1] svg models.
+
+[^1]: Rein's models are zero-config zero-rigging, toggles and keyframes can be
+optionally added to improve models.
 
 ### Roadmap
 
-This is a (rein)carnation of a dead project I made in 2022, the plan is to
-update it to use the ARkit standard 52 blendshapes instead of the current face
-landmarks method, and support all those blendshapes with toggles in the model
-parser.
+This is a reincarnation of a dead project I started in 2022.\
+The plan is to update and improve on it, create a friendly GUI, and make it
+available for everyone.
 
-![demo image showing a man with his eyes and mouth wide open and a vtuber avatar mimicking that facial structure](tests/snapshots/basic_model/test_eyes_mouth.jpeg)
+![An image showing a human with their eyes and mouth wide open and a cartoony-looking avatar mimicking their facial expression](tests/snapshots/basic_model/test_eyes_mouth.jpeg)
 
 #### How to use
 

preview_image.py

@@ -14,7 +14,8 @@
     tracking.set_model(model)
 
     frame = cv2.imread("tests/images/test_normal.jpeg")
-    frame = cv2.imread("tests/images/test_irises.jpeg")
+    # frame = cv2.imread("tests/images/test_closed_eyes.jpeg")
+    # frame = cv2.imread("tests/images/test_irises.jpeg")
     # frame = cv2.imread("tests/images/test_eyes_mouth.jpeg")
     # frame = cv2.imread("tests/images/test_head_tilted_2.jpeg")
     # frame = cv2.imread("tests/images/test_lowlight.jpeg")

src/tracking/tracking.py

@@ -158,7 +158,7 @@ def _get(indices):
 
     def __get_mouth_size__(self):
         w = 1.0 - self.__face_blendshape["mouthPucker"]
-        h = self.__face_blendshape["jawOpen"] / 2
+        h = self.__face_blendshape["jawOpen"] * 0.5
         return self.__filter__("mouth-0", w), self.__filter__("mouth-1", h)
 
     def __get_iris_diff__(self):

src/tracking/utils.py

@@ -1,15 +1,39 @@
 import cv2
+import numpy as np
 
 
-def optimize_image(image, width=480, brightness=50):
+def optimize_image(image):
+    width = 480
+
     if image.shape[1] == width:
         return image
 
     aspect_ratio = float(image.shape[0]) / float(image.shape[1])
 
     height = round(width * aspect_ratio)
 
-    resized_image = cv2.resize(image, (width, height))
-    brightened_image = cv2.convertScaleAbs(resized_image, alpha=1, beta=brightness)
+    image = cv2.resize(image, (width, height))
+
+    # grayscale
+    gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
+
+    # calculate the histogram of the grayscale image
+    hist = cv2.calcHist([gray], [0], None, [256], [0, 256])
+
+    cv2.normalize(hist, hist, alpha=0, beta=255, norm_type=cv2.NORM_MINMAX)
+
+    # determine if the image is likely to be underexposed
+    low_light_threshold = 15
+
+    total_pixels = np.sum(hist)
+
+    dark_pixels_sum = np.sum(hist[:low_light_threshold]) / total_pixels
+
+    # if image is low-light
+    # use a brighten grayscale version
+    if dark_pixels_sum > 0.2:
+        image = cv2.equalizeHist(gray)
+        image = cv2.cvtColor(image, cv2.COLOR_GRAY2BGR)
+        image = cv2.convertScaleAbs(image, alpha=1, beta=50)
 
-    return brightened_image
+    return image

tests/test_apply.py

@@ -8,6 +8,7 @@
 
 from src.svg.model import Model
 from src.tracking.tracking import Tracking
+from src.tracking.utils import optimize_image
 from tests.utils import compare_img, overlay_webcam
 
 with open("tests/models/face.svg") as file:
@@ -34,7 +35,7 @@ def test_basic_model(file):
     image_bytes = resvg_python.svg_to_png(model.tostring())
     image_decoded = cv2.imdecode(np.array(bytearray(image_bytes)), cv2.IMREAD_COLOR)
 
-    final_image = overlay_webcam(frame, image_decoded)
+    final_image = overlay_webcam(optimize_image(frame), image_decoded)
 
     assert compare_img(os.path.basename(file), "basic_model", final_image) < 4.5
 
@@ -56,6 +57,6 @@ def test_normal_model(file):
     image_bytes = resvg_python.svg_to_png(model.tostring())
     image_decoded = cv2.imdecode(np.array(bytearray(image_bytes)), cv2.IMREAD_COLOR)
 
-    final_image = overlay_webcam(frame, image_decoded)
+    final_image = overlay_webcam(optimize_image(frame), image_decoded)
 
     assert compare_img(os.path.basename(file), "normal_model", final_image) < 4.5