expose dynamic lighting, improve camera properties

include/sapien/component/sapien_renderer/camera_component.h

@@ -5,6 +5,7 @@
 #include "sapien/math/pose.h"
 #include "sapien/serialize.h"
 #include <svulkan2/scene/camera.h>
+#include <variant>
 
 namespace sapien {
 class Entity;
@@ -101,6 +102,8 @@ class SapienRenderCameraComponent : public Component {
   float mSkew{};
   std::string mShaderDir;
 
+  std::map<std::string, std::variant<int, float>> mProperties;
+
   std::unique_ptr<SapienRenderCameraInternal> mCamera;
   Pose mLocalPose;
 };

include/sapien/component/sapien_renderer/light_component.h

@@ -13,7 +13,7 @@ namespace component {
 class SapienRenderLightComponent : public Component {
 public:
   Vec3 getColor() const { return mColor; }
-  void setColor(Vec3 color) { mColor = color; }
+  virtual void setColor(Vec3 color);
 
   bool getShadowEnabled() const { return mShadowEnabled; }
   void setShadowEnabled(bool enabled) { mShadowEnabled = enabled; }
@@ -59,6 +59,7 @@ class SapienRenderPointLightComponent : public SapienRenderLightComponent {
   void onRemoveFromScene(Scene &scene) override;
 
   void internalUpdate() override;
+  void setColor(Vec3 color) override;
 
   template <class Archive> void save(Archive &ar) const {
     ar(cereal::base_class<SapienRenderLightComponent>(this));
@@ -80,6 +81,7 @@ class SapienRenderDirectionalLightComponent : public SapienRenderLightComponent
   void onRemoveFromScene(Scene &scene) override;
 
   void internalUpdate() override;
+  void setColor(Vec3 color) override;
 
   template <class Archive> void save(Archive &ar) const {
     ar(cereal::base_class<SapienRenderLightComponent>(this));
@@ -106,6 +108,7 @@ class SapienRenderSpotLightComponent : public SapienRenderLightComponent {
   void onRemoveFromScene(Scene &scene) override;
 
   void internalUpdate() override;
+  void setColor(Vec3 color) override;
 
   template <class Archive> void save(Archive &ar) const {
     ar(cereal::base_class<SapienRenderLightComponent>(this));
@@ -160,6 +163,7 @@ class SapienRenderParallelogramLightComponent : public SapienRenderLightComponen
   float getAngle() const { return mAngle; }
 
   void internalUpdate() override;
+  void setColor(Vec3 color) override;
 
   template <class Archive> void save(Archive &ar) const {
     ar(cereal::base_class<SapienRenderLightComponent>(this));

manualtest/stereodepth.py

@@ -103,13 +103,17 @@ def build_scene(render_system, physx_system):
     box.name = "box"
     box.pose = Pose([0.05, 0.26797, 0.09], [1, 0, 0, 0])
 
-    # scene.set_ambient_light([0.3, 0.3, 0.3])
-    # scene.add_directional_light([0, 0.5, -1], color=[3.0, 3.0, 3.0])
+    scene.set_ambient_light([0.3, 0.3, 0.3])
+    scene.add_directional_light([0, 0.5, -1], color=[0.5, 0.5, 0.5])
 
     return scene
 
 
 def main():
+    sapien.render.set_camera_shader_dir("rt")
+    sapien.render.set_ray_tracing_denoiser("optix")
+    sapien.render.set_ray_tracing_samples_per_pixel(4)
+
     render_system = RenderSystem()
     physx_system = PhysxSystem()
     scene = build_scene(render_system, physx_system)
@@ -121,17 +125,6 @@ def main():
     # Test infrared light
     from sapien.render import RenderTexturedLightComponent, RenderTexture2D
 
-    alight = RenderTexturedLightComponent()
-    alight.color = [1, 0, 0]
-    alight.inner_fov = 1.57
-    alight.outer_fov = 1.57
-    alight.texture = RenderTexture2D(
-        "../python/py_package/sensor/assets/patterns/d415.png"
-    )
-    alight.local_pose = Pose()
-    alight.name = "infrared_light"
-    sensor_entity.add_component(alight)
-
     scene.add_entity(sensor_entity)
     sensor_entity.set_pose(
         Pose(

python/py_package/sensor/stereodepth.py

@@ -22,8 +22,7 @@ def __init__(self):
         )
         """Path to active light pattern file. Use RGB modality if set to None."""
 
-        # TODO: implement camera exposure
-        # self.ir_camera_exposure = 0.01
+        self.ir_camera_exposure = 0.01
         # """Camera exposure for infrared cameras."""
 
         self.rgb_resolution = (1920, 1080)
@@ -148,8 +147,7 @@ def __init__(
 
         # Active light
         self._alight = None
-        # TODO: uncomment this after real time color adjustment implemented
-        # self._create_light()
+        self._create_light()
 
         # Simsense component
         self._ss = SimSenseComponent(
@@ -175,7 +173,7 @@ def __init__(
             config.uniqueness_ratio,
             config.lr_max_diff,
             config.median_filter_size,
-            config.depth_dilation
+            config.depth_dilation,
         )
         self._mount.add_component(self._ss)
 
@@ -407,8 +405,6 @@ def _create_cameras(self):
             self._config.ir_intrinsic[1, 2],
             self._config.ir_intrinsic[0, 1],
         )
-        # TODO: uncomment after exposure is implemented
-        # self._cam_ir_l.set_property('exposure', self._config.ir_camera_exposure)
 
         self._cam_ir_r = RenderCameraComponent(*self._config.ir_resolution)
         self._cam_ir_r.local_pose = self._pose * self._config.trans_pose_r
@@ -422,13 +418,14 @@ def _create_cameras(self):
             self._config.ir_intrinsic[1, 2],
             self._config.ir_intrinsic[0, 1],
         )
-        # TODO: uncomment after exposure is implemented
-        # self._cam_ir_r.set_property('exposure', self._config.ir_camera_exposure)
 
         self._mount.add_component(self._cam_rgb)
         self._mount.add_component(self._cam_ir_l)
         self._mount.add_component(self._cam_ir_r)
 
+        self._cam_ir_l.set_property("exposure", float(self._config.ir_camera_exposure))
+        self._cam_ir_r.set_property("exposure", float(self._config.ir_camera_exposure))
+
     def _create_light(self):
         # Active Light
         self._alight = RenderTexturedLightComponent()
@@ -443,15 +440,9 @@ def _create_light(self):
     def _ir_mode(self):
         if self._config.light_pattern is None:
             return
-        else:
-            # TODO: real time color adjustment
-            # self._alight.color = [100, 0, 0]
-            return
+        self._alight.color = [100, 0, 0]
 
     def _normal_mode(self):
         if self._config.light_pattern is None:
             return
-        else:
-            # TODO: real time color adjustment
-            # self._alight.color = [0, 0, 0]
-            return
+        self._alight.color = [0, 0, 0]

src/component/sapien_renderer/camera_component.cpp

@@ -147,16 +147,16 @@ struct SapienRenderCameraInternal {
 };
 
 void SapienRenderCameraComponent::setProperty(std::string const &name, int property) {
-  if (!mCamera) {
-    throw std::runtime_error("camera property is only available for cameras added to scene");
+  mProperties[name] = property;
+  if (mCamera) {
+    mCamera->mRenderer->setCustomProperty(name, property);
   }
-  mCamera->mRenderer->setCustomProperty(name, property);
 }
 void SapienRenderCameraComponent::setProperty(std::string const &name, float property) {
-  if (!mCamera) {
-    throw std::runtime_error("camera property is only available for cameras added to scene");
+  mProperties[name] = property;
+  if (mCamera) {
+    mCamera->mRenderer->setCustomProperty(name, property);
   }
-  mCamera->mRenderer->setCustomProperty(name, property);
 }
 void SapienRenderCameraComponent::setTexture(std::string const &name,
                                              std::shared_ptr<SapienRenderTexture> texture) {
@@ -192,11 +192,17 @@ SapienRenderCameraComponent::SapienRenderCameraComponent(uint32_t width, uint32_
 
 void SapienRenderCameraComponent::onAddToScene(Scene &scene) {
   auto system = scene.getSapienRendererSystem();
-  // TODO: fix shader dir
   mCamera = std::make_unique<SapienRenderCameraInternal>(getWidth(), getHeight(), mShaderDir,
                                                          system->getScene());
   mCamera->mCamera->setPerspectiveParameters(mNear, mFar, mFx, mFy, mCx, mCy, mWidth, mHeight,
                                              mSkew);
+  for (auto &[k, v] : mProperties) {
+    if (std::holds_alternative<int>(v)) {
+      mCamera->mRenderer->setCustomProperty(k, std::get<int>(v));
+    } else {
+      mCamera->mRenderer->setCustomProperty(k, std::get<float>(v));
+    }
+  }
   system->registerComponent(
       std::static_pointer_cast<SapienRenderCameraComponent>(shared_from_this()));
 }

src/component/sapien_renderer/light_component.cpp

@@ -146,5 +146,31 @@ void SapienRenderLightComponent::setLocalPose(Pose const &pose) { mLocalPose = p
 Pose SapienRenderLightComponent::getLocalPose() const { return mLocalPose; }
 Pose SapienRenderLightComponent::getGlobalPose() const { return getPose() * mLocalPose; }
 
+void SapienRenderLightComponent::setColor(Vec3 color) { mColor = color; }
+void SapienRenderPointLightComponent::setColor(Vec3 color) {
+  mColor = color;
+  if (mPointLight) {
+    mPointLight->setColor({color.x, color.y, color.z});
+  }
+}
+void SapienRenderDirectionalLightComponent::setColor(Vec3 color) {
+  mColor = color;
+  if (mDirectionalLight) {
+    mDirectionalLight->setColor({color.x, color.y, color.z});
+  }
+}
+void SapienRenderSpotLightComponent::setColor(Vec3 color) {
+  mColor = color;
+  if (mSpotLight) {
+    mSpotLight->setColor({color.x, color.y, color.z});
+  }
+}
+void SapienRenderParallelogramLightComponent::setColor(Vec3 color) {
+  mColor = color;
+  if (mParallelogramLight) {
+    mParallelogramLight->setColor({color.x, color.y, color.z});
+  }
+}
+
 } // namespace component
 } // namespace sapien

vulkan_shader/default/composite0.frag

@@ -1,5 +1,7 @@
 #version 450
 
+layout (constant_id = 0) const float exposure = 1.0;
+
 layout(set = 0, binding = 0) uniform sampler2D samplerLighting;
 layout(set = 0, binding = 1) uniform usampler2D samplerSegmentation;
 layout(set = 0, binding = 2) uniform sampler2D samplerPosition;
@@ -95,7 +97,7 @@ vec4 colors[60] = {
 
 void main() {
   outColor = texture(samplerLighting, inUV);
-  outColor = pow(outColor, vec4(1/2.2, 1/2.2, 1/2.2, 1));
+  outColor.rgb = pow(outColor.rgb * exposure, vec3(1/2.2));
   outColor = clamp(outColor, vec4(0), vec4(1));
 
   outDepthLinear.x = -texture(samplerPosition, inUV).z;