From 7b0b6c294acac8d9fa8679a13070251c834a6174 Mon Sep 17 00:00:00 2001 From: Fanbo Xiang Date: Thu, 2 Nov 2023 13:32:24 -0700 Subject: [PATCH] expose dynamic lighting, improve camera properties --- .../sapien_renderer/camera_component.h | 3 +++ .../sapien_renderer/light_component.h | 6 ++++- manualtest/stereodepth.py | 19 +++++--------- python/py_package/sensor/stereodepth.py | 25 ++++++------------ .../sapien_renderer/camera_component.cpp | 20 +++++++++----- .../sapien_renderer/light_component.cpp | 26 +++++++++++++++++++ vulkan_shader/default/composite0.frag | 4 ++- 7 files changed, 64 insertions(+), 39 deletions(-) diff --git a/include/sapien/component/sapien_renderer/camera_component.h b/include/sapien/component/sapien_renderer/camera_component.h index 1a4e6c78..10e5ae5e 100644 --- a/include/sapien/component/sapien_renderer/camera_component.h +++ b/include/sapien/component/sapien_renderer/camera_component.h @@ -5,6 +5,7 @@ #include "sapien/math/pose.h" #include "sapien/serialize.h" #include +#include namespace sapien { class Entity; @@ -101,6 +102,8 @@ class SapienRenderCameraComponent : public Component { float mSkew{}; std::string mShaderDir; + std::map> mProperties; + std::unique_ptr mCamera; Pose mLocalPose; }; diff --git a/include/sapien/component/sapien_renderer/light_component.h b/include/sapien/component/sapien_renderer/light_component.h index e026e031..b89ba23f 100644 --- a/include/sapien/component/sapien_renderer/light_component.h +++ b/include/sapien/component/sapien_renderer/light_component.h @@ -13,7 +13,7 @@ namespace component { class SapienRenderLightComponent : public Component { public: Vec3 getColor() const { return mColor; } - void setColor(Vec3 color) { mColor = color; } + virtual void setColor(Vec3 color); bool getShadowEnabled() const { return mShadowEnabled; } void setShadowEnabled(bool enabled) { mShadowEnabled = enabled; } @@ -59,6 +59,7 @@ class SapienRenderPointLightComponent : public SapienRenderLightComponent { void onRemoveFromScene(Scene &scene) override; void internalUpdate() override; + void setColor(Vec3 color) override; template void save(Archive &ar) const { ar(cereal::base_class(this)); @@ -80,6 +81,7 @@ class SapienRenderDirectionalLightComponent : public SapienRenderLightComponent void onRemoveFromScene(Scene &scene) override; void internalUpdate() override; + void setColor(Vec3 color) override; template void save(Archive &ar) const { ar(cereal::base_class(this)); @@ -106,6 +108,7 @@ class SapienRenderSpotLightComponent : public SapienRenderLightComponent { void onRemoveFromScene(Scene &scene) override; void internalUpdate() override; + void setColor(Vec3 color) override; template void save(Archive &ar) const { ar(cereal::base_class(this)); @@ -160,6 +163,7 @@ class SapienRenderParallelogramLightComponent : public SapienRenderLightComponen float getAngle() const { return mAngle; } void internalUpdate() override; + void setColor(Vec3 color) override; template void save(Archive &ar) const { ar(cereal::base_class(this)); diff --git a/manualtest/stereodepth.py b/manualtest/stereodepth.py index 77f0347b..9fd7294a 100644 --- a/manualtest/stereodepth.py +++ b/manualtest/stereodepth.py @@ -103,13 +103,17 @@ def build_scene(render_system, physx_system): box.name = "box" box.pose = Pose([0.05, 0.26797, 0.09], [1, 0, 0, 0]) - # scene.set_ambient_light([0.3, 0.3, 0.3]) - # scene.add_directional_light([0, 0.5, -1], color=[3.0, 3.0, 3.0]) + scene.set_ambient_light([0.3, 0.3, 0.3]) + scene.add_directional_light([0, 0.5, -1], color=[0.5, 0.5, 0.5]) return scene def main(): + sapien.render.set_camera_shader_dir("rt") + sapien.render.set_ray_tracing_denoiser("optix") + sapien.render.set_ray_tracing_samples_per_pixel(4) + render_system = RenderSystem() physx_system = PhysxSystem() scene = build_scene(render_system, physx_system) @@ -121,17 +125,6 @@ def main(): # Test infrared light from sapien.render import RenderTexturedLightComponent, RenderTexture2D - alight = RenderTexturedLightComponent() - alight.color = [1, 0, 0] - alight.inner_fov = 1.57 - alight.outer_fov = 1.57 - alight.texture = RenderTexture2D( - "../python/py_package/sensor/assets/patterns/d415.png" - ) - alight.local_pose = Pose() - alight.name = "infrared_light" - sensor_entity.add_component(alight) - scene.add_entity(sensor_entity) sensor_entity.set_pose( Pose( diff --git a/python/py_package/sensor/stereodepth.py b/python/py_package/sensor/stereodepth.py index f8794aea..4448abd3 100644 --- a/python/py_package/sensor/stereodepth.py +++ b/python/py_package/sensor/stereodepth.py @@ -22,8 +22,7 @@ def __init__(self): ) """Path to active light pattern file. Use RGB modality if set to None.""" - # TODO: implement camera exposure - # self.ir_camera_exposure = 0.01 + self.ir_camera_exposure = 0.01 # """Camera exposure for infrared cameras.""" self.rgb_resolution = (1920, 1080) @@ -148,8 +147,7 @@ def __init__( # Active light self._alight = None - # TODO: uncomment this after real time color adjustment implemented - # self._create_light() + self._create_light() # Simsense component self._ss = SimSenseComponent( @@ -175,7 +173,7 @@ def __init__( config.uniqueness_ratio, config.lr_max_diff, config.median_filter_size, - config.depth_dilation + config.depth_dilation, ) self._mount.add_component(self._ss) @@ -407,8 +405,6 @@ def _create_cameras(self): self._config.ir_intrinsic[1, 2], self._config.ir_intrinsic[0, 1], ) - # TODO: uncomment after exposure is implemented - # self._cam_ir_l.set_property('exposure', self._config.ir_camera_exposure) self._cam_ir_r = RenderCameraComponent(*self._config.ir_resolution) self._cam_ir_r.local_pose = self._pose * self._config.trans_pose_r @@ -422,13 +418,14 @@ def _create_cameras(self): self._config.ir_intrinsic[1, 2], self._config.ir_intrinsic[0, 1], ) - # TODO: uncomment after exposure is implemented - # self._cam_ir_r.set_property('exposure', self._config.ir_camera_exposure) self._mount.add_component(self._cam_rgb) self._mount.add_component(self._cam_ir_l) self._mount.add_component(self._cam_ir_r) + self._cam_ir_l.set_property("exposure", float(self._config.ir_camera_exposure)) + self._cam_ir_r.set_property("exposure", float(self._config.ir_camera_exposure)) + def _create_light(self): # Active Light self._alight = RenderTexturedLightComponent() @@ -443,15 +440,9 @@ def _create_light(self): def _ir_mode(self): if self._config.light_pattern is None: return - else: - # TODO: real time color adjustment - # self._alight.color = [100, 0, 0] - return + self._alight.color = [100, 0, 0] def _normal_mode(self): if self._config.light_pattern is None: return - else: - # TODO: real time color adjustment - # self._alight.color = [0, 0, 0] - return + self._alight.color = [0, 0, 0] diff --git a/src/component/sapien_renderer/camera_component.cpp b/src/component/sapien_renderer/camera_component.cpp index 009821e0..b8219f01 100644 --- a/src/component/sapien_renderer/camera_component.cpp +++ b/src/component/sapien_renderer/camera_component.cpp @@ -147,16 +147,16 @@ struct SapienRenderCameraInternal { }; void SapienRenderCameraComponent::setProperty(std::string const &name, int property) { - if (!mCamera) { - throw std::runtime_error("camera property is only available for cameras added to scene"); + mProperties[name] = property; + if (mCamera) { + mCamera->mRenderer->setCustomProperty(name, property); } - mCamera->mRenderer->setCustomProperty(name, property); } void SapienRenderCameraComponent::setProperty(std::string const &name, float property) { - if (!mCamera) { - throw std::runtime_error("camera property is only available for cameras added to scene"); + mProperties[name] = property; + if (mCamera) { + mCamera->mRenderer->setCustomProperty(name, property); } - mCamera->mRenderer->setCustomProperty(name, property); } void SapienRenderCameraComponent::setTexture(std::string const &name, std::shared_ptr texture) { @@ -192,11 +192,17 @@ SapienRenderCameraComponent::SapienRenderCameraComponent(uint32_t width, uint32_ void SapienRenderCameraComponent::onAddToScene(Scene &scene) { auto system = scene.getSapienRendererSystem(); - // TODO: fix shader dir mCamera = std::make_unique(getWidth(), getHeight(), mShaderDir, system->getScene()); mCamera->mCamera->setPerspectiveParameters(mNear, mFar, mFx, mFy, mCx, mCy, mWidth, mHeight, mSkew); + for (auto &[k, v] : mProperties) { + if (std::holds_alternative(v)) { + mCamera->mRenderer->setCustomProperty(k, std::get(v)); + } else { + mCamera->mRenderer->setCustomProperty(k, std::get(v)); + } + } system->registerComponent( std::static_pointer_cast(shared_from_this())); } diff --git a/src/component/sapien_renderer/light_component.cpp b/src/component/sapien_renderer/light_component.cpp index 8796aaa3..12b9556a 100644 --- a/src/component/sapien_renderer/light_component.cpp +++ b/src/component/sapien_renderer/light_component.cpp @@ -146,5 +146,31 @@ void SapienRenderLightComponent::setLocalPose(Pose const &pose) { mLocalPose = p Pose SapienRenderLightComponent::getLocalPose() const { return mLocalPose; } Pose SapienRenderLightComponent::getGlobalPose() const { return getPose() * mLocalPose; } +void SapienRenderLightComponent::setColor(Vec3 color) { mColor = color; } +void SapienRenderPointLightComponent::setColor(Vec3 color) { + mColor = color; + if (mPointLight) { + mPointLight->setColor({color.x, color.y, color.z}); + } +} +void SapienRenderDirectionalLightComponent::setColor(Vec3 color) { + mColor = color; + if (mDirectionalLight) { + mDirectionalLight->setColor({color.x, color.y, color.z}); + } +} +void SapienRenderSpotLightComponent::setColor(Vec3 color) { + mColor = color; + if (mSpotLight) { + mSpotLight->setColor({color.x, color.y, color.z}); + } +} +void SapienRenderParallelogramLightComponent::setColor(Vec3 color) { + mColor = color; + if (mParallelogramLight) { + mParallelogramLight->setColor({color.x, color.y, color.z}); + } +} + } // namespace component } // namespace sapien diff --git a/vulkan_shader/default/composite0.frag b/vulkan_shader/default/composite0.frag index aca07828..4dce9c87 100644 --- a/vulkan_shader/default/composite0.frag +++ b/vulkan_shader/default/composite0.frag @@ -1,5 +1,7 @@ #version 450 +layout (constant_id = 0) const float exposure = 1.0; + layout(set = 0, binding = 0) uniform sampler2D samplerLighting; layout(set = 0, binding = 1) uniform usampler2D samplerSegmentation; layout(set = 0, binding = 2) uniform sampler2D samplerPosition; @@ -95,7 +97,7 @@ vec4 colors[60] = { void main() { outColor = texture(samplerLighting, inUV); - outColor = pow(outColor, vec4(1/2.2, 1/2.2, 1/2.2, 1)); + outColor.rgb = pow(outColor.rgb * exposure, vec3(1/2.2)); outColor = clamp(outColor, vec4(0), vec4(1)); outDepthLinear.x = -texture(samplerPosition, inUV).z;