diff --git a/include/sapien/sapien_renderer/camera_component.h b/include/sapien/sapien_renderer/camera_component.h index 9ead13d5..e1c33e72 100644 --- a/include/sapien/sapien_renderer/camera_component.h +++ b/include/sapien/sapien_renderer/camera_component.h @@ -3,10 +3,10 @@ #include "image.h" #include "sapien/math/mat.h" #include "sapien/math/pose.h" +#include #include #include #include -#include namespace sapien { class Entity; @@ -14,6 +14,8 @@ namespace sapien_renderer { struct SapienRenderCameraInternal; class SapienRenderTexture; +enum class CameraMode { ePerspective, eOrthographic }; + class SapienRenderCameraComponent : public Component { public: SapienRenderCameraComponent(uint32_t width, uint32_t height, std::string const &shaderDir); @@ -38,6 +40,11 @@ class SapienRenderCameraComponent : public Component { inline float getPrincipalPointY() const { return mCy; } inline float getSkew() const { return mSkew; } + inline float getOrthoLeft() const { return mLeft; } + inline float getOrthoRight() const { return mRight; } + inline float getOrthoBottom() const { return mBottom; } + inline float getOrthoTop() const { return mTop; } + Mat4 getModelMatrix() const; Mat4 getProjectionMatrix() const; Mat3 getIntrinsicMatrix() const; @@ -54,6 +61,12 @@ class SapienRenderCameraComponent : public Component { void setPrincipalPoint(float cx, float cy); void setSkew(float s); + void setOrthographicParameters(float near, float far, float top); + void setOrthographicParameters(float near, float far, float left, float right, float bottom, + float top); + + inline CameraMode getMode() const { return mMode; } + void takePicture(); std::vector getImageNames() const; SapienRenderImageCpu getImage(std::string const &name); @@ -85,6 +98,9 @@ class SapienRenderCameraComponent : public Component { SapienRenderCameraComponent &operator=(SapienRenderCameraComponent const &&) = delete; private: + CameraMode mMode{CameraMode::ePerspective}; + void checkMode(CameraMode mode) const; + uint32_t mWidth{}; uint32_t mHeight{}; float mFx{}; @@ -94,6 +110,13 @@ class SapienRenderCameraComponent : public Component { float mNear{}; float mFar{}; float mSkew{}; + + // ortho only + float mLeft{}; + float mRight{}; + float mBottom{}; + float mTop{}; + std::string mShaderDir; std::map> mProperties; @@ -112,4 +135,3 @@ class SapienRenderCameraComponent : public Component { } // namespace sapien_renderer } // namespace sapien - diff --git a/include/sapien/sapien_renderer/window.h b/include/sapien/sapien_renderer/window.h index 477d9316..d734caed 100644 --- a/include/sapien/sapien_renderer/window.h +++ b/include/sapien/sapien_renderer/window.h @@ -1,4 +1,5 @@ #pragma once +#include "camera_component.h" #include "image.h" #include "material.h" #include "sapien/math/mat.h" @@ -60,6 +61,8 @@ class SapienRendererWindow { void setCameraParameters(float near, float far, float fovy); void setCameraIntrinsicParameters(float near, float far, float fx, float fy, float cx, float cy, float skew); + void setCameraOrthoParameters(float near, float far, float top); + CameraMode getCameraMode(); void setCameraPose(Pose const &pose); Pose getCameraPose(); @@ -85,6 +88,7 @@ class SapienRendererWindow { float getCameraNear(); float getCameraFar(); float getCameraFovy(); + float getCameraOrthoTop(); glm::mat4 getCameraProjectionMatrix(); Mat4 getCameraModelMatrix(); diff --git a/python/py_package/__init__.pyi b/python/py_package/__init__.pyi index da5a8eec..7638f6d5 100644 --- a/python/py_package/__init__.pyi +++ b/python/py_package/__init__.pyi @@ -34,5 +34,5 @@ from . import utils from . import version from . import wrapper __all__ = ['ActorBuilder', 'ArticulationBuilder', 'Component', 'CudaArray', 'Device', 'Engine', 'Entity', 'Path', 'PinocchioModel', 'Pose', 'SapienRenderer', 'Scene', 'SceneConfig', 'System', 'Widget', 'asset', 'internal_renderer', 'math', 'os', 'physx', 'pkg_resources', 'platform', 'profile', 'pysapien', 'pysapien_pinocchio', 'render', 'set_log_level', 'simsense', 'utils', 'version', 'warn', 'wrapper'] -__version__: str = '3.0.0.dev20240516+9b8c9cc0' +__version__: str = '3.0.0.dev20240520+3037022b' __warningregistry__: dict = {'version': 0} diff --git a/python/py_package/pysapien/render.pyi b/python/py_package/pysapien/render.pyi index b7496dc4..717e0fb5 100644 --- a/python/py_package/pysapien/render.pyi +++ b/python/py_package/pysapien/render.pyi @@ -69,6 +69,8 @@ class RenderCameraComponent(sapien.pysapien.Component): """ def get_local_pose(self) -> sapien.pysapien.Pose: ... + def get_mode(self) -> typing.Literal['perspective', 'orthographic']: + ... def get_model_matrix(self) -> numpy.ndarray[tuple[typing.Literal[4], typing.Literal[4]], numpy.dtype[numpy.float32]]: """ Get model matrix (inverse of extrinsic matrix) used in rendering (Y up, Z back) @@ -118,6 +120,12 @@ class RenderCameraComponent(sapien.pysapien.Component): ... def set_near(self, near: float) -> None: ... + @typing.overload + def set_orthographic_parameters(self, near: float, far: float, top: float) -> None: + ... + @typing.overload + def set_orthographic_parameters(self, near: float, far: float, left: float, right: float, bottom: float, top: float) -> None: + ... def set_perspective_parameters(self, near: float, far: float, fx: float, fy: float, cx: float, cy: float, skew: float) -> None: ... def set_principal_point(self, cx: float, cy: float) -> None: @@ -169,6 +177,21 @@ class RenderCameraComponent(sapien.pysapien.Component): def height(self) -> int: ... @property + def mode(self) -> typing.Literal['perspective', 'orthographic']: + ... + @property + def ortho_bottom(self) -> float: + ... + @property + def ortho_left(self) -> float: + ... + @property + def ortho_right(self) -> float: + ... + @property + def ortho_top(self) -> float: + ... + @property def width(self) -> int: ... class RenderCameraGroup: @@ -844,6 +867,8 @@ class RenderWindow: ... def resize(self, width: int, height: int) -> None: ... + def set_camera_orthographic_parameters(self, near: float, far: float, top: float) -> None: + ... def set_camera_parameters(self, near: float, far: float, fovy: float) -> None: ... def set_camera_pose(self, pose: sapien.pysapien.Pose) -> None: @@ -896,6 +921,9 @@ class RenderWindow: def alt(self) -> bool: ... @property + def camera_mode(self) -> typing.Literal['perspective', 'orthographic']: + ... + @property def ctrl(self) -> bool: ... @property @@ -925,6 +953,9 @@ class RenderWindow: def near(self) -> float: ... @property + def ortho_top(self) -> float: + ... + @property def shift(self) -> bool: ... @property diff --git a/python/py_package/utils/viewer/control_window.py b/python/py_package/utils/viewer/control_window.py index 853cd972..fa29a8e5 100644 --- a/python/py_package/utils/viewer/control_window.py +++ b/python/py_package/utils/viewer/control_window.py @@ -201,6 +201,10 @@ def single_step(self, _): self._single_step = True def copy_camera_settings(self, _=None): + if self.orthographic: + print('copying from orthographic camera is not supported yet') + return + p = self.window.get_camera_position() q = self.window.get_camera_rotation() width, height = self.window.size @@ -276,7 +280,19 @@ def build(self): .append( self.ui_camera, self.ui_camera_image, - R.UISliderAngle().Label("FOV Y").Min(1).Max(179).Bind(self, "fovy"), + R.UICheckbox().Label("Orthographic").Bind(self, "orthographic"), + R.UIConditional() + .Bind(self, "orthographic") + .append( + R.UISliderFloat() + .Label("Ortho Scale") + .Min(0.1) + .Max(10) + .Bind(self, "ortho_scale"), + ), + R.UIConditional().Bind(self, "_perspective").append( + R.UISliderAngle().Label("FOV Y").Min(1).Max(179).Bind(self, "fovy"), + ), R.UIButton() .Label("Copy Camera Settings") .Callback(self.copy_camera_settings), @@ -772,6 +788,35 @@ def _clear_camera_linesets(self): render_scene.remove_node(n) self.camera_linesets = [] + @property + def orthographic(self): + return self.window.camera_mode == "orthographic" + + @property + def _perspective(self): + return not self.orthographic + + @orthographic.setter + def orthographic(self, v): + if v: + self.window.set_camera_orthographic_parameters( + self.window.near, self.window.far, 1 + ) + else: + self.window.set_camera_parameters( + self.window.near, self.window.far, np.pi / 3 + ) + + @property + def ortho_scale(self): + return self.window.ortho_top + + @ortho_scale.setter + def ortho_scale(self, v): + self.window.set_camera_orthographic_parameters( + self.window.near, self.window.far, v + ) + @property def fovy(self): return self.window.fovy diff --git a/python/pybind/sapien_renderer.cpp b/python/pybind/sapien_renderer.cpp index 478a29d5..61129dcf 100644 --- a/python/pybind/sapien_renderer.cpp +++ b/python/pybind/sapien_renderer.cpp @@ -15,6 +15,35 @@ using namespace sapien; using namespace sapien::sapien_renderer; namespace pybind11::detail { + +template <> struct type_caster { + PYBIND11_TYPE_CASTER(CameraMode, _("typing.Literal['perspective', 'orthographic']")); + + bool load(py::handle src, bool convert) { + std::string name = py::cast(src); + if (name == "perspective" || name == "pers") { + value = CameraMode::ePerspective; + return true; + } else if (name == "orthographic" || name == "ortho") { + value = CameraMode::eOrthographic; + return true; + } + return false; + } + + static py::handle cast(CameraMode const &src, py::return_value_policy policy, + py::handle parent) { + switch (src) { + case CameraMode::ePerspective: + return py::str("perspective").release(); + case CameraMode::eOrthographic: + return py::str("orthographic").release(); + default: + return py::str("none").release(); + } + } +}; + template <> struct type_caster { PYBIND11_TYPE_CASTER(svulkan2::renderer::RTRenderer::DenoiserType, _("typing.Literal['none', 'oidn', 'optix']")); @@ -933,10 +962,26 @@ This function waits for any pending CUDA operations on cuda stream provided by : .def("get_skew", &SapienRenderCameraComponent::getSkew) .def("set_skew", &SapienRenderCameraComponent::setSkew, py::arg("skew")) + .def_property_readonly("mode", &SapienRenderCameraComponent::getMode) + .def("get_mode", &SapienRenderCameraComponent::getMode) .def("set_perspective_parameters", &SapienRenderCameraComponent::setPerspectiveParameters, py::arg("near"), py::arg("far"), py::arg("fx"), py::arg("fy"), py::arg("cx"), py::arg("cy"), py::arg("skew")) + .def("set_orthographic_parameters", + py::overload_cast( + &SapienRenderCameraComponent::setOrthographicParameters), + py::arg("near"), py::arg("far"), py::arg("top")) + .def("set_orthographic_parameters", + py::overload_cast( + &SapienRenderCameraComponent::setOrthographicParameters), + py::arg("near"), py::arg("far"), py::arg("left"), py::arg("right"), py::arg("bottom"), + py::arg("top")) + .def_property_readonly("ortho_left", &SapienRenderCameraComponent::getOrthoLeft) + .def_property_readonly("ortho_right", &SapienRenderCameraComponent::getOrthoRight) + .def_property_readonly("ortho_bottom", &SapienRenderCameraComponent::getOrthoBottom) + .def_property_readonly("ortho_top", &SapienRenderCameraComponent::getOrthoTop) + .def("get_intrinsic_matrix", &SapienRenderCameraComponent::getIntrinsicMatrix, "Get 3x3 intrinsic camera matrix in OpenCV format.") .def("get_extrinsic_matrix", &SapienRenderCameraComponent::getExtrinsicMatrix, @@ -1118,6 +1163,12 @@ consumer library. Make a copy if needed. .def("set_intrinsic_parameters", &SapienRendererWindow::setCameraIntrinsicParameters, py::arg("near"), py::arg("far"), py::arg("fx"), py::arg("fy"), py::arg("cx"), py::arg("cy"), py::arg("skew")) + + .def("set_camera_orthographic_parameters", &SapienRendererWindow::setCameraOrthoParameters, + py::arg("near"), py::arg("far"), py::arg("top")) + .def_property_readonly("camera_mode", &SapienRendererWindow::getCameraMode) + .def_property_readonly("ortho_top", &SapienRendererWindow::getCameraOrthoTop) + .def("set_camera_pose", &SapienRendererWindow::setCameraPose, py::arg("pose")) .def("set_camera_position", &SapienRendererWindow::setCameraPosition, py::arg("position")) .def("set_camera_rotation", &SapienRendererWindow::setCameraRotation, py::arg("quat")) diff --git a/src/sapien_renderer/camera_component.cpp b/src/sapien_renderer/camera_component.cpp index 4f137550..94cd1344 100644 --- a/src/sapien_renderer/camera_component.cpp +++ b/src/sapien_renderer/camera_component.cpp @@ -1,4 +1,3 @@ -#include #include "sapien/sapien_renderer/camera_component.h" #include "../logger.h" #include "sapien/entity.h" @@ -7,6 +6,7 @@ #include "sapien/sapien_renderer/sapien_renderer_system.h" #include "sapien/sapien_renderer/texture.h" #include "sapien/scene.h" +#include #include #include @@ -308,33 +308,27 @@ SapienRenderImageCuda SapienRenderCameraComponent::getImageCuda(std::string cons #endif } +static Mat4 mat4glm2eigen(glm::mat4 const &mat) { + return Eigen::Map>(&mat[0][0], 4, 4); +} + // getters Mat4 SapienRenderCameraComponent::getModelMatrix() const { return PoseToEigenMat4(getGlobalPose() * POSE_GL_TO_ROS); } Mat4 SapienRenderCameraComponent::getProjectionMatrix() const { - Mat4 mat = Mat4::Identity(); - float fx = getFocalLengthX(); - float fy = getFocalLengthY(); - float width = getWidth(); - float height = getHeight(); - float far = getFar(); - float near = getNear(); - float cx = getPrincipalPointX(); - float cy = getPrincipalPointY(); - float skew = getSkew(); - mat(0, 0) = (2.f * fx) / width; - mat(1, 1) = -(2.f * fy) / height; - mat(2, 2) = -far / (far - near); - mat(2, 3) = -far * near / (far - near); - mat(3, 2) = -1.f; - mat(0, 2) = -2.f * cx / width + 1; - mat(1, 2) = -2.f * cy / height + 1; - mat(3, 3) = 0.f; - mat(0, 1) = -2 * skew / width; - return mat; + checkMode(CameraMode::ePerspective); + switch (mMode) { + case CameraMode::ePerspective: + return mat4glm2eigen( + svulkan2::math::fullPerspective(mNear, mFar, mFx, mFy, mCx, mCy, mWidth, mHeight, mSkew)); + case CameraMode::eOrthographic: + return mat4glm2eigen(svulkan2::math::ortho(mLeft, mRight, mBottom, mTop, mNear, mFar)); + } + throw std::runtime_error("corrupted camera mode"); } Mat3 SapienRenderCameraComponent::getIntrinsicMatrix() const { + checkMode(CameraMode::ePerspective); Mat3 mat = Mat3::Identity(); mat(0, 0) = getFocalLengthX(); mat(1, 1) = getFocalLengthY(); @@ -342,6 +336,13 @@ Mat3 SapienRenderCameraComponent::getIntrinsicMatrix() const { mat(1, 2) = getPrincipalPointY(); mat(0, 1) = getSkew(); return mat; + + // TODO: support ortho, check if the following is right + // Mat3 mat = Mat3::Identity(); + // mat(0, 0) = getWidth() / (getOrthoRight() - getOrthoLeft()); + // mat(1, 1) = getHeight() / (getOrthoTop() - getOrthoBottom()); + // mat(0, 2) = getOrthoLeft() / (getOrthoLeft() - getOrthoRight()); + // mat(1, 2) = getOrthoTop() / (getOrthoTop() - getOrthoBottom()); } Mat34 SapienRenderCameraComponent::getExtrinsicMatrix() const { Mat34 ros2cv; @@ -353,6 +354,7 @@ Mat34 SapienRenderCameraComponent::getExtrinsicMatrix() const { void SapienRenderCameraComponent::setPerspectiveParameters(float near, float far, float fx, float fy, float cx, float cy, float skew) { + mMode = CameraMode::ePerspective; mNear = near; mFar = far; mFx = fx; @@ -365,34 +367,62 @@ void SapienRenderCameraComponent::setPerspectiveParameters(float near, float far mSkew); } } + +void SapienRenderCameraComponent::setOrthographicParameters(float near, float far, float top) { + float aspect = mWidth / mHeight; + setOrthographicParameters(near, far, -top * aspect, top * aspect, -top, top); +} + +void SapienRenderCameraComponent::setOrthographicParameters(float near, float far, float left, + float right, float bottom, float top) { + mMode = CameraMode::eOrthographic; + mNear = near; + mFar = far; + mLeft = left; + mRight = right; + mBottom = bottom; + mTop = top; + if (mCamera) { + mCamera->mCamera->setOrthographicParameters(near, far, left, right, bottom, top, mWidth, + mHeight); + } +} + void SapienRenderCameraComponent::setFocalLengths(float fx, float fy) { + checkMode(CameraMode::ePerspective); setPerspectiveParameters(getNear(), getFar(), fx, fy, getPrincipalPointX(), getPrincipalPointY(), getSkew()); } void SapienRenderCameraComponent::setFovX(float fovx, bool computeY) { + checkMode(CameraMode::ePerspective); float fx = getWidth() / 2.f / std::tan(fovx / 2); float fy = computeY ? fx : getFocalLengthY(); setFocalLengths(fx, fy); } void SapienRenderCameraComponent::setFovY(float fovy, bool computeX) { + checkMode(CameraMode::ePerspective); float fy = getHeight() / 2.f / std::tan(fovy / 2); float fx = computeX ? fy : getFocalLengthX(); setFocalLengths(fx, fy); } void SapienRenderCameraComponent::setNear(float near) { + checkMode(CameraMode::ePerspective); // TODO: implement for ortho setPerspectiveParameters(near, getFar(), getFocalLengthX(), getFocalLengthY(), getPrincipalPointX(), getPrincipalPointY(), getSkew()); } void SapienRenderCameraComponent::setFar(float far) { + checkMode(CameraMode::ePerspective); // TODO: implement for ortho setPerspectiveParameters(getNear(), far, getFocalLengthX(), getFocalLengthY(), getPrincipalPointX(), getPrincipalPointY(), getSkew()); } void SapienRenderCameraComponent::setPrincipalPoint(float cx, float cy) { + checkMode(CameraMode::ePerspective); setPerspectiveParameters(getNear(), getFar(), getFocalLengthX(), getFocalLengthY(), cx, cy, getSkew()); } void SapienRenderCameraComponent::setSkew(float s) { + checkMode(CameraMode::ePerspective); setPerspectiveParameters(getNear(), getFar(), getFocalLengthX(), getFocalLengthY(), getPrincipalPointX(), getPrincipalPointY(), s); } @@ -410,6 +440,17 @@ void SapienRenderCameraComponent::internalUpdate() { mUpdatedWithoutTakingPicture = true; } +void SapienRenderCameraComponent::checkMode(CameraMode mode) const { + if (mMode == mode) { + return; + } + if (mMode == CameraMode::ePerspective) { + throw std::runtime_error("this function is only available for perspective camera"); + } else { + throw std::runtime_error("this function is only available for orthographic camera"); + } +} + void SapienRenderCameraComponent::gpuInit() { if (mGpuInitialized) { return; diff --git a/src/sapien_renderer/window.cpp b/src/sapien_renderer/window.cpp index 74bf812e..1f73d15f 100644 --- a/src/sapien_renderer/window.cpp +++ b/src/sapien_renderer/window.cpp @@ -202,6 +202,23 @@ void SapienRendererWindow::setCameraIntrinsicParameters(float near, float far, f getCamera()->setPerspectiveParameters(near, far, fx, fy, cx, cy, width, height, skew); } +void SapienRendererWindow::setCameraOrthoParameters(float near, float far, float top) { + uint32_t width = std::max(mWindow->getWidth(), 1u); + uint32_t height = std::max(mWindow->getHeight(), 1u); + getCamera()->setOrthographicParameters(near, far, top, width, height); +} + +CameraMode SapienRendererWindow::getCameraMode() { + switch (getCamera()->getCameraType()) { + case svulkan2::scene::Camera::Type::eOrthographic: + return CameraMode::eOrthographic; + case svulkan2::scene::Camera::Type::ePerspective: + return CameraMode::ePerspective; + default: + throw std::runtime_error("unsupported camera mode"); + } +} + void SapienRendererWindow::setCameraPose(Pose const &pose) { auto glpose = pose * POSE_GL_TO_ROS; auto cam = getCamera(); @@ -277,7 +294,8 @@ Mat4 SapienRendererWindow::getCameraModelMatrix() { float SapienRendererWindow::getCameraNear() { return getCamera()->getNear(); } float SapienRendererWindow::getCameraFar() { return getCamera()->getFar(); } -float SapienRendererWindow::getCameraFovy() { return getCamera()->getFovy(); }; +float SapienRendererWindow::getCameraFovy() { return getCamera()->getFovy(); } +float SapienRendererWindow::getCameraOrthoTop() { return getCamera()->getOrthographicTop(); } std::vector SapienRendererWindow::getDisplayTargetNames() const { return mSVulkanRenderer->getDisplayTargetNames();