diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index a184df2ec42..fafb45431f4 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -269,6 +269,11 @@ jobs: brew install ccache ccache -M 2G # See .github/workflows/readme.md for ccache strategy. + - name: Install Open3D-ML Python dependencies + working-directory: ${{ env.OPEN3D_ML_ROOT }} + run: | + python -m pip install -r requirements.txt + - name: Config and build wheel run: | PATH=/usr/local/var/homebrew/linked/ccache/libexec:$PATH diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml index 043cb0c7b97..9871c13eed3 100644 --- a/.github/workflows/style.yml +++ b/.github/workflows/style.yml @@ -27,7 +27,7 @@ jobs: python-version: '3.8' - name: Install dependencies run: | - pip install -U clang-format~=10.0.0 yapf==0.30.0 nbformat + pip install -U -r python/requirements_style.txt - name: Run style check run: | python util/check_style.py diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 04a408802d0..8fab5818504 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -21,8 +21,6 @@ concurrency: env: PIP_VER: "23.2.1" - WHEEL_VER: "0.38.4" - STOOLS_VER: "67.3.2" JEDI_VER: "0.17.2" # https://github.com/ipython/ipython/issues/12740 IDNA_VER: "2.8" # https://github.com/psf/requests/issues/5710 CUDA_VERSION: "12.1.0" @@ -50,6 +48,7 @@ jobs: - BUILD_CUDA_MODULE: ON env: BUILD_WEBRTC: ${{ ( matrix.BUILD_SHARED_LIBS == 'OFF' && matrix.STATIC_RUNTIME == 'ON' ) && 'ON' || 'OFF' }} + WITH_STUBGEN: ${{ ( matrix.BUILD_SHARED_LIBS == 'ON' || matrix.BUILD_CUDA_MODULE == 'ON' ) && 'OFF' || 'ON' }} steps: - name: Disk space used @@ -123,6 +122,7 @@ jobs: -DBUILD_WEBRTC=${{ env.BUILD_WEBRTC }} ` -DBUILD_UNIT_TESTS=ON ` -DBUILD_CUDA_MODULE=${{ matrix.BUILD_CUDA_MODULE }} ` + -DWITH_STUBGEN=${{ env.WITH_STUBGEN }} ` ${{ env.SRC_DIR }} - name: Build @@ -213,15 +213,23 @@ jobs: cmake --build . --config ${{ matrix.CONFIG }} .\${{ matrix.CONFIG }}\Draw.exe --skip-for-unit-test Remove-Item "C:\Program Files\Open3D" -Recurse + - name: Install Open3D python build requirements + working-directory: ${{ env.SOURCE_DIR }} + run: | + $ErrorActionPreference = 'Stop' + python -m pip install -U pip==${{ env.PIP_VER }} + python -m pip install -U -r python/requirements_build.txt + python -m pip install -U jedi==${{ env.JEDI_VER }} idna==${{ env.IDNA_VER }} + - name: Install Open3D python requirements for stubgen + working-directory: ${{ env.SOURCE_DIR }} + if: ${{ env.WITH_STUBGEN == 'ON' }} + run: | + $ErrorActionPreference = 'Stop' + python -m pip install -U -r python/requirements.txt - name: Install Python package working-directory: ${{ env.BUILD_DIR }} run: | $ErrorActionPreference = 'Stop' - python -m pip install --upgrade pip==${{ env.PIP_VER }} ` - wheel==${{ env.WHEEL_VER }} ` - setuptools==${{ env.STOOLS_VER }} ` - jedi==${{ env.JEDI_VER }} ` - idna==${{ env.IDNA_VER }} cmake --build . --config ${{ matrix.CONFIG }} --target install-pip-package - name: Import python package # If BUILD_SHARED_LIBS == ON, Open3D.dll needs to be copied, which is not recommended for python. @@ -272,7 +280,9 @@ jobs: working-directory: ${{ env.SRC_DIR }} run: | $ErrorActionPreference = 'Stop' + python -m pip install -U pip==${{ env.PIP_VER }} python -m pip install -r python/requirements.txt + python -m pip install -r python/requirements_build.txt python -m pip install -r python/requirements_jupyter_build.txt - name: Config @@ -300,9 +310,6 @@ jobs: working-directory: ${{ env.BUILD_DIR }} run: | $ErrorActionPreference = 'Stop' - python -m pip install --upgrade pip==${{ env.PIP_VER }} ` - wheel==${{ env.WHEEL_VER }} ` - setuptools==${{ env.STOOLS_VER }} cmake --build . --parallel ${{ env.NPROC }} --config Release --target pip-package $PIP_PKG_NAME=(Get-ChildItem lib/python_package/pip_package/open3d*.whl).Name echo "PIP_PKG_NAME=$PIP_PKG_NAME" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append @@ -366,9 +373,8 @@ jobs: python -m venv open3d_test_venv open3d_test_venv\Scripts\Activate.ps1 - python -m pip install --upgrade pip==${{ env.PIP_VER }} ` - wheel==${{ env.WHEEL_VER }} ` - setuptools==${{ env.STOOLS_VER }} + python -m pip install -U pip==${{ env.PIP_VER }} + python -m pip install -U -r python/requirements_build.txt python -m pip install -U -r python/requirements_test.txt $py_tag=(python -c "import sys; print(f'cp{sys.version_info.major}{sys.version_info.minor}')") if (Test-Path -Path "pip_package") { diff --git a/CHANGELOG.md b/CHANGELOG.md index fccf0b78497..b8ad64fd2f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,6 +53,7 @@ - Fix render to depth image on Apple Retina displays (PR #7001) - Fix infinite loop in segment_plane if num_points < ransac_n (PR #7032) - Add select_by_index method to Feature class (PR #7039) +- Include typing stubs in python package (PR #6917) ## 0.13 diff --git a/CMakeLists.txt b/CMakeLists.txt index e1bd4706288..2433ecee515 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,6 +39,8 @@ option(BUILD_EXAMPLES "Build Open3D examples programs" ON option(BUILD_UNIT_TESTS "Build Open3D unit tests" OFF) option(BUILD_BENCHMARKS "Build the micro benchmarks" OFF) option(BUILD_PYTHON_MODULE "Build the python module" ON ) +option(WITH_STUBGEN "Use pybind11-stubgen to generate stubs" ON ) +option(IGNORE_STUBGEN_ERRORS "Ignore all errors during stub generation" ON ) option(BUILD_CUDA_MODULE "Build the CUDA module" OFF) option(BUILD_WITH_CUDA_STATIC "Build with static CUDA libraries" ON ) option(BUILD_COMMON_CUDA_ARCHS "Build for common CUDA GPUs (for release)" OFF) diff --git a/cmake/Open3DPrintConfigurationSummary.cmake b/cmake/Open3DPrintConfigurationSummary.cmake index 3e3d42da11e..bd4cfc700bc 100644 --- a/cmake/Open3DPrintConfigurationSummary.cmake +++ b/cmake/Open3DPrintConfigurationSummary.cmake @@ -37,6 +37,10 @@ function(open3d_print_configuration_summary) open3d_aligned_print("Build Unit Tests" "${BUILD_UNIT_TESTS}") open3d_aligned_print("Build Examples" "${BUILD_EXAMPLES}") open3d_aligned_print("Build Python Module" "${BUILD_PYTHON_MODULE}") + open3d_aligned_print("Generate Typing Stubs" "${WITH_STUBGEN}") + if(WITH_STUBGEN) + open3d_aligned_print("Ignore All Stubgen Errors" "${IGNORE_STUBGEN_ERRORS}") + endif() open3d_aligned_print("Build Jupyter Extension" "${BUILD_JUPYTER_EXTENSION}") open3d_aligned_print("Build TensorFlow Ops" "${BUILD_TENSORFLOW_OPS}") open3d_aligned_print("Build PyTorch Ops" "${BUILD_PYTORCH_OPS}") diff --git a/cpp/pybind/CMakeLists.txt b/cpp/pybind/CMakeLists.txt index 6efae9a17fd..b569119835a 100644 --- a/cpp/pybind/CMakeLists.txt +++ b/cpp/pybind/CMakeLists.txt @@ -226,6 +226,8 @@ add_custom_target(python-package -DPYTHON_VERSION=${PYTHON_VERSION} "-DCOMPILED_MODULE_PATH_LIST=${COMPILED_MODULE_PATH_LIST}" "-DPYTHON_EXTRA_LIBRARIES=${PYTHON_EXTRA_LIBRARIES}" + -DWITH_STUBGEN=${WITH_STUBGEN} + -DIGNORE_STUBGEN_ERRORS=${IGNORE_STUBGEN_ERRORS} -DBUILD_JUPYTER_EXTENSION=${BUILD_JUPYTER_EXTENSION} -DBUILD_TENSORFLOW_OPS=${BUILD_TENSORFLOW_OPS} -DBUILD_PYTORCH_OPS=${BUILD_PYTORCH_OPS} diff --git a/cpp/pybind/docstring.cpp b/cpp/pybind/docstring.cpp index fce26d58688..e9d03711d35 100644 --- a/cpp/pybind/docstring.cpp +++ b/cpp/pybind/docstring.cpp @@ -289,17 +289,9 @@ std::string FunctionDoc::ToGoogleDocString() const { return rc.str(); } -std::string FunctionDoc::NamespaceFix(const std::string& s) { - std::string rc = std::regex_replace(s, std::regex("::(\\S)"), ".$1"); - rc = std::regex_replace(rc, std::regex("open3d\\.(cpu|cuda)\\.pybind\\."), - "open3d."); - return rc; -} - std::string FunctionDoc::StringCleanAll(std::string& s, const std::string& white_space) { std::string rc = utility::StripString(s, white_space); - rc = NamespaceFix(rc); return rc; } @@ -313,7 +305,7 @@ ArgumentDoc FunctionDoc::ParseArgumentToken(const std::string& argument_token) { std::smatch matches; if (std::regex_search(argument_token, matches, rgx_with_default)) { argument_doc.name_ = matches[1].str(); - argument_doc.type_ = NamespaceFix(matches[2].str()); + argument_doc.type_ = matches[2].str(); argument_doc.default_ = matches[3].str(); // Handle long default value. Long default has multiple lines and thus @@ -335,7 +327,7 @@ ArgumentDoc FunctionDoc::ParseArgumentToken(const std::string& argument_token) { "([A-Za-z_][A-Za-z\\d_:\\.\\[\\]\\(\\) ,]*)"); if (std::regex_search(argument_token, matches, rgx_without_default)) { argument_doc.name_ = matches[1].str(); - argument_doc.type_ = NamespaceFix(matches[2].str()); + argument_doc.type_ = matches[2].str(); } } diff --git a/cpp/pybind/geometry/pointcloud.cpp b/cpp/pybind/geometry/pointcloud.cpp index 2dbb4c741d3..566cdd6b50d 100644 --- a/cpp/pybind/geometry/pointcloud.cpp +++ b/cpp/pybind/geometry/pointcloud.cpp @@ -150,7 +150,7 @@ void pybind_pointcloud_definitions(py::module &m) { &PointCloud::OrientNormalsConsistentTangentPlane, "Function to orient the normals with respect to consistent " "tangent planes", - "k"_a, "lambda"_a = 0.0, "cos_alpha_tol"_a = 1.0) + "k"_a, "lambda_penalty"_a = 0.0, "cos_alpha_tol"_a = 1.0) .def("compute_point_cloud_distance", &PointCloud::ComputePointCloudDistance, "For each point in the source point cloud, compute the " diff --git a/cpp/pybind/make_python_package.cmake b/cpp/pybind/make_python_package.cmake index 01e0d5663f5..d1b38659770 100644 --- a/cpp/pybind/make_python_package.cmake +++ b/cpp/pybind/make_python_package.cmake @@ -136,3 +136,17 @@ file(COPY "${PYTHON_PACKAGE_SRC_DIR}/../examples/python/" DESTINATION "${PYTHON_PACKAGE_DST_DIR}/open3d/examples") file(COPY "${PYTHON_PACKAGE_SRC_DIR}/../examples/python/" DESTINATION "${PYTHON_PACKAGE_DST_DIR}/open3d/examples") + +# Generate typing stub files (.pyi) and py.typed marker file. +if(WITH_STUBGEN) + if(NOT IGNORE_STUBGEN_ERRORS) + list(APPEND PYBIND11_STUBGEN_FLAGS "--exit-code") + endif() + message(STATUS "Generating typing stubs...") + execute_process( + COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH=${PYTHON_PACKAGE_DST_DIR} pybind11-stubgen open3d -o ${PYTHON_PACKAGE_DST_DIR} ${PYBIND11_STUBGEN_FLAGS} + COMMAND_ECHO STDOUT + COMMAND_ERROR_IS_FATAL ANY + ) + file(WRITE "${PYTHON_PACKAGE_DST_DIR}/open3d/py.typed") +endif() diff --git a/cpp/pybind/t/geometry/pointcloud.cpp b/cpp/pybind/t/geometry/pointcloud.cpp index e5bac211b90..172f92c2356 100644 --- a/cpp/pybind/t/geometry/pointcloud.cpp +++ b/cpp/pybind/t/geometry/pointcloud.cpp @@ -331,17 +331,17 @@ infinite value. It also removes the corresponding attributes. pointcloud.def( "orient_normals_consistent_tangent_plane", &PointCloud::OrientNormalsConsistentTangentPlane, "k"_a, - "lambda"_a = 0.0, "cos_alpha_tol"_a = 1.0, + "lambda_penalty"_a = 0.0, "cos_alpha_tol"_a = 1.0, R"(Function to consistently orient the normals of a point cloud based on tangent planes. The algorithm is described in Hoppe et al., "Surface Reconstruction from Unorganized Points", 1992. -Additional information about the choice of lambda and cos_alpha_tol for complex +Additional information about the choice of lambda_penalty and cos_alpha_tol for complex point clouds can be found in Piazza, Valentini, Varetti, "Mesh Reconstruction from Point Cloud", 2023 (https://eugeniovaretti.github.io/meshreco/Piazza_Valentini_Varetti_MeshReconstructionFromPointCloud_2023.pdf). Args: k (int): Number of neighbors to use for tangent plane estimation. - lambda (float): A non-negative real parameter that influences the distance + lambda_penalty (float): A non-negative real parameter that influences the distance metric used to identify the true neighbors of a point in complex geometries. It penalizes the distance between a point and the tangent plane defined by the reference point and its normal vector, helping to @@ -354,7 +354,7 @@ point clouds can be found in Piazza, Valentini, Varetti, "Mesh Reconstruction fr Example: We use Bunny point cloud to compute its normals and orient them consistently. The initial reconstruction adheres to Hoppe's algorithm (raw), whereas the - second reconstruction utilises the lambda and cos_alpha_tol parameters. + second reconstruction utilises the lambda_penalty and cos_alpha_tol parameters. Due to the high density of the Bunny point cloud available in Open3D a larger value of the parameter k is employed to test the algorithm. Usually you do not have at disposal such a refined point clouds, thus you cannot find a @@ -379,7 +379,7 @@ point clouds can be found in Piazza, Valentini, Varetti, "Mesh Reconstruction fr poisson_mesh.compute_vertex_normals() o3d.visualization.draw_geometries([poisson_mesh]) - # Case 2, reconstruction using lambda and cos_alpha_tol parameters: + # Case 2, reconstruction using lambda_penalty and cos_alpha_tol parameters: pcd_robust = o3d.io.read_point_cloud(data.path) # Compute normals and orient them consistently, using k=100 neighbours diff --git a/docker/Dockerfile.wheel b/docker/Dockerfile.wheel index eb60286b620..edf30a1015c 100644 --- a/docker/Dockerfile.wheel +++ b/docker/Dockerfile.wheel @@ -109,11 +109,15 @@ RUN /root/Open3D/util/install_deps_ubuntu.sh assume-yes \ # Open3D Python dependencies COPY ./util/ci_utils.sh /root/Open3D/util/ COPY ./python/requirements.txt /root/Open3D/python/ +COPY ./python/requirements_build.txt /root/Open3D/python/ COPY ./python/requirements_jupyter_build.txt /root/Open3D/python/ COPY ./python/requirements_jupyter_install.txt /root/Open3D/python/ RUN source /root/Open3D/util/ci_utils.sh \ && install_python_dependencies with-jupyter +# Open3D-ML Python dependencies +RUN python -m pip install -r "${OPEN3D_ML_ROOT}/requirements.txt" + # Open3D Jupyter dependencies RUN curl -fsSL https://deb.nodesource.com/setup_16.x | bash - \ && apt-get install -y nodejs \ diff --git a/docs/conf.py b/docs/conf.py index eac752217c7..61d9234ec72 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -264,8 +264,31 @@ def skip(app, what, name, obj, would_skip, options): return would_skip +def fix_namespace(text): + """Fixes namespace in a string by removing .[cpu|cuda].pybind.""" + return (text.replace("open3d.cpu.pybind.", "open3d.").replace( + "open3d.cuda.pybind.cuda.", "open3d.") if text is not None else None) + + +def process_signature(app, what, name, obj, options, signature, + return_annotation): + """Fixes namespace in signature by removing .[cpu|cuda].pybind.""" + return ( + fix_namespace(signature), + fix_namespace(return_annotation), + ) + + +def process_docstring(app, what, name, obj, options, lines): + """Fixes namespace in docstring by removing .[cpu|cuda].pybind.""" + for i, line in enumerate(lines): + lines[i] = fix_namespace(line) + + def setup(app): app.connect("autodoc-skip-member", skip) + app.connect("autodoc-process-signature", process_signature) + app.connect("autodoc-process-docstring", process_docstring) # Add Google analytics app.add_js_file("https://www.googletagmanager.com/gtag/js?id=G-3TQPKGV6Z3", **{'async': 'async'}) diff --git a/python/open3d/__init__.py b/python/open3d/__init__.py index d4d5c613b0e..5a8c11556fb 100644 --- a/python/open3d/__init__.py +++ b/python/open3d/__init__.py @@ -212,4 +212,4 @@ def _jupyter_nbextension_paths(): if sys.platform == "win32": _win32_dll_dir.close() -del os, sys, CDLL, load_cdll, find_library, Path, warnings, _insert_pybind_names +del os, sys, CDLL, load_cdll, find_library, Path, warnings, _insert_pybind_names, open3d diff --git a/python/pyproject.toml b/python/pyproject.toml index 3d229011876..f65c9888bcb 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -1,3 +1,12 @@ [build-system] -requires = ["ipywidgets>=8.0.3", "pygments>=2.7.4", "jupyter_packaging~=0.12", "jupyterlab>=3.0.0,==3.*", "setuptools>=50.3.2", "wheel==0.38.4"] +requires = [ + "ipywidgets>=8.0.4", + "pygments>=2.7.4", + "jupyter_packaging~=0.12", + "jupyterlab>=3.0.0,==3.*", + "setuptools>=67.3.2", + "wheel==0.38.4", + "yapf==0.30.0", + "pybind11-stubgen==2.5.1", +] build-backend = "setuptools.build_meta" diff --git a/python/requirements_build.txt b/python/requirements_build.txt index b9bb3b1f28a..9d8dc63cb4b 100644 --- a/python/requirements_build.txt +++ b/python/requirements_build.txt @@ -1,3 +1,4 @@ setuptools>=67.3.2 wheel==0.38.4 yapf==0.30.0 +pybind11-stubgen==2.5.1 \ No newline at end of file diff --git a/util/ci_utils.sh b/util/ci_utils.sh index 1adb4b7d8c5..8c8f4217361 100644 --- a/util/ci_utils.sh +++ b/util/ci_utils.sh @@ -30,9 +30,6 @@ TORCH_VER="2.2.2" TORCH_REPO_URL="https://download.pytorch.org/whl/torch/" # Python PIP_VER="23.2.1" -WHEEL_VER="0.38.4" -STOOLS_VER="67.3.2" -YAPF_VER="0.30.0" PROTOBUF_VER="4.24.0" OPEN3D_INSTALL_DIR=~/open3d_install @@ -42,10 +39,10 @@ install_python_dependencies() { echo "Installing Python dependencies" options="$(echo "$@" | tr ' ' '|')" - python -m pip install --upgrade pip=="$PIP_VER" wheel=="$WHEEL_VER" \ - setuptools=="$STOOLS_VER" + python -m pip install --upgrade pip=="$PIP_VER" + python -m pip install -U -r "${OPEN3D_SOURCE_ROOT}/python/requirements_build.txt" if [[ "with-unit-test" =~ ^($options)$ ]]; then - python -m pip install -U -r python/requirements_test.txt + python -m pip install -U -r "${OPEN3D_SOURCE_ROOT}/python/requirements_test.txt" fi if [[ "with-cuda" =~ ^($options)$ ]]; then TF_ARCH_NAME=tensorflow @@ -64,7 +61,6 @@ install_python_dependencies() { TORCH_GLNX="torch==${TORCH_VER}+cpu" fi - # TODO: modify other locations to use requirements.txt python -m pip install -r "${OPEN3D_SOURCE_ROOT}/python/requirements.txt" if [[ "with-jupyter" =~ ^($options)$ ]]; then python -m pip install -r "${OPEN3D_SOURCE_ROOT}/python/requirements_jupyter_build.txt" @@ -88,7 +84,6 @@ install_python_dependencies() { fi fi if [ "$BUILD_TENSORFLOW_OPS" == "ON" ] || [ "$BUILD_PYTORCH_OPS" == "ON" ]; then - python -m pip install -U yapf=="$YAPF_VER" # Fix Protobuf compatibility issue # https://stackoverflow.com/a/72493690/1255535 # https://github.com/protocolbuffers/protobuf/issues/10051 @@ -241,8 +236,8 @@ test_wheel() { python -m venv open3d_test.venv # shellcheck disable=SC1091 source open3d_test.venv/bin/activate - python -m pip install --upgrade pip=="$PIP_VER" wheel=="$WHEEL_VER" \ - setuptools=="$STOOLS_VER" + python -m pip install -U pip=="$PIP_VER" + python -m pip install -U -r "${OPEN3D_SOURCE_ROOT}/python/requirements_build.txt" echo -n "Using python: $(command -v python)" python --version echo -n "Using pip: " @@ -285,7 +280,7 @@ test_wheel() { run_python_tests() { # shellcheck disable=SC1091 source open3d_test.venv/bin/activate - python -m pip install -U -r python/requirements_test.txt + python -m pip install -U -r "${OPEN3D_SOURCE_ROOT}/python/requirements_test.txt" echo Add --randomly-seed=SEED to the test command to reproduce test order. pytest_args=("$OPEN3D_SOURCE_ROOT"/python/test/) if [ "$BUILD_PYTORCH_OPS" == "OFF" ] && [ "$BUILD_TENSORFLOW_OPS" == "OFF" ]; then @@ -356,9 +351,8 @@ install_docs_dependencies() { echo Install Python dependencies for building docs command -v python python -V - python -m pip install -U -q "wheel==$WHEEL_VER" \ - "pip==$PIP_VER" - python -m pip install -U -q "yapf==$YAPF_VER" + python -m pip install -U -q "pip==$PIP_VER" + python -m pip install -U -r "${OPEN3D_SOURCE_ROOT}/python/requirements_build.txt" if [[ -d "$1" ]]; then OPEN3D_ML_ROOT="$1" echo Installing Open3D-ML dependencies from "${OPEN3D_ML_ROOT}"