Move I/O operations to separate geoarrow-rust-io Python package (#754)

.github/workflows/python-io-wheels.yml

@@ -0,0 +1,277 @@
+name: Python Wheels (IO)
+
+on:
+  push:
+    tags:
+      - "py-v*"
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  linux:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        target: [x86_64]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v4
+        with:
+          python-version: 3.8
+          architecture: x64
+
+      - name: Build wheels
+        uses: PyO3/maturin-action@v1
+        with:
+          target: ${{ matrix.target }}
+          manylinux: manylinux_2_28
+          args: --release --out dist -m python/geoarrow-io/Cargo.toml
+          before-script-linux: |
+            yum update -y
+            yum install openssl openssl-devel perl-IPC-Cmd -y
+
+      - name: Install built wheel
+        if: matrix.target == 'x86_64'
+        run: |
+          pip install geoarrow-rust-io --no-index --find-links dist --force-reinstall
+          python -c "import geoarrow.rust.io"
+
+      # Have to set path from root
+      # https://github.com/actions/upload-artifact/issues/232#issuecomment-964235360
+      - name: Upload wheels
+        uses: actions/upload-artifact@v3
+        with:
+          name: wheels
+          path: dist/*.whl
+
+  # linux-cross:
+  #   runs-on: ubuntu-latest
+  #   strategy:
+  #     matrix:
+  #       target: [aarch64, armv7, s390x, ppc64le, ppc64]
+  #   steps:
+  #     - uses: actions/checkout@v4
+  #     - uses: actions/setup-python@v4
+  #       with:
+  #         python-version: 3.8
+
+  #     - name: Build wheels
+  #       uses: PyO3/maturin-action@v1
+  #       with:
+  #         target: ${{ matrix.target }}
+  #         manylinux: auto
+  #         args: --release --out dist -m python/geoarrow-io/Cargo.toml
+
+  #     # This is currently failing with
+  #     # python: command not found
+
+  #     # - uses: uraimo/run-on-arch-action@v2.5.1
+  #     #   if: matrix.target == 'aarch64'
+  #     #   name: Install built wheel
+  #     #   with:
+  #     #     arch: ${{ matrix.target }}
+  #     #     distro: ubuntu20.04
+  #     #     githubToken: ${{ github.token }}
+  #     #     install: |
+  #     #       apt-get update
+  #     #       apt-get install -y --no-install-recommends python3 python3-pip
+  #     #       pip3 install -U pip
+  #     #     run: |
+  #     #       pip install geoarrow-rust-io --no-index --find-links dist --force-reinstall
+  #     #       python -c "import geoarrow.rust.io"
+
+  #     # Have to set path from root
+  #     # https://github.com/actions/upload-artifact/issues/232#issuecomment-964235360
+  #     - name: Upload wheels
+  #       uses: actions/upload-artifact@v3
+  #       with:
+  #         name: wheels
+  #         path: dist/*.whl
+
+  macos:
+    name: Build ${{ matrix.arch }} wheels on ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: macos-11
+            arch: x86_64
+            target: x86_64
+            python-version: 3.8
+          - os: macos-14
+            arch: arm64
+            target: aarch64
+            # Earliest python version on Github for macos arm64
+            # https://raw.githubusercontent.com/actions/python-versions/main/versions-manifest.json
+            python-version: 3.11.3
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - uses: dtolnay/rust-toolchain@stable
+
+      - name: Build wheels - ${{ matrix.target }}
+        uses: PyO3/maturin-action@v1
+        with:
+          target: ${{ matrix.target }}
+          args: --release --out dist --sdist -m python/geoarrow-io/Cargo.toml
+
+      - name: Install built wheel - ${{ matrix.target }}
+        run: |
+          pip install geoarrow-rust-io --no-index --find-links dist --force-reinstall
+          python -c "import geoarrow.rust.io"
+
+      # Have to set path from root
+      # https://github.com/actions/upload-artifact/issues/232#issuecomment-964235360
+      - name: Upload wheels
+        uses: actions/upload-artifact@v3
+        with:
+          name: wheels
+          path: dist
+
+  windows:
+    runs-on: windows-latest
+    strategy:
+      matrix:
+        target: [x64, x86]
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: 3.8
+          architecture: ${{ matrix.target }}
+
+      - uses: dtolnay/rust-toolchain@stable
+
+      - name: Build wheels
+        uses: PyO3/maturin-action@v1
+        with:
+          target: ${{ matrix.target }}
+          args: --release --out dist -m python/geoarrow-io/Cargo.toml
+
+      - name: Install built wheel
+        run: |
+          pip install geoarrow-rust-io --no-index --find-links dist --force-reinstall
+          python -c "import geoarrow.rust.io"
+
+      - name: Upload wheels
+        uses: actions/upload-artifact@v3
+        with:
+          name: wheels
+          path: dist/*.whl
+
+  # NOTE: musllinux builds are commented out because importing them failed with:
+  # ImportError: Error loading shared library libgcc_s-60abea67.so.1: No such file or directory
+
+  # musllinux:
+  #   runs-on: ubuntu-latest
+  #   strategy:
+  #     matrix:
+  #       target:
+  #         - x86_64-unknown-linux-musl
+  #         - i686-unknown-linux-musl
+  #   steps:
+  #     - uses: actions/checkout@v4
+  #     - uses: actions/setup-python@v4
+  #       with:
+  #         python-version: 3.8
+  #         architecture: x64
+
+  #     - name: Build wheels
+  #       uses: PyO3/maturin-action@v1
+  #       with:
+  #         target: ${{ matrix.target }}
+  #         manylinux: musllinux_1_2
+  #         args: --release --out dist -m python/geoarrow-io/Cargo.toml
+
+  #     - name: Install built wheel
+  #       if: matrix.target == 'x86_64-unknown-linux-musl'
+  #       uses: addnab/docker-run-action@v3
+  #       with:
+  #         image: alpine:latest
+  #         options: -v ${{ github.workspace }}:/io -w /io
+  #         run: |
+  #           apk add py3-pip
+  #           pip3 install -U pip
+  #           pip3 install geoarrow-rust-io --no-index --find-links /io/dist/ --force-reinstall
+  #           python3 -c "import geoarrow.rust.io"
+
+  #     - name: Upload wheels
+  #       uses: actions/upload-artifact@v3
+  #       with:
+  #         name: wheels
+  #         path: dist
+
+  # musllinux-cross:
+  #   runs-on: ubuntu-latest
+  #   strategy:
+  #     matrix:
+  #       platform:
+  #         - target: aarch64-unknown-linux-musl
+  #           arch: aarch64
+  #         - target: armv7-unknown-linux-musleabihf
+  #           arch: armv7
+
+  #   steps:
+  #     - uses: actions/checkout@v4
+
+  #     - uses: actions/setup-python@v4
+  #       with:
+  #         python-version: 3.8
+
+  #     - name: Build wheels
+  #       uses: PyO3/maturin-action@v1
+  #       with:
+  #         target: ${{ matrix.platform.target }}
+  #         manylinux: musllinux_1_2
+  #         args: --release --out dist -m python/geoarrow-io/Cargo.toml
+
+  #     - uses: uraimo/run-on-arch-action@v2.5.1
+  #       name: Install built wheel
+  #       with:
+  #         arch: ${{ matrix.platform.arch }}
+  #         distro: alpine_latest
+  #         githubToken: ${{ github.token }}
+  #         install: |
+  #           apk add py3-pip
+  #           pip3 install -U pip
+  #         run: |
+  #           pip3 install geoarrow-rust-io --no-index --find-links dist/ --force-reinstall
+  #           python3 -c "import geoarrow.rust.io"
+
+  #     - name: Upload wheels
+  #       uses: actions/upload-artifact@v3
+  #       with:
+  #         name: wheels
+  #         path: dist
+
+  release:
+    name: Release
+    environment:
+      name: release
+      url: https://pypi.org/p/geoarrow-rust-io
+    permissions:
+      # IMPORTANT: this permission is mandatory for trusted publishing
+      id-token: write
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
+    needs: [macos, windows, linux]
+    steps:
+      - uses: actions/download-artifact@v3
+        with:
+          name: wheels
+          path: dist
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: 3.8
+
+      - name: Publish package distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1

.github/workflows/python.yml

@@ -16,7 +16,7 @@ jobs:
     runs-on: ubuntu-latest
     defaults:
       run:
-        working-directory: python/core
+        working-directory: python
     steps:
       - uses: actions/checkout@v4
         with:
@@ -55,11 +55,19 @@ jobs:
         with:
           python-version: ${{ env.PYTHON_VERSION }}
       - run: pip install pyodide-build
-      - name: Build wheels
+
+      - name: Build wheels (core)
         uses: PyO3/maturin-action@v1
         with:
           target: wasm32-unknown-emscripten
-          args: --no-default-features --out dist -m python/core/Cargo.toml
+          args: --no-default-features -m python/geoarrow-core/Cargo.toml
+          rust-toolchain: nightly
+
+      - name: Build wheels (io)
+        uses: PyO3/maturin-action@v1
+        with:
+          target: wasm32-unknown-emscripten
+          args: --no-default-features -m python/geoarrow-io/Cargo.toml
           rust-toolchain: nightly
 
   # lint-python:
@@ -87,41 +95,56 @@ jobs:
         python-version: ["3.9", "3.12"]
     defaults:
       run:
-        working-directory: python/core
+        working-directory: python
     steps:
       - uses: actions/checkout@v4
         with:
           submodules: "recursive"
+
       - name: Install Rust
         uses: dtolnay/rust-toolchain@stable
+
       - uses: Swatinem/rust-cache@v2
+
       - name: Set up Python
         id: setup-python
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
+
       - name: Install and configure Poetry
         uses: snok/install-poetry@v1
         with:
           version: 1.8.2
           virtualenvs-create: true
           virtualenvs-in-project: true
           installer-parallel: true
+
       - name: Check Poetry lockfile up to date
         run: |
           poetry check --lock
+
       - name: Load cached venv
         id: cached-poetry-dependencies
         uses: actions/cache@v4
         with:
-          path: python/core/.venv
+          path: .venv
           key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
+
       - name: Install dependencies
         if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
         run: poetry install --no-interaction --no-root
+
       - name: Install root project
         run: poetry install --no-interaction
+
+      - name: Build rust submodules
+        run: |
+          # Note: core module should be first, because it could be depended on
+          # by others in the future
+          poetry run maturin develop -m geoarrow-core/Cargo.toml
+          poetry run maturin develop -m geoarrow-io/Cargo.toml
+
       - name: Run python tests
         run: |
-          poetry run maturin develop
           poetry run pytest tests

CHANGELOG.md

@@ -1,6 +1,6 @@
 # Changelog
 
-**This is the changelog for the core Rust library**. There's a [separate changelog](./python/core/CHANGELOG.md) for the Python bindings, and there will be another for the JS bindings.
+**This is the changelog for the core Rust library**. There's a [separate changelog](./python/CHANGELOG.md) for the Python bindings, and there will be another for the JS bindings.
 
 ## [0.3.0-beta.2] - 2024-08-23
 

README.md

@@ -7,7 +7,7 @@
 
 A Rust implementation of the [GeoArrow](https://github.com/geoarrow/geoarrow) specification and bindings to [GeoRust algorithms](https://github.com/georust/geo) for efficient spatial operations on GeoArrow memory.
 
-This repository also includes [Python bindings](https://github.com/geoarrow/geoarrow-rs/blob/main/python/core/README.md) and [JavaScript (WebAssembly) bindings](https://github.com/geoarrow/geoarrow-rs/blob/main/js/README.md), wrapping the GeoArrow memory layout and offering vectorized geometry operations.
+This repository also includes [Python bindings](https://github.com/geoarrow/geoarrow-rs/blob/main/python/README.md) and [JavaScript (WebAssembly) bindings](https://github.com/geoarrow/geoarrow-rs/blob/main/js/README.md), wrapping the GeoArrow memory layout and offering vectorized geometry operations.
 
 ## Documentation