better

.github/workflows/tests.yml

@@ -10,7 +10,7 @@ jobs:
       matrix:
         flags: ['', '--asan', '--ubsan']
         backend: ['MSGQ', 'ZMQ']
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.11", "3.12"]
     steps:
     - uses: actions/checkout@v4
     - uses: actions/setup-python@v5

.github/workflows/wheels.yml

@@ -0,0 +1,51 @@
+name: wheels
+
+on: [push, pull_request]
+
+jobs:
+  wheels:
+    runs-on: ${{ matrix.platform.os }}
+    strategy:
+      fail-fast: true
+      matrix:
+        platform: [ {os: "ubuntu-latest", target: "manylinux_x86_64",  arch: "x86_64"},
+                    {os: "ubuntu-latest", target: "manylinux_aarch64", arch: "aarch64"},
+                    {os: "macos-14",      target: "macosx_arm64",      arch: "arm64"} ]
+        python:   [ {cp: "cp311", py: "3.11"}, {cp: "cp312", py: "3.12"} ]
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up QEMU
+      if: runner.os == 'Linux'
+      uses: docker/setup-qemu-action@v3
+      with:
+        platforms: all
+
+    - name: Building wheel
+      uses: pypa/cibuildwheel@v2.19.2
+      env:
+        CIBW_BUILD: "${{ matrix.python.cp }}-${{ matrix.platform.target }}"
+        CIBW_ARCHS: "${{ matrix.platform.arch }}"
+        CIBW_BEFORE_ALL_LINUX: "bash {project}/scripts/manylinux_dependencies.sh"
+        CIBW_BEFORE_BUILD_MACOS: "bash {project}/scripts/macos_dependencies.sh"
+        CIBW_MANYLINUX_X86_64_IMAGE: "manylinux_2_28"
+        CIBW_MANYLINUX_AARCH64_IMAGE: "manylinux_2_28"
+        CIBW_TEST_COMMAND: pip install -r {project}/requirements.txt && pytest {package}
+
+
+    - uses: actions/setup-python@v5
+      if: ${{ matrix.platform.arch }} != 'aarch64'
+      with:
+        python-version: ${{ matrix.python.py }}
+
+    - name: Installing the wheel
+      if: ${{ matrix.platform.arch }} != 'aarch64'
+      run: |
+        pip install --break-system-packages ./wheelhouse/*.whl
+
+    - name: Saving wheel
+      uses: actions/upload-artifact@v4
+      with:
+        name: cibw-wheels-${{ matrix.platform.target }}-${{ matrix.python.cp }}
+        path: ./wheelhouse/*.whl

pyproject.toml

@@ -1,17 +1,17 @@
 [project]
 name = "msgq"
-requires-python = ">= 3.8"
+requires-python = ">= 3.11"
 version = "0.1.0"
 
 dependencies = [
   "numpy",
   "Cython",
   "pycapnp",
+  "setuptools",
 ]
 
 [project.optional-dependencies]
 dev = [
-  "pyyaml",
   "scons",
   "pre-commit",
   "ruff",

requirements.txt

@@ -0,0 +1,50 @@
+# This file was autogenerated by uv via the following command:
+#    uv pip compile --all-extras -o requirements.txt pyproject.toml
+cfgv==3.4.0
+    # via pre-commit
+coverage==7.6.0
+    # via
+    #   msgq (pyproject.toml)
+    #   pytest-cov
+cython==3.0.10
+    # via msgq (pyproject.toml)
+distlib==0.3.8
+    # via virtualenv
+filelock==3.15.4
+    # via virtualenv
+identify==2.6.0
+    # via pre-commit
+iniconfig==2.0.0
+    # via pytest
+nodeenv==1.9.1
+    # via pre-commit
+numpy==2.0.0
+    # via msgq (pyproject.toml)
+packaging==24.1
+    # via pytest
+parameterized==0.9.0
+    # via msgq (pyproject.toml)
+platformdirs==4.2.2
+    # via virtualenv
+pluggy==1.5.0
+    # via pytest
+pre-commit==3.7.1
+    # via msgq (pyproject.toml)
+pycapnp==2.0.0
+    # via msgq (pyproject.toml)
+pytest==8.2.2
+    # via
+    #   msgq (pyproject.toml)
+    #   pytest-cov
+pytest-cov==5.0.0
+    # via msgq (pyproject.toml)
+pyyaml==6.0.1
+    # via pre-commit
+ruff==0.5.3
+    # via msgq (pyproject.toml)
+scons==4.8.0
+    # via msgq (pyproject.toml)
+setuptools==71.0.4
+    # via msgq (pyproject.toml)
+virtualenv==20.26.3
+    # via pre-commit

scripts/macos_dependencies.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+brew bundle --file=- <<-EOS
+brew "zeromq"
+cask "gcc-arm-embedded"
+brew "gcc@13"
+EOS

scripts/manylinux_dependencies.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+dnf install -y clang opencl-headers ocl-icd-devel cppcheck wget
+
+wget https://github.com/zeromq/libzmq/releases/download/v4.3.5/zeromq-4.3.5.tar.gz
+tar -xvf zeromq-4.3.5.tar.gz
+cd zeromq-4.3.5
+./configure
+make -j$(nproc)
+make install

scripts/ubuntu_dependencies.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+if [[ ! $(id -u) -eq 0 ]]; then
+  if [[ -z $(which sudo) ]]; then
+    echo "Please install sudo or run as root"
+    exit 1
+  fi
+  SUDO="sudo"
+fi
+
+$SUDO apt-get update
+$SUDO apt-get install -y --no-install-recommends clang opencl-headers libzmq3-dev ocl-icd-opencl-dev cppcheck

setup.py

@@ -1,8 +1,13 @@
-from setuptools import Command, setup
+from setuptools import Command, setup, Distribution
 from setuptools.command.build import build
 import subprocess
 import os
 
+class BinaryDistribution(Distribution):
+  def has_ext_modules(self):
+    return True
+
+
 class SconsBuild(Command):
   def initialize_options(self) -> None:
     pass
@@ -12,14 +17,16 @@ def finalize_options(self) -> None:
 
   def run(self) -> None:
     scons_flags = '' if 'BUILD_TESTS' in os.environ else '--minimal'
-    subprocess.run([f"scons {scons_flags} -j$(nproc)"], shell=True).check_returncode()
+    subprocess.run([f"scons {scons_flags} -j$(nproc || sysctl -n hw.logicalcpu)"], shell=True).check_returncode()
+
 
 class CustomBuild(build):
   sub_commands = [('scons_build', None)] + build.sub_commands
 
 setup(
     packages = ["msgq", "msgq.visionipc"],
-    package_data={'msgq': ['**/*.cc', '**/*.h', '**/*.pxd', '**/*.pyx', '**/*.so']},
+    package_data={'': ['**/*.cc', '**/*.c', '**/*.h', '**/*.pxd', '**/*.pyx', '**/*.py', '**/*.so', '**/*.npy']},
     include_package_data=True,
-    cmdclass={'build': CustomBuild, 'scons_build': SconsBuild}
+    cmdclass={'build': CustomBuild, 'scons_build': SconsBuild},
+    distclass=BinaryDistribution,
     )