Skip to content

Build and publish package to PyPI #496

Build and publish package to PyPI

Build and publish package to PyPI #496

Workflow file for this run

name: Build and publish package to PyPI
on:
release:
types: [published]
schedule:
# Run at 10 am UTC on day-of-month 1 and 15.
- cron: "0 10 1,15 * *"
workflow_dispatch:
inputs:
target:
description: 'Deployment target. Can be "pypi" or "testpypi"'
default: "testpypi"
debug_enabled:
type: boolean
description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
required: false
default: false
# Set options available for all jobs that use cibuildwheel
env:
PYBAMM_DISABLE_TELEMETRY: "true"
# Increase pip debugging output, equivalent to `pip -vv`
CIBW_BUILD_VERBOSITY: 2
# Disable build isolation to allow pre-installing build-time dependencies.
# Note: CIBW_BEFORE_BUILD must be present in all jobs using cibuildwheel.
CIBW_BUILD_FRONTEND: "pip; args: --no-build-isolation"
# Skip PyPy and MUSL builds in any and all jobs
CIBW_SKIP: "pp* *musllinux*"
FORCE_COLOR: 3
jobs:
build_windows_wheels:
name: Wheels (windows-latest)
runs-on: windows-latest
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
with:
python-version: 3.11
- name: Get number of cores on Windows
id: get_num_cores
shell: python
run: |
from os import environ, cpu_count
num_cpus = cpu_count()
output_file = environ['GITHUB_OUTPUT']
with open(output_file, "a", encoding="utf-8") as output_stream:
output_stream.write(f"count={num_cpus}\n")
- name: Clone pybind11 repo (no history)
run: git clone --depth 1 --branch v2.12.0 https://github.com/pybind/pybind11.git -c advice.detachedHead=false
- name: Cache packages installed through vcpkg on Windows
uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
env:
cache-name: vckpg_binary_cache
with:
path: C:\Users\runneradmin\AppData\Local\vcpkg\archives
key: ${{ runner.os }}-build-VS2022-${{ env.cache-name }}-${{ hashFiles('vcpkg*.json') }}
# Enable tmate debugging of manually-triggered workflows if the input option was provided
- name: Setup tmate session
uses: mxschmitt/action-tmate@e5c7151931ca95bad1c6f4190c730ecf8c7dde48 # v3.19
if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}
- name: Build 64-bit wheels on Windows
run: pipx run cibuildwheel --output-dir wheelhouse
env:
CIBW_ENVIRONMENT: >
PYBAMM_DISABLE_TELEMETRY="true"
PYBAMM_USE_VCPKG=ON
VCPKG_ROOT_DIR=C:\vcpkg
VCPKG_DEFAULT_TRIPLET=x64-windows-static-md
VCPKG_FEATURE_FLAGS=manifests,registries
CMAKE_GENERATOR="Visual Studio 17 2022"
CMAKE_GENERATOR_PLATFORM=x64
CMAKE_BUILD_PARALLEL_LEVEL=${{ steps.get_num_cores.outputs.count }}
CIBW_ARCHS: AMD64
CIBW_BEFORE_BUILD: python -m pip install setuptools delvewheel # skip CasADi and CMake
# Fix access violation because GHA runners have modified PATH that picks wrong
# msvcp140.dll, see https://github.com/adang1345/delvewheel/issues/54
CIBW_REPAIR_WHEEL_COMMAND: delvewheel repair --add-path C:/Windows/System32 -w {dest_dir} {wheel}
CIBW_TEST_EXTRAS: "all,dev,jax"
CIBW_TEST_COMMAND: |
python -c "import pybamm; print(pybamm.IDAKLUSolver())"
python -m pytest -m cibw {project}/tests/unit
- name: Upload Windows wheels
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
with:
name: wheels_windows
path: ./wheelhouse/*.whl
if-no-files-found: error
build_manylinux_wheels:
name: Wheels (linux-amd64)
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
name: Check out PyBaMM repository
- uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
name: Set up Python
with:
python-version: 3.11
- name: Clone pybind11 repo (no history)
run: git clone --depth 1 --branch v2.12.0 https://github.com/pybind/pybind11.git -c advice.detachedHead=false
- name: Build wheels on Linux
run: pipx run cibuildwheel --output-dir wheelhouse
env:
CIBW_ENVIRONMENT: >
PYBAMM_DISABLE_TELEMETRY="true"
CIBW_ARCHS_LINUX: x86_64
CIBW_BEFORE_ALL_LINUX: >
yum -y install openblas-devel lapack-devel &&
bash scripts/install_sundials.sh 6.0.3 6.5.0
CIBW_BEFORE_BUILD_LINUX: python -m pip install cmake casadi setuptools wheel
CIBW_REPAIR_WHEEL_COMMAND_LINUX: auditwheel repair -w {dest_dir} {wheel}
CIBW_TEST_EXTRAS: "all,dev,jax"
CIBW_TEST_COMMAND: |
set -e -x
python -c "import pybamm; print(pybamm.IDAKLUSolver())"
python -m pytest -m cibw {project}/tests/unit
- name: Upload wheels for Linux
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
with:
name: wheels_manylinux
path: ./wheelhouse/*.whl
if-no-files-found: error
build_macos_wheels:
name: Wheels (${{ matrix.os }})
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [macos-13, macos-14]
fail-fast: false
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
with:
python-version: '3.11'
- name: Clone pybind11 repo (no history)
run: git clone --depth 1 --branch v2.12.0 https://github.com/pybind/pybind11.git -c advice.detachedHead=false
- name: Install cibuildwheel
run: python -m pip install cibuildwheel
- name: Build wheels on macOS
shell: bash
run: |
set -e -x
# Set LLVM-OpenMP URL
if [[ $(uname -m) == "x86_64" ]]; then
OPENMP_URL="https://anaconda.org/conda-forge/llvm-openmp/11.1.0/download/osx-64/llvm-openmp-11.1.0-hda6cdc1_1.tar.bz2"
elif [[ $(uname -m) == "arm64" ]]; then
OPENMP_URL="https://anaconda.org/conda-forge/llvm-openmp/11.1.0/download/osx-arm64/llvm-openmp-11.1.0-hf3c4609_1.tar.bz2"
fi
# Download gfortran with proper macOS minimum version (11.0)
if [[ $(uname -m) == "x86_64" ]]; then
GFORTRAN_URL="https://github.com/isuruf/gcc/releases/download/gcc-11.3.0-2/gfortran-darwin-x86_64-native.tar.gz"
KNOWN_SHA256="981367dd0ad4335613e91bbee453d60b6669f5d7e976d18c7bdb7f1966f26ae4 gfortran.tar.gz"
elif [[ $(uname -m) == "arm64" ]]; then
GFORTRAN_URL="https://github.com/isuruf/gcc/releases/download/gcc-11.3.0-2/gfortran-darwin-arm64-native.tar.gz"
KNOWN_SHA256="84364eee32ba843d883fb8124867e2bf61a0cd73b6416d9897ceff7b85a24604 gfortran.tar.gz"
fi
# Validate gfortran tarball
curl -L $GFORTRAN_URL -o gfortran.tar.gz
if ! echo "$KNOWN_SHA256" != "$(shasum --algorithm 256 gfortran.tar.gz)"; then
echo "Checksum failed"
exit 1
fi
mkdir -p gfortran_installed
tar -xv -C gfortran_installed/ -f gfortran.tar.gz
if [[ $(uname -m) == "x86_64" ]]; then
export FC=$(pwd)/gfortran_installed/gfortran-darwin-x86_64-native/bin/gfortran
export PATH=$(pwd)/gfortran_installed/gfortran-darwin-x86_64-native/bin:$PATH
elif [[ $(uname -m) == "arm64" ]]; then
export FC=$(pwd)/gfortran_installed/gfortran-darwin-arm64-native/bin/gfortran
export PATH=$(pwd)/gfortran_installed/gfortran-darwin-arm64-native/bin:$PATH
fi
# link libgfortran dylibs and place them in $HOME/.local/lib
# and then change rpath to $HOME/.local/lib for each of them
# Note: libgcc_s.1.dylib not available on macOS arm64; skip for now
mkdir -p $HOME/.local/lib
if [[ $(uname -m) == "x86_64" ]]; then
lib_dir=$(pwd)/gfortran_installed/gfortran-darwin-x86_64-native/lib
for lib in libgfortran.5.dylib libgfortran.dylib libquadmath.0.dylib libquadmath.dylib libgcc_s.1.dylib libgcc_s.1.1.dylib; do
cp $lib_dir/$lib $HOME/.local/lib/
install_name_tool -id $HOME/.local/lib/$lib $HOME/.local/lib/$lib
codesign --force --sign - $HOME/.local/lib/$lib
done
elif [[ $(uname -m) == "arm64" ]]; then
lib_dir=$(pwd)/gfortran_installed/gfortran-darwin-arm64-native/lib
for lib in libgfortran.5.dylib libgfortran.dylib libquadmath.0.dylib libquadmath.dylib libgcc_s.1.1.dylib; do
cp $lib_dir/$lib $HOME/.local/lib/
install_name_tool -id $HOME/.local/lib/$lib $HOME/.local/lib/$lib
codesign --force --sign - $HOME/.local/lib/$lib
done
fi
export SDKROOT=${SDKROOT:-$(xcrun --show-sdk-path)}
# Can't download LLVM-OpenMP directly, use conda/mamba and set environment variables
brew install miniforge
mamba create -n pybamm-dev $OPENMP_URL
if [[ $(uname -m) == "x86_64" ]]; then
PREFIX="/usr/local/Caskroom/miniforge/base/envs/pybamm-dev"
elif [[ $(uname -m) == "arm64" ]]; then
PREFIX="/opt/homebrew/Caskroom/miniforge/base/envs/pybamm-dev"
fi
# Copy libomp.dylib from PREFIX to $HOME/.local/lib, needed for wheel repair
cp $PREFIX/lib/libomp.dylib $HOME/.local/lib/
install_name_tool -id $HOME/.local/lib/libomp.dylib $HOME/.local/lib/libomp.dylib
codesign --force --sign - $HOME/.local/lib/libomp.dylib
export CC=/usr/bin/clang
export CXX=/usr/bin/clang++
export CPPFLAGS="$CPPFLAGS -Xpreprocessor -fopenmp"
export CFLAGS="$CFLAGS -I$PREFIX/include"
export CXXFLAGS="$CXXFLAGS -I$PREFIX/include"
export LDFLAGS="$LDFLAGS -L$PREFIX/lib -lomp"
# cibuildwheel not recognising its environment variable, so set manually
export CIBUILDWHEEL="1"
python scripts/install_KLU_Sundials.py
python -m cibuildwheel --output-dir wheelhouse
env:
CIBW_ENVIRONMENT: >
PYBAMM_DISABLE_TELEMETRY="true"
# 10.13 for Intel (macos-13), 11.0 for Apple Silicon (macos-14 and macos-latest)
MACOSX_DEPLOYMENT_TARGET: ${{ matrix.os == 'macos-14' && '11.0' || '10.13' }}
CIBW_ARCHS_MACOS: auto
CIBW_BEFORE_BUILD: python -m pip install cmake casadi setuptools delocate
CIBW_REPAIR_WHEEL_COMMAND: |
if [[ $(uname -m) == "x86_64" ]]; then
delocate-listdeps {wheel} && delocate-wheel -v -w {dest_dir} {wheel}
elif [[ $(uname -m) == "arm64" ]]; then
# Use higher macOS target for now since casadi/libc++.1.0.dylib is still not fixed
delocate-listdeps {wheel} && delocate-wheel -v -w {dest_dir} {wheel} --require-target-macos-version 11.1
for file in {dest_dir}/*.whl; do mv "$file" "${file//macosx_11_1/macosx_11_0}"; done
fi
CIBW_TEST_EXTRAS: "all,dev,jax"
CIBW_TEST_COMMAND: |
set -e -x
python -c "import pybamm; print(pybamm.IDAKLUSolver())"
python -m pytest -m cibw {project}/tests/unit
- name: Upload wheels for macOS (amd64, arm64)
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
with:
name: wheels_${{ matrix.os }}
path: ./wheelhouse/*.whl
if-no-files-found: error
build_sdist:
name: Build SDist
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
with:
python-version: 3.12
- name: Build SDist
run: pipx run build --sdist
- name: Upload SDist
uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
with:
name: sdist
path: ./dist/*.tar.gz
if-no-files-found: error
publish_pypi:
# This job is only of value to PyBaMM and would always be skipped in forks
if: github.event_name != 'schedule' && github.repository == 'pybamm-team/PyBaMM'
name: Upload package to PyPI
needs: [
build_manylinux_wheels,
build_macos_wheels,
build_windows_wheels,
build_sdist
]
runs-on: ubuntu-latest
environment:
name: pypi
url: https://pypi.org/p/pybamm
permissions:
id-token: write
steps:
- name: Download all artifacts
uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
with:
path: artifacts
merge-multiple: true
- name: Sanity check downloaded artifacts
run: ls -lA artifacts/
- name: Publish to PyPI
if: github.event.inputs.target == 'pypi' || github.event_name == 'release'
uses: pypa/gh-action-pypi-publish@67339c736fd9354cd4f8cb0b744f2b82a74b5c70 # v1.12.3
with:
packages-dir: artifacts/
- name: Publish to TestPyPI
if: github.event.inputs.target == 'testpypi'
uses: pypa/gh-action-pypi-publish@67339c736fd9354cd4f8cb0b744f2b82a74b5c70 # v1.12.3
with:
packages-dir: artifacts/
repository-url: https://test.pypi.org/legacy/
open_failure_issue:
needs: [
build_windows_wheels,
build_manylinux_wheels,
build_macos_wheels,
build_sdist
]
name: Open an issue if build fails
if: ${{ always() && contains(needs.*.result, 'failure') && github.repository_owner == 'pybamm-team'}}
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
- uses: JasonEtco/create-an-issue@1b14a70e4d8dc185e5cc76d3bec9eab20257b2c5 # v2.9.2
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
LOGS: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
with:
filename: .github/wheel_failure.md