Skip to content

Tests

Tests #4703

Workflow file for this run

---
name: Tests
"on":
push:
branches:
- main
pull_request:
schedule:
# Run tests every Monday at 9:17 to catch regressions.
- cron: "17 9 * * 1"
# XXX Concurrency detection sucks and jobs gets killed randomly.
# concurrency:
# # Group workflow jobs so new commits cancels in-progress execution triggered by previous commits.
# # Source: https://mail.python.org/archives/list/pypa-committers@python.org/thread/PCBCQMJF64JGRBOX7E2EE4YLKHT4DI55/
# group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
# cancel-in-progress: true
jobs:
test-matrix:
# There is no way to selective flags collections of elements in a matrix, without having to flag all combinations.
# This will became unmaintainable and tedious so we use this job to pre-compute which jobs is going to get our
# "stable" flag.
name: "OS/Python/stable matrix pre-compute"
runs-on: ubuntu-24.04
outputs:
test_matrix: ${{ steps.create_matrix.outputs.matrix }}
steps:
- name: Create JSON matrix
id: create_matrix
shell: python
run: |
import json
import os
from itertools import product
from pathlib import Path
# To speed up the tests, we only test the latest available OS for each platform. For Python, we only test
# against the oldest and newest major supported version and skip intermediates. We also test on the upcoming
# development version to anticipate issues.
variants: dict[str, set[str]] = {
# Available OS: https://github.com/actions/runner-images#available-images
"os": {
"ubuntu-24.04",
"macos-15",
"windows-2022",
},
# Available Python: https://github.com/actions/python-versions/blob/main/versions-manifest.json
"python-version": {
"3.10",
"3.13",
"3.14-dev",
}
}
# TODO: List of additional variants to include in the matrix.
include: list[dict[str, str]] = []
# List of variants to exclude from the matrix.
exclude: list[dict[str, str]] = []
# List of unstable criterions.
unstable: list[dict[str, str]] = [
# XXX error: the configured Python interpreter version (3.14) is
# newer than PyO3's maximum supported version (3.13)
{"python-version": "3.14-dev"},
]
# Build the job matrix.
jobs: list[dict[str, str]] = []
for variants in product(*[{(key, value) for value in values} for key, values in variants.items()]):
job = dict(variants)
# Match the job against the exclude criterions.
exclude_job = False
for criterion in exclude:
if set(criterion.items()).issubset(job.items()):
exclude_job = True
break
if exclude_job:
continue
# Match the job against the unstable criterions.
job["state"] = "stable"
for criterion in unstable:
if set(criterion.items()).issubset(job.items()):
job["state"] = "unstable"
break
jobs.append(job)
matrix = json.dumps({"include": jobs})
env_file = Path(os.getenv("GITHUB_OUTPUT"))
env_file.write_text(f"matrix={matrix}")
- name: Print JSON matrix
run: |
echo '${{ steps.create_matrix.outputs.matrix }}'
jq -aR <<< echo '${{ steps.create_matrix.outputs.matrix }}'
tests:
needs:
- test-matrix
strategy:
fail-fast: false
matrix: ${{ fromJson(needs.test-matrix.outputs.test_matrix) }}
runs-on: ${{ matrix.os }}
# We keep going when a job flagged as not stable fails.
continue-on-error: ${{ matrix.state == 'unstable' }}
env:
# XXX Workaround for Windows runners redirecting the output of commands to files. See:
# https://github.com/databrickslabs/dbx/issues/455#issuecomment-1312770919
# https://github.com/pallets/click/issues/2121#issuecomment-1312773882
# https://gist.github.com/NodeJSmith/e7e37f2d3f162456869f015f842bcf15
PYTHONIOENCODING: "utf8"
# Deactivate Homebrew verbose output and auto-update.
HOMEBREW_NO_ENV_HINTS: "1"
HOMEBREW_NO_AUTO_UPDATE: "1"
# Do not let brew use its live API to fetch the latest version.
# This variable forces brew to use the local repository instead (which we need in destructive tests to checkout
# formulas from the past): https://github.com/Homebrew/brew/blob/10845a1/Library/Homebrew/env_config.rb#L314-L317
# See: meta_package_manager/test_manager_homebrew.py::TestCask test.
HOMEBREW_NO_INSTALL_FROM_API: "1"
steps:
- uses: actions/checkout@v4.2.2
- name: Python ${{ matrix.python-version }} - Install
uses: actions/setup-python@v5.3.0
with:
python-version: ${{ matrix.python-version }}
- name: Linux - Fix runner cache folder permissions
if: runner.os == 'Linux'
run: |
mkdir -p /home/runner/.cache
sudo chown -R runner /home/runner/.cache
### Install or upgrade package managers on each platform, and made them
### ready to use for mpm.
# Pip
- name: Pip & Setuptools - Upgrade
run: |
python -m pip install --upgrade setuptools pip
# Pipx
- name: Pipx - Install
run: |
python -m pip install --upgrade pipx
# Homebrew
- name: Homebrew - macOS upgrade
if: runner.os == 'macOS'
run: |
brew upgrade
brew update
- name: Homebrew - macOS cask repository copy
if: runner.os == 'macOS'
# Clone the full copy of the cask repository, so tests on macOS can locally checkout past versions of casks.
# This is performed by the destructive meta_package_manager/test_manager_homebrew.py::TestCask test.
run: |
brew tap homebrew/cask
- name: Homebrew - Linux install
if: runner.os == 'Linux'
run: |
/bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
echo "/home/linuxbrew/.linuxbrew/bin/" >> "$GITHUB_PATH"
- name: Homebrew - Doctor and tap repair
if: runner.os != 'Windows'
run: |
brew doctor || true
brew tap --repair
# Apt
- name: APT - Linux upgrade
if: runner.os == 'Linux'
run: |
sudo apt --quiet --yes update
# Chocolatey
- name: Chocolatey - Windows self-upgrade
if: runner.os == 'Windows'
run: |
choco upgrade chocolatey --no-progress --yes
- name: Chocolatey - Pin un-upgradeable packages
if: runner.os == 'Windows'
# XXX The upgrade all step below fails for strawberryperl package on windows-2022 runners v2.313.0:
#
# You have nginx v1.25.3 installed. Version 1.25.4 is available based on your source(s).
#
# nginx v1.25.4 [Approved]
# nginx package files upgrade completed. Performing other installation steps.
# Port '80' is in use by 'System'...
# ERROR: Please specify a different port number...
# Environment Vars (like PATH) have changed. Close/reopen your shell to
# see the changes (or in powershell/cmd.exe just type `refreshenv`).
# The upgrade of nginx was NOT successful.
# Error while running 'C:\ProgramData\chocolatey\lib\nginx\tools\chocolateyInstall.ps1'.
# See log for details.
#
# You have strawberryperl v5.32.1.1 installed. Version 5.38.2.2 is available based on your source(s).
#
# strawberryperl v5.38.2.2 [Approved]
# strawberryperl package files upgrade completed. Performing other installation steps.
# Downloading strawberryperl 64 bit
# from 'https://github.com/StrawberryPerl/Perl-Dist-Strawberry/strawberry-perl-5.38.2.2-64bit.msi'
#
# Download of strawberry-perl-5.38.2.2-64bit.msi (171.74 MB) completed.
# Hashes match.
# Installing strawberryperl...
# WARNING: Generic MSI Error. This is a local environment error, not an issue with a package or the MSI
# itself - it could mean a pending reboot is necessary prior to install or something else (like the same
# version is already installed). Please see MSI log if available. If not, try again adding
# '--install-arguments="'/l*v c:\StrawberryPerl_msi_install.log'"'. Then search the MSI Log for
# "Return Value 3" and look above that for the error.
# ERROR: Running ["C:\Windows\System32\msiexec.exe" /i "C:\Users\runneradmin\AppData\Local\Temp\chocolatey
# \StrawberryPerl\5.38.2.2\strawberry-perl-5.38.2.2-64bit.msi" /qn /norestart ] was not successful. Exit
# code was '1603'. Exit code indicates the following: Generic MSI Error. This is a local environment error,
# not an issue with a package or the MSI itself - it could mean a pending reboot is necessary prior to
# install or something else (like the same version is already installed). Please see MSI log if available.
# If not, try again adding '--install-arguments="'/l*v c:\StrawberryPerl_msi_install.log'"'. Then search
# the MSI Log for "Return Value 3" and look above that for the error..
# The upgrade of strawberryperl was NOT successful.
#
# You have R.Project v4.4.1 installed. Version 4.4.2 is available based on your source(s).
# PATH environment variable has C:\Program Files\R\R-4.4.1\bin in it. Removing...
# Downloading package from source 'https://community.chocolatey.org/api/v2/'
#
# R.Project v4.4.2 [Approved]
# R.Project package files upgrade completed. Performing other installation steps.
# Installing r.project...
# r.project has been installed.
# R.Project can be automatically uninstalled.
# The upgrade of R.Project was successful.
# Deployed to 'C:\Program Files\R\R-4.4.2\'
#
# You have rtools v4.4.6104 installed. Version 4.4.6335 is available based on your source(s).
# Downloading package from source 'https://community.chocolatey.org/api/v2/'
#
# rtools v4.4.6335 [Approved]
# rtools package files upgrade completed. Performing other installation steps.
# Downloading rtools 64 bit
# from 'https://github.com/r-windows/rtools-chocolatey/releases/download/6335/rtools44-6335-6327.exe'
#
# Download of rtools44-6335-6327.exe (435.06 MB) completed.
# Hashes match.
# Installing rtools...
# Chocolatey timed out waiting for the command to finish. The timeout
# specified (or the default value) was '2700' seconds. Perhaps try a
# higher `--execution-timeout`? See `choco -h` for details.
# The upgrade of rtools was NOT successful.
# Error while running 'C:\ProgramData\chocolatey\lib\rtools\tools\chocolateyInstall.ps1'.
# See log for details.
run: |
choco pin add --name="nginx"
choco pin add --name="strawberryperl"
choco pin add --name="rtools"
- name: Chocolatey - Windows full upgrade
if: runner.os == 'Windows'
# We upgrade all packages preemptively as the test suite will attempt it but fail,
# because of mpm's enforced timeout.
run: |
choco upgrade all --no-progress --yes --ignore-package-exit-codes --ignore-detected-reboot
# NPM
- name: NPM - Linux install
if: runner.os == 'Linux'
run: |
sudo apt --quiet --yes install npm
- name: NPM - macOS install
if: runner.os == 'macOS'
run: |
brew install npm
- name: NPM - Windows upgrade
if: runner.os == 'Windows'
run: |
npm install --global npm
# Cargo
- name: Cargo - macOS install
if: runner.os == 'macOS'
run: |
brew install rust
- name: Cargo - Linux install
if: runner.os == 'Linux'
run: |
sudo apt --quiet --yes install cargo
- name: Cargo - Windows install
if: runner.os == 'Windows'
run: |
choco install rust --no-progress --yes
# Yarn
- name: Yarn - macOS install
if: runner.os == 'macOS'
run: |
brew install yarn
# XXX Upgrading Yarn on Linux via mpm is not working. Like in Windows below.
#
# β–Ί sudo apt --quiet --yes install yarn
# The following NEW packages will be installed:
# yarn
# (...)
# Setting up yarn (1.22.19-1) ...
# (...)
# _____________ TestUpgrade.test_manager_selection[single_exclusion] _____________
# (...)
# ----------------------------- Captured stdout call -----------------------------
# (...)
# β–Ί mpm --exclude pip upgrade --all
# Upgrade all outdated packages from apt...
# (...)
# Upgrade all outdated packages from yarn...
# Return code: 1
# Traceback (most recent call last):
# File ".../site-packages/click/testing.py", line 408, in invoke
# return_value = cli.main(args=args or (), prog_name=prog_name, **extra)
# File ".../site-packages/click_extra/commands.py", line 159, in main
# return super().main(*args, **kwargs)
# File ".../site-packages/click/core.py", line 1055, in main
# rv = self.invoke(ctx)
# File ".../site-packages/click_extra/commands.py", line 213, in invoke
# return super().invoke(ctx)
# File ".../site-packages/click/core.py", line 1657, in invoke
# return _process_result(sub_ctx.command.invoke(sub_ctx))
# File ".../site-packages/click/core.py", line 1404, in invoke
# return ctx.invoke(self.callback, **ctx.params)
# File ".../site-packages/click/core.py", line 760, in invoke
# return __callback(*args, **kwargs)
# File ".../site-packages/click/decorators.py", line 26, in new_func
# return f(get_current_context(), *args, **kwargs)
# File ".../meta_package_manager/cli.py", line 892, in upgrade
# output = manager.upgrade()
# File ".../meta_package_manager/base.py", line 974, in upgrade
# return self.run(cli, extra_env=self.extra_env)
# File ".../meta_package_manager/base.py", line 643, in run
# code, output, error = run_cmd(
# File ".../site-packages/click_extra/run.py", line 110, in run_cmd
# process = subprocess.run(
# File ".../python3.10/subprocess.py", line 503, in run
# with Popen(*popenargs, **kwargs) as process:
# File ".../python3.10/subprocess.py", line 971, in __init__
# self._execute_child(args, executable, preexec_fn, close_fds,
# File ".../python3.10/subprocess.py", line 1847, in _execute_child
# raise child_exception_type(errno_num, err_msg, err_filename)
# FileNotFoundError: [Errno 2] No such file or directory: '/usr/local/bin/yarn'
#
# - name: Yarn - Linux install
# if: runner.os == 'Linux'
# run: |
# curl -sSL https://dl.yarnpkg.com/debian/pubkey.gpg | sudo apt-key add -
# echo "deb https://dl.yarnpkg.com/debian/ stable main" | sudo tee /etc/apt/sources.list.d/yarn.list
# sudo apt --quiet --yes update
# sudo apt --quiet --yes install yarn
# XXX Upgrading Yarn on Windows via mpm is not working, as it seems to requires PATH re-evaluation, which cannot
# be achieve unless we re-open a new shell. A thing we cannot do in a GitHub Action. So we simply do not install
# it on Windows for now.
#
# β–Ί choco install yarn --no-progress --yes
# Chocolatey v1.3.0
# Download of yarn-1.22.19.msi (1.6 MB) completed.
# yarn has been installed.
# (...)
# Environment Vars (like PATH) have changed. Close/reopen your shell to
# see the changes (or in powershell/cmd.exe just type `refreshenv`).
# The install of yarn was successful.
# Software installed as 'msi', install location is likely default.
# (...)
# ___________ TestUpgrade.test_manager_selection[multiple_exclusions] ___________
# (...)
# ---------------------------- Captured stdout call -----------------------------
# β–Ί mpm --exclude pip --exclude gem upgrade --all
# mwarning: cargo does not implement Operations.upgrade_all.
# Upgrade all outdated packages from choco...
# (...)
# warning: Skip unavailable scoop manager.
# warning: steamcmd does not implement Operations.upgrade_all.
# warning: vscode does not implement Operations.upgrade_all.
# Upgrade all outdated packages from yarn...
# Return code: 1
# Traceback (most recent call last):
# File "...\site-packages\click\testing.py", line 408, in invoke
# return_value = cli.main(args=args or (), prog_name=prog_name, **extra)
# File "...\site-packages\click_extra\commands.py", line 159, in main
# return super().main(*args, **kwargs)
# File "...\site-packages\click\core.py", line 1055, in main
# rv = self.invoke(ctx)
# File "...\site-packages\click_extra\commands.py", line 213, in invoke
# return super().invoke(ctx)
# File "...\site-packages\click\core.py", line 1657, in invoke
# return _process_result(sub_ctx.command.invoke(sub_ctx))
# File "...\site-packages\click\core.py", line 1404, in invoke
# return ctx.invoke(self.callback, **ctx.params)
# File "...\site-packages\click\core.py", line 760, in invoke
# return __callback(*args, **kwargs)
# File "...\site-packages\click\decorators.py", line 26, in new_func
# return f(get_current_context(), *args, **kwargs)
# File "...\meta_package_manager\cli.py", line 892, in upgrade
# output = manager.upgrade()
# File "...\meta_package_manager\base.py", line 974, in upgrade
# return self.run(cli, extra_env=self.extra_env)
# File "...\meta_package_manager\base.py", line 643, in run
# code, output, error = run_cmd(
# File "...\site-packages\click_extra\run.py", line 110, in run_cmd
# process = subprocess.run(
# File "...\Python\3.11.2\x64\Lib\subprocess.py", line 548, in run
# with Popen(*popenargs, **kwargs) as process:
# File "...\Python\3.11.2\x64\Lib\subprocess.py", line 1024, in __init__
# self._execute_child(args, executable, preexec_fn, close_fds,
# File "...\Python\3.11.2\x64\Lib\subprocess.py", line 1493, in _execute_child
# hp, ht, pid, tid = _winapi.CreateProcess(executable, args,
# FileNotFoundError: [WinError 2] The system cannot find the file specified
#
# - name: Yarn - Windows install
# if: runner.os == 'Windows'
# run: |
# choco install yarn --no-progress --yes
# Composer
- name: Composer - macOS install
if: runner.os == 'macOS'
run: |
brew install composer
- name: Composer - Windows upgrade
if: runner.os == 'Windows'
run: |
choco upgrade composer --no-progress --yes
# MAS
- name: MAS - macOS install
if: runner.os == 'macOS'
run: |
brew install mas
# Flatpak
- name: Flatpak - Linux install
if: runner.os == 'Linux'
run: |
sudo apt --quiet --yes install flatpak
# Scoop
- name: Scoop - Windows install
if: runner.os == 'Windows'
shell: pwsh
run: |
iwr get.scoop.sh -outfile 'install.ps1'
.\install.ps1 -RunAsAdmin
# WinGet
# XXX Rely on a hacked action as WinGet is not installable on GitHub action runners as-is:
# https://github.com/microsoft/winget-cli/issues/3872
- name: WinGet - Windows install
if: runner.os == 'Windows'
uses: Cyberboss/install-winget@v1.1.1
# run: |
# winget upgrade winget --accept-package-agreements --accept-source-agreements --disable-interactivity
# DNF
- name: DNF - Linux install
if: runner.os == 'Linux'
run: |
sudo apt --quiet --yes install dnf
# Zypper
- name: Zypper - Linux install
if: runner.os == 'Linux'
run: |
sudo apt --quiet --yes install zypper
# Snap
- name: Snap - Linux upgrade
if: runner.os == 'Linux'
run: |
sudo apt --quiet --yes install snapd
# VSCode
- name: VSCode - macOS install
if: runner.os == 'macOS'
# XXX Tweak permission to fix this issue:
# Hint: https://github.com/Homebrew/discussions/discussions/633#discussioncomment-1400084
# > Run brew install --cask visual-studio-code
# ==>Downloading https://update.code.visualstudio.com/1.68.1/darwin/stable
# ==>Downloading from https://az764295.vo.msecnd.net/stable/35f373630/VSCode-darwin.zip
# Warning: macOS's Gatekeeper has been disabled for this Cask
# ==>Installing Cask visual-studio-code
# ==>Purging files for version 1.68.1 of Cask visual-studio-code
# Error: Directory not empty @ dir_s_rmdir - /private/tmp/d20220706-3993-qik29s
# Error: Process completed with exit code 1.
run: |
sudo chown -R "$USER" /private/tmp/
brew install --cask visual-studio-code
- name: VSCode - Linux install
if: runner.os == 'Linux'
run: |
sudo snap install --classic code
- name: VSCode - Windows install
if: runner.os == 'Windows'
run: |
choco install vscode --no-progress --yes
# Gem
- name: Ruby & Gem - Windows upgrade
if: runner.os == 'Windows'
# The 'refreshenv' call forces the new Ruby version installed by 'choco' to be registered in the 'PATH'.
# The Import-Module workaround is inspired by:
# https://github.com/actions/runner-images/discussions/6065
# https://docs.chocolatey.org/en-us/troubleshooting#why-does-choco-tab-not-work-for-me
run: |
choco upgrade ruby --no-progress --yes
Import-Module "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1"
refreshenv
gem update --quiet --system
- name: Ruby & Gem - macOS
if: runner.os == 'macOS'
run: |
sudo gem update --quiet --system
- name: Ruby & Gem - Ubuntu
# As of Ubuntu 21.04, RubyGems is exclusively managed by APT. A call to "sudo gem update --system"
# ends up with the following error:
# ERROR: Your RubyGems was installed trough APT, and upgrading it through RubyGems
# itself is unsupported. If you really need the latest version of RubyGems (tip:
# you usually don't), then you need to install RubyGems (and Ruby) manually,
# maybe using tools like ruby-install, rvm, etc.
if: runner.os == 'Linux'
run: |
sudo apt --quiet --yes install ruby-rubygems
# SteamCMD
- name: SteamCMD - macOS install
if: runner.os == 'macOS'
run: |
brew install --cask steamcmd
- name: SteamCMD - Linux install
if: runner.os == 'Linux'
# Some notes on the Steam situation on Linux:
# https://popey.com/blog/2023/08/i386-in-ubuntu-wont-die/
run: |
sudo add-apt-repository --yes multiverse
sudo dpkg --add-architecture i386
sudo apt --quiet --yes update
# Accept Steam license.
# Source: https://askubuntu.com/a/1017487
echo steam steam/question select "I AGREE" | sudo debconf-set-selections
echo steam steam/license note '' | sudo debconf-set-selections
sudo debconf-show steam
sudo apt --quiet --yes install libgcc-s1:i386 libstdc++6:i386 libatomic1:i386
sudo apt --quiet --yes install steam
sudo apt --quiet --yes install steamcmd
- name: SteamCMD - Windows install
if: runner.os == 'Windows'
run: |
choco install steamcmd --no-progress --yes
- name: Install uv
run: |
python -m pip install -r https://raw.githubusercontent.com/kdeldycke/workflows/v4.7.2/requirements/uv.txt
- name: Python 3.14 dependencies for lxml
if: runner.os == 'Linux' && matrix.python-version == '3.14-dev'
run: |
sudo apt --quiet --yes install libxml-dev libxslt-dev
- name: Install project
run: |
uv --no-progress venv
uv --no-progress sync --extra test
# CLI tests.
- name: mpm --help
run: |
uv --no-progress run --frozen -- mpm
- name: mpm --version
run: |
uv --no-progress run --frozen -- mpm --version
- name: mpm managers
run: |
uv --no-progress run --frozen -- mpm managers
- name: mpm --all-managers managers
run: |
uv --no-progress run --frozen -- mpm --all-managers managers
- name: mpm installed
run: |
uv --no-progress run --frozen -- mpm installed
- name: mpm outdated
run: |
uv --no-progress run --frozen -- mpm outdated
- name: mpm which mpm
run: |
uv --no-progress run --frozen -- mpm which mpm
- name: mpm backup
run: |
uv --no-progress run --frozen -- mpm backup
- name: mpm sbom --spdx
run: |
uv --no-progress run --frozen -- mpm sbom --spdx
- name: mpm sbom --cyclonedx
run: |
uv --no-progress run --frozen -- mpm sbom --cyclonedx
- name: Parallel non-destructive tests
run: >
uv --no-progress run --frozen --
pytest --exitfirst
--numprocesses=auto --run-non-destructive --skip-destructive
--ignore=./tests/test_bar_plugin.py
- name: Sequential random destructive tests
run: >
uv --no-progress run --frozen --
pytest --exitfirst
--numprocesses=0 --skip-non-destructive --run-destructive
--ignore=./tests/test_bar_plugin.py
# Bar plugin test runs.
- name: bar_plugin.py --search-mpm
if: runner.os == 'macOS'
run: |
./meta_package_manager/bar_plugin.py --search-mpm
- name: uv run bar_plugin.py --search-mpm
if: runner.os == 'macOS'
run: |
uv --no-progress run --frozen -- ./meta_package_manager/bar_plugin.py --search-mpm
- name: bar_plugin.py
if: runner.os == 'macOS'
run: |
./meta_package_manager/bar_plugin.py
- name: uv run bar_plugin.py
if: runner.os == 'macOS'
run: |
uv --no-progress run --frozen -- ./meta_package_manager/bar_plugin.py
- name: Sequential, bar plugin tests - macOS
if: runner.os == 'macOS'
# XXX bar plugin calls way too many external commands to be run in parallel.
run: >
uv --no-progress run --frozen --
pytest --exitfirst
--numprocesses=0 --run-non-destructive --run-destructive
./tests/test_bar_plugin.py
- name: Coverage - Upload Codecov
uses: codecov/codecov-action@v5.0.2
with:
token: ${{ secrets.CODECOV_TOKEN }}