From 75f72956979ecc16bfd37fefb2da2fdcfab6a73a Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Tue, 17 Oct 2023 14:19:26 +0200 Subject: [PATCH 1/3] Fix macOS detection when running standalone --- .github/workflows/pr-check.yml | 41 ++++++++++++++++++++++++++++++++++ alibuild_helpers/utilities.py | 23 +++++++++++++------ tests/test_utilities.py | 13 +++++++++++ tox.ini | 24 +++++++++++--------- 4 files changed, 83 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pr-check.yml b/.github/workflows/pr-check.yml index cfc15595..4f297694 100644 --- a/.github/workflows/pr-check.yml +++ b/.github/workflows/pr-check.yml @@ -63,6 +63,47 @@ jobs: with: files: coverage.json + mac-unittest: + name: python${{ matrix.python-version }} + runs-on: macos-latest + + strategy: + matrix: + python-version: + - '3.11' + + steps: + - uses: actions/checkout@v3 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Install test dependencies + run: | + brew install modules alisw/system-deps/o2-full-deps + python3 -m pip install --upgrade tox tox-gh-actions coverage + + - name: Run tests + run: tox -e darwin + + - name: Convert coverage information + if: ${{ always() && github.event.repository.owner.login == 'alisw' }} + run: | + coverage combine .tox/coverage.* + # Codecov only understands XML, JSON and LCOV files. + # Apparently, patching os.readlink in unit tests interferes with + # finding some source files, but our source shouldn't be affected, so + # ignore these errors. + coverage json --ignore-errors -o coverage.json + + - name: Upload coverage information + if: ${{ always() && github.event.repository.owner.login == 'alisw' }} + uses: codecov/codecov-action@v3 + with: + files: coverage.json + lint: name: lint runs-on: ubuntu-latest diff --git a/alibuild_helpers/utilities.py b/alibuild_helpers/utilities.py index c7361355..cf9a402c 100644 --- a/alibuild_helpers/utilities.py +++ b/alibuild_helpers/utilities.py @@ -131,13 +131,22 @@ def format(s, **kwds): return decode_with_fallback(s) % kwds -def doDetectArch(hasOsRelease, osReleaseLines, platformTuple, platformSystem, platformProcessor): - if platformSystem == "Darwin": - import platform - if platform.machine() == "x86_64": - return "osx_x86-64" - else: - return "osx_arm64" +def doDetectArch(hasOsRelease, osReleaseLines, platformTuple, pSystem, platformProcessor): + print(platformTuple) + print(pSystem) + print(pSystem == "Darwin") + + if pSystem == "Darwin": + print("Show must go on") + if pSystem == "Darwin": + processor = platformProcessor + if not processor: + import platform + if platform.machine() == "x86_64": + processor = "x86-64" + else: + processor = "arm64" + return "osx_%s" % processor.replace("_", "-") distribution, version, flavour = platformTuple distribution = distribution.lower() # If platform.dist does not return something sensible, diff --git a/tests/test_utilities.py b/tests/test_utilities.py index b8254053..9ff6ea44 100644 --- a/tests/test_utilities.py +++ b/tests/test_utilities.py @@ -115,6 +115,7 @@ architecturePayloads = [ ['osx_x86-64', False, [], ('','',''), 'Darwin', 'x86-64'], + ['osx_arm64', False, [], ('','',''), 'Darwin', 'arm64'], ['slc5_x86-64', False, [], ('redhat', '5.XX', 'Boron'), 'Linux', 'x86-64'], ['slc6_x86-64', False, [], ('centos', '6.X', 'Carbon'), 'Linux', 'x86-64'], ['slc7_x86-64', False, [], ('centos', '7.X', 'Ptor'), 'Linux', 'x86-64'], @@ -131,11 +132,23 @@ ['sabayon2_x86-64', True, SABAYON2_OS_RELEASE.split("\n"), ('gentoo', '2.2', ''), 'Linux', 'x86_64'] ] +macOSArchitecturePayloads = [ + ['osx_x86-64', False, [], ('','',''), 'Darwin', 'x86_64'], + ['osx_arm64', False, [], ('','',''), 'Darwin', 'arm64'], +] + class TestUtilities(unittest.TestCase): def test_osx(self): for payload in architecturePayloads: result, hasOsRelease, osReleaseLines, platformTuple, platformSystem, platformProcessor = payload self.assertEqual(result, doDetectArch(hasOsRelease, osReleaseLines, platformTuple, platformSystem, platformProcessor)) + # Test by mocking platform.processor + def test_osx_mock(self): + for payload in macOSArchitecturePayloads: + result, hasOsRelease, osReleaseLines, platformTuple, platformSystem, platformProcessor = payload + with patch('platform.machine', return_value=platformProcessor): + platformProcessor = None + self.assertEqual(result, doDetectArch(hasOsRelease, osReleaseLines, platformTuple, platformSystem, None)) def test_Hasher(self): h = Hasher() h("foo") diff --git a/tox.ini b/tox.ini index 06a35a42..ed710071 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] minversion = 3.20 -envlist = lint, py{27, 36, 37, 38, 39, 310, 311} +envlist = lint, py{27, 36, 37, 38, 39, 310, 311}, darwin [gh-actions] # The "lint" job is run separately. @@ -57,6 +57,8 @@ setenv = # Keep coverage info for later upload, if needed. Files in {envtmpdir} are # deleted after each run. COVERAGE_FILE = {toxworkdir}/coverage.{envname} + ARCHITECTURE = slc7_x86-64 + darwin: ARCHITECTURE = osx_x86-64 changedir = {envtmpdir} commands = @@ -78,10 +80,10 @@ commands = git clone -b O2-v1.3.0 --depth 1 https://github.com/alisw/alidist - coverage run --source={toxinidir} -a {toxinidir}/aliBuild -a slc7_x86-64 -z test-init init zlib + coverage run --source={toxinidir} -a {toxinidir}/aliBuild -a {env:ARCHITECTURE} -z test-init init zlib # This command is expected to fail, but run it for the coverage anyway. # A leading "-" means tox ignores the exit code. - - coverage run --source={toxinidir} -a {toxinidir}/aliBuild build non-existing -a slc7_x86-64 --no-system --disable GCC-Toolchain + - coverage run --source={toxinidir} -a {toxinidir}/aliBuild build non-existing -a {env:ARCHITECTURE} --no-system --disable GCC-Toolchain # TODO: do we need these? This seems to be at least partially covered by # unit tests in tests/tests_parseRecipe.py. @@ -93,19 +95,19 @@ commands = sh -c 'coverage run --source={toxinidir} -a {toxinidir}/aliBuild -c {toxinidir}/tests/testdist build broken6 --force-unknown-architecture --no-system --disable GCC-Toolchain 2>&1 | tee /dev/stderr | grep "while scanning a quoted scalar"' sh -c 'coverage run --source={toxinidir} -a {toxinidir}/aliBuild -c {toxinidir}/tests/testdist build broken7 --force-unknown-architecture --no-system --disable GCC-Toolchain 2>&1 | tee /dev/stderr | grep "Malformed entry prefer_system"' - coverage run --source={toxinidir} -a {toxinidir}/aliBuild build zlib -a slc7_x86-64 --no-system --disable GCC-Toolchain - alienv -a slc7_x86-64 q - alienv -a slc7_x86-64 setenv zlib/latest -c bash -c '[[ $LD_LIBRARY_PATH == */zlib/* ]]' - coverage run --source={toxinidir} -a {toxinidir}/aliBuild -a slc7_x86-64 doctor AliPhysics - coverage run --source={toxinidir} -a {toxinidir}/aliBuild -a slc7_x86-64 build zlib --dry-run - coverage run --source={toxinidir} -a {toxinidir}/aliBuild --aggressive-cleanup --docker -a slc7_x86-64 --always-prefer-system -d build zlib + coverage run --source={toxinidir} -a {toxinidir}/aliBuild build zlib -a {env:ARCHITECTURE} --no-system --disable GCC-Toolchain + alienv -a {env:ARCHITECTURE} q + alienv -a {env:ARCHITECTURE} setenv zlib/latest -c bash -c '[[ $LD_LIBRARY_PATH == */zlib/* ]]' + coverage run --source={toxinidir} -a {toxinidir}/aliBuild -a {env:ARCHITECTURE} doctor AliPhysics + coverage run --source={toxinidir} -a {toxinidir}/aliBuild -a {env:ARCHITECTURE} build zlib --dry-run + py311: coverage run --source={toxinidir} -a {toxinidir}/aliBuild --aggressive-cleanup --docker -a slc7_x86-64 --always-prefer-system -d build zlib # Test for devel packages coverage run --source={toxinidir} -a {toxinidir}/aliBuild init zlib - coverage run --source={toxinidir} -a {toxinidir}/aliBuild --aggressive-cleanup --docker -a slc7_x86-64 --always-prefer-system -d build zlib + py311: coverage run --source={toxinidir} -a {toxinidir}/aliBuild --aggressive-cleanup --docker -a slc7_x86-64 --always-prefer-system -d build zlib # Test that we complain if we have a devel package with an untracked file coverage run --source={toxinidir} -a {toxinidir}/aliBuild init zlib touch zlib/foo - coverage run --source={toxinidir} -a {toxinidir}/aliBuild -a slc7_x86-64 --no-system --disable GCC-Toolchain build zlib + coverage run --source={toxinidir} -a {toxinidir}/aliBuild -a {env:ARCHITECTURE} --no-system --disable GCC-Toolchain build zlib [coverage:run] branch = True From f5af805ff727cfa713b287a864184410e7634370 Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Tue, 17 Oct 2023 14:23:18 +0200 Subject: [PATCH 2/3] Always import platform Long gone are the days this was not available --- alibuild_helpers/utilities.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/alibuild_helpers/utilities.py b/alibuild_helpers/utilities.py index cf9a402c..03076fef 100644 --- a/alibuild_helpers/utilities.py +++ b/alibuild_helpers/utilities.py @@ -7,6 +7,8 @@ import sys import os import re +import platform + from datetime import datetime try: from collections import OrderedDict @@ -141,7 +143,6 @@ def doDetectArch(hasOsRelease, osReleaseLines, platformTuple, pSystem, platformP if pSystem == "Darwin": processor = platformProcessor if not processor: - import platform if platform.machine() == "x86_64": processor = "x86-64" else: @@ -200,7 +201,6 @@ def detectArch(): osReleaseLines = [] hasOsRelease = False try: - import platform if platform.system() == "Darwin": if platform.machine() == "x86_64": return "osx_x86-64" @@ -209,7 +209,7 @@ def detectArch(): except: pass try: - import platform, distro + import distro platformTuple = distro.linux_distribution() platformSystem = platform.system() platformProcessor = platform.processor() From 80c0afdc4753b8d2e1e45e229dcefe003e2b8dc9 Mon Sep 17 00:00:00 2001 From: Giulio Eulisse <10544+ktf@users.noreply.github.com> Date: Tue, 17 Oct 2023 14:55:53 +0200 Subject: [PATCH 3/3] Attempts to publish coverage invo --- tox.ini | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index ed710071..ee2e406f 100644 --- a/tox.ini +++ b/tox.ini @@ -76,8 +76,6 @@ commands = coverage run --source={toxinidir} -a {toxinidir}/aliBuild version - coverage run --source={toxinidir} -a -m unittest discover {toxinidir}/tests - git clone -b O2-v1.3.0 --depth 1 https://github.com/alisw/alidist coverage run --source={toxinidir} -a {toxinidir}/aliBuild -a {env:ARCHITECTURE} -z test-init init zlib @@ -108,6 +106,7 @@ commands = coverage run --source={toxinidir} -a {toxinidir}/aliBuild init zlib touch zlib/foo coverage run --source={toxinidir} -a {toxinidir}/aliBuild -a {env:ARCHITECTURE} --no-system --disable GCC-Toolchain build zlib + coverage run --source={toxinidir} -a -m unittest discover {toxinidir}/tests [coverage:run] branch = True