Merge branch '5.0.x' into single-pip-check-5.x

.github/workflows/unit_tests.yml

@@ -14,7 +14,7 @@ jobs:
     strategy:
       matrix:
         python: [3.6, 3.7, 3.8, 3.9, '3.10', '3.11', '3.12']
-        modules_tool: [Lmod-8.1.14, modules-tcl-1.147, modules-3.2.10, modules-4.1.4]
+        modules_tool: [Lmod-8.1.14, modules-tcl-1.147, modules-3.2.10, modules-4.5.3]
         module_syntax: [Lua, Tcl]
         # exclude some configuration for non-Lmod modules tool:
         # - don't test with Lua module syntax (only supported in Lmod)
@@ -24,7 +24,7 @@ jobs:
             module_syntax: Lua
           - modules_tool: modules-3.2.10
             module_syntax: Lua
-          - modules_tool: modules-4.1.4
+          - modules_tool: modules-4.5.3
             module_syntax: Lua
           - modules_tool: modules-tcl-1.147
             python: 3.7
@@ -50,17 +50,17 @@ jobs:
             python: '3.11'
           - modules_tool: modules-3.2.10
             python: '3.12'
-          - modules_tool: modules-4.1.4
+          - modules_tool: modules-4.5.3
             python: 3.7
-          - modules_tool: modules-4.1.4
+          - modules_tool: modules-4.5.3
             python: 3.8
-          - modules_tool: modules-4.1.4
+          - modules_tool: modules-4.5.3
             python: 3.9
-          - modules_tool: modules-4.1.4
+          - modules_tool: modules-4.5.3
             python: '3.10'
-          - modules_tool: modules-4.1.4
+          - modules_tool: modules-4.5.3
             python: '3.11'
-          - modules_tool: modules-4.1.4
+          - modules_tool: modules-4.5.3
             python: '3.12'
       fail-fast: false
     steps:

RELEASE_NOTES

@@ -6,6 +6,63 @@ These release notes can also be consulted at http://easybuild.readthedocs.org/en
 The latest version of easybuild-easyblocks provides 259 software-specific easyblocks and 43 generic easyblocks.
 
 
+v4.9.3 (14 September 2024)
+--------------------------
+
+update/bugfix release
+- minor updates, including:
+  - update custom easyblock for Tensorflow for versions 2.14 + 2.15 (#3303)
+  - add support for versions >= 2024a to MCR easyblock (#3369)
+  - update custom easyblock for Bazel for versions >= 7.0 (#3370)
+  - don't consider lib/release for impi >= 2021.11 (#3375)
+  - update ORCA easyblock for version 6 (#3395)
+  - update custom easyblock for psmpi to support the renaming of a couple of options in 5.10.0-1 (#3420)
+  - update sanity check in OpenFOAM easyblock, since there's no `modifyMesh` in OpenFOAM 12 (#3434)
+  - fix sanity check for Geant4 >= v11.2 (#3439)
+- minor enhancements, including:
+  - enhance psmpi easyblock to activate CUDA support when it is a dependency + make check for static libs in MPICH easyblock optional (#2787)
+  - make a project environment and manage LOAD_PATH for JuliaPackage (#3239)
+  - set build type for PyTorch explicitely (#3332)
+  - add `sanity_check_test_inputs` custom easyconfig parameter, mapping for zen4, and support building of non-stable versions to LAMMPS easyblock (#3336)
+  - add RISCV64 support and update config options of new versions of Extrae (#3339)
+  - set `$ESMFMKFILE` environment variable in environment module for ESMF (#3368)
+  - use build dir for big files/folders while building TensorFlow (#3371)
+  - update `PythonPackage` easyblock to allow installation of Python packages with `$PIP_REQUIRE_VIRTUALENV` set + move temporary pip folder into build dir (#3374)
+  - add RISC-V support to Boost easyblock (#3376)
+  - add support for generating `.gem` files from `.gemspec` files and support for `preinstallopts` in the `RubyGem` easyblock (#3381)
+  - simplify install step in custom easyblock for `Tkinter` (#3382)
+  - add MSA and specially PMIx support for ParaStationMPI (#3383)
+  - update custom easyblock for PETSc to consider `include/suitesparse` subdirectory for SuiteSparse headers (#3391)
+  - enhance custom easyblock for GCC to use `with-arch` option for nvptx with 13.1+ (#3396)
+  - support revisions in crates extraction of cargo packages (#3405)
+  - set `$R_LIBS_USER` in `RPackage` easyblock to avoid picking up on R packages installed in home directory (#3407)
+  - update custom easyblock for QuantumESPRESSO to be aware of MPI tests being disabled in EasyBuild configuration (#3412)
+  - disable the version check of pip in Python packages by defining `$PIP_DISABLE_PIP_VERSION_CHECK` (#3427)
+  - make sure user packages are not used in sanity check of PythonBundle (#3435)
+  - enhance OpenFOAM easyblock to also build the plugins for OpenFOAM >= v2406 (#3436)
+  - enhance binutils easyblock to explicitely pass msgpack configure option (#3438)
+- various bug fixes, including:
+  - patch `ctypes` for Python installations when filtering `LD_LIBRARY_PATH` + fix path to `ldconfig` when using alternate sysroot (#3352)
+  - also consider 'normalised' package name with underscore rather than dash in EasyBuild easyblock (#3358)
+  - use PRRTE MCA environment variable for oversubscription in OpenMPI easyblock (#3360)
+  - fix import of `parse` in `openssl_wrapper.py` for Python 2.7 (#3364)
+  - make sure that OpenFOAM's wmake can find MPFR and GMP if CGAL >= 5.0 (header-only) is used (#3366)
+  - remove dummy license server info from ANSYS (#3377)
+  - explicitly call `csh` in custom easyblock for WPS (#3384)
+  - use relative paths to object files when compiling shared libraries in the OpenFOAM easyblock (#3388)
+  - change extra `$PATH` entry for OCaml to use `opam/default` rather than `opam/system` (#3390)
+  - force `--without-unwind` for ARM and add `--with-libz` option for all architectures in custom easyblock for Extrae (#3393)
+  - fix `--sanity-check-only` for impi (#3403)
+  - fix crash in Cargo easyblock when no crates are specified (#3404)
+  - fix double initialization of `Cargo` by `CargoPythonPackage` by removing incorrect custom `__init__` implementation + fix use of `super()` in PALM easyblock (since that doesn't work with Python 2.7) (#3406)
+  - fix typo in log message in `PythonPackage` easyblock (#3408)
+  - make sure to raise an error if `pick_python_cmd` returns `False` for Python bundles/packages (#3430)
+  - don't wipe build environment before building opam in install step of OCaml easyblock (#3443)
+- other changes:
+  - merge of the `ConfigureMake` and `CMakeMake` versions of the easyblock for QuantumESPRESSO (#3338)
+  - remove redundant backslashes in GEANT easyblock (#3394)
+
+
 v4.9.2 (12 June 2024)
 ---------------------
 

easybuild/easyblocks/b/binutils.py

@@ -148,6 +148,15 @@ def configure_step(self):
                 else:
                     libs.append(libz_path)
 
+        msgpackroot = get_software_root('msgpack-c')
+        if LooseVersion(self.version) >= LooseVersion('2.39'):
+            if msgpackroot:
+                self.cfg.update('configopts', '--with-msgpack')
+            else:
+                self.cfg.update('configopts', '--without-msgpack')
+        elif msgpackroot:
+            raise EasyBuildError('msgpack is only supported since binutils 2.39. Remove the dependency!')
+
         env.setvar('LIBS', ' '.join(libs))
 
         # explicitly configure binutils to use / as sysroot

easybuild/easyblocks/g/gcc.py

@@ -367,6 +367,70 @@ def prep_extra_src_dirs(self, stage, target_prefix=None):
             'versions': versions
         }
 
+    def map_nvptx_capability(self):
+        """
+        Convert PTX ISA architecture passed via EasyBuild configs to a version which is understood by GCC.
+        Valid architecture strings include 'sm_30', 'sm_35', 'sm_53', 'sm_70', 'sm_75', 'sm_80'.
+        (as of GCC 14.1.0). Some additional architectures may be mapped.
+        See: https://github.com/gcc-mirror/gcc/commit/de0ef04419e90eacf0d1ddb265552a1b08c18d4b
+
+        As this list is updated regularly, try to parse the GCC source file (gcc/config/nvptx/nvptx.opt)
+        and extract the supported architectures and their mapping. Based on the result, determine the lowest
+        architecture required to support all 'cuda_compute_capabilities' and return this value.
+        """
+        cuda_cc_list = build_option('cuda_compute_capabilities') or self.cfg['cuda_compute_capabilities']
+        architecture_mappings_file = os.path.join(self.cfg['start_dir'], 'gcc', 'config', 'nvptx', 'nvptx.opt')
+        architecture_mappings_flag = "march-map="
+        architecture_mappings_replacement = "misa=,"
+
+        # Determine which compute capabilities are configured. If there are none, return immediately.
+        if cuda_cc_list is None:
+            return None
+        cuda_sm_list = [f"sm_{cc.replace('.', '')}" for cc in cuda_cc_list]
+
+        if not os.path.exists(architecture_mappings_file):
+            warn_msg = f"Tried to parse nvptx.opt but file {architecture_mappings_file} was not found. " \
+                       "Please check the path and update the EasyBlock if necessary!"
+            self.log.warning(warn_msg)
+            return None
+
+        # We want to read the mappings found in the GCC sources and create a map for this.
+        # We're searching for the following pattern:
+        # march-map=sm_32
+        # Target RejectNegative Alias(misa=,sm_30)
+        gcc_architecture_mappings = {}
+        file_content = read_file(architecture_mappings_file).splitlines()
+        for line_idx, line in enumerate(file_content):
+            line = line.strip()
+            if line.startswith(architecture_mappings_flag):
+                key = line.split('=')[1]
+                # Mapped architecture can be found in the following line
+                line = file_content[line_idx + 1]
+                if architecture_mappings_replacement not in line:
+                    warn_msg = "Tried to parse nvptx.opt but failed to extract mapped architectures! " \
+                                f"Expected to find substring '{architecture_mappings_replacement}' in " \
+                                f"line {line_idx + 1} but found '{line}'. Choosing default of GCC."
+                    self.log.warning(warn_msg)
+                    # Bail out, since results of mapping cannot be trusted
+                    return None
+                value = line.split(architecture_mappings_replacement)[1].rstrip(')')
+                gcc_architecture_mappings[key] = value
+        self.log.info(f"Available architecture mappings in GCC {self.version}:\n {str(gcc_architecture_mappings)}")
+
+        # Map compute capabilities to GCC ones
+        # If no compute capability can be mapped, stick to default of GCC and return None
+        gcc_cc = [gcc_architecture_mappings[cc] if cc in gcc_architecture_mappings else None for cc in cuda_sm_list]
+        self.log.info(f"Mapped architectures: {str(cuda_sm_list)} -> {str(gcc_cc)}")
+        if any(cc is None for cc in gcc_cc):
+            self.log.info("At least one architecture could not be mapped. Choosing default of GCC.")
+            return None
+
+        # Get the lowest architecture mapping and return it
+        sorted_gcc_cc = sorted(gcc_cc)
+        self.log.info("Choosing first architecture in sorted list as default nvptx "
+                      f"architecture: {str(sorted_gcc_cc)}")
+        return sorted_gcc_cc[0]
+
     def run_configure_cmd(self, cmd):
         """
         Run a configure command, with some extra checking (e.g. for unrecognized options).
@@ -545,6 +609,10 @@ def configure_step(self):
                         self.create_dir("build-nvptx-gcc")
                         target = 'nvptx-none'
                         self.cfg.update('configopts', "--enable-newlib-io-long-long")
+                        if LooseVersion(self.version) >= LooseVersion('13.1.0'):
+                            cuda_cc = self.map_nvptx_capability()
+                            if cuda_cc:
+                                self.cfg.update('configopts', f'--with-arch={cuda_cc}')
                     else:
                         # compile AMD GCN target compiler
                         self.create_dir("build-amdgcn-gcc")

easybuild/easyblocks/g/geant4.py

@@ -34,8 +34,9 @@
 
 import os
 
-from easybuild.framework.easyconfig import CUSTOM
 from easybuild.easyblocks.generic.cmakemake import CMakeMake
+from easybuild.framework.easyconfig import CUSTOM
+from easybuild.tools import LooseVersion
 
 
 class EB_Geant4(CMakeMake):
@@ -100,8 +101,16 @@ def sanity_check_step(self):
         Custom sanity check for Geant4
         """
         bin_files = ["bin/geant4-config", "bin/geant4.sh", "bin/geant4.csh"]
-        lib_files = ["lib64/libG4%s.so" % x for x in ['analysis', 'event', 'GMocren', 'materials',
-                                                      'persistency', 'readout', 'Tree', 'VRML']]
+        libs = ['analysis', 'event', 'GMocren', 'materials', 'readout', 'Tree', 'VRML']
+
+        # G4Persistency library was split up in Geant v11.2,
+        # see https://geant4.web.cern.ch/download/release-notes/notes-v11.2.0.html
+        if LooseVersion(self.version) >= LooseVersion('11.2'):
+            libs.extend(['gdml', 'geomtext', 'mctruth', 'geomtext'])
+        else:
+            libs.append('persistency')
+
+        lib_files = ["lib64/libG4%s.so" % x for x in libs]
         include_dir = 'include/Geant4'
 
         custom_paths = {

easybuild/easyblocks/generic/bundle.py

@@ -84,6 +84,10 @@ def __init__(self, *args, **kwargs):
             if self.cfg['patches']:
                 raise EasyBuildError("List of patches for bundle itself must be empty, found %s", self.cfg['patches'])
 
+        # copy EasyConfig instance before we make changes to it
+        # (like adding component sources to top-level sources easyconfig parameter)
+        self.cfg = self.cfg.copy()
+
         # disable templating to avoid premature resolving of template values
         self.cfg.enable_templating = False
 

easybuild/easyblocks/generic/cargo.py

@@ -131,34 +131,35 @@ def __init__(self, *args, **kwargs):
         env.setvar('RUST_LOG', 'DEBUG')
         env.setvar('RUST_BACKTRACE', '1')
 
-        # Populate sources from "crates" list of tuples (only once)
-        if self.cfg['crates']:
-            # Move 'crates' list from easyconfig parameter to property,
-            # to avoid that creates are processed into 'sources' easyconfig parameter again
-            # when easyblock is initialized again using the same parsed easyconfig
-            # (for example when check_sha256_checksums function is called, like in easyconfigs test suite)
-            self.crates = self.cfg['crates']
-            self.cfg['crates'] = []
-            sources = []
-            for crate_info in self.crates:
-                if len(crate_info) == 2:
-                    sources.append({
-                        'download_filename': self.crate_download_filename(*crate_info),
-                        'filename': self.crate_src_filename(*crate_info),
-                        'source_urls': [CRATESIO_SOURCE],
-                        'alt_location': 'crates.io',
-                    })
-                else:
-                    crate, version, repo, rev = crate_info
-                    url, repo_name = repo.rsplit('/', maxsplit=1)
-                    if repo_name.endswith('.git'):
-                        repo_name = repo_name[:-4]
-                    sources.append({
-                        'git_config': {'url': url, 'repo_name': repo_name, 'commit': rev},
-                        'filename': self.crate_src_filename(crate, version),
-                    })
+        # Populate sources from "crates" list of tuples
+        sources = []
+        for crate_info in self.crates:
+            if len(crate_info) == 2:
+                sources.append({
+                    'download_filename': self.crate_download_filename(*crate_info),
+                    'filename': self.crate_src_filename(*crate_info),
+                    'source_urls': [CRATESIO_SOURCE],
+                    'alt_location': 'crates.io',
+                })
+            else:
+                crate, version, repo, rev = crate_info
+                url, repo_name = repo.rsplit('/', maxsplit=1)
+                if repo_name.endswith('.git'):
+                    repo_name = repo_name[:-4]
+                sources.append({
+                    'git_config': {'url': url, 'repo_name': repo_name, 'commit': rev},
+                    'filename': self.crate_src_filename(crate, version),
+                })
+
+        # copy EasyConfig instance before we make changes to it
+        self.cfg = self.cfg.copy()
 
-            self.cfg.update('sources', sources)
+        self.cfg.update('sources', sources)
+
+    @property
+    def crates(self):
+        """Return the crates as defined in the EasyConfig"""
+        return self.cfg['crates']
 
     def extract_step(self):
         """
@@ -314,13 +315,14 @@ def generate_crate_list(sourcedir):
                 if dep['source'] == 'registry+https://github.com/rust-lang/crates.io-index':
                     crates.append((name, version))
                 else:
-                    # Lock file has revision#revision in the url
+                    # Lock file has #revision in the url
                     url, rev = dep['source'].rsplit('#', maxsplit=1)
                     for prefix in ('registry+', 'git+'):
                         if url.startswith(prefix):
                             url = url[len(prefix):]
-                    # Remove branch name if present
+                    # Remove branch name and revision URL parameters if present
                     url = re.sub(r'\?branch=\w+$', '', url)
+                    url = re.sub(r'\?rev=%s+$' % rev, '', url)
                     crates.append((name, version, url, rev))
         else:
             other_crates.append((name, version))

easybuild/easyblocks/generic/cargopythonpackage.py

@@ -47,11 +47,6 @@ def extra_options(extra_vars=None):
 
         return extra_vars
 
-    def __init__(self, *args, **kwargs):
-        """Constructor for CargoPythonPackage easyblock."""
-        Cargo.__init__(self, *args, **kwargs)
-        PythonPackage.__init__(self, *args, **kwargs)
-
     def extract_step(self):
         """Specifically use the overloaded variant from Cargo as is populates vendored sources with checksums."""
         return Cargo.extract_step(self)

easybuild/easyblocks/generic/juliabundle.py

@@ -94,6 +94,6 @@ def sanity_check_step(self, *args, **kwargs):
         super(JuliaBundle, self).sanity_check_step(custom_paths=custom_paths)
 
     def make_module_extra(self, *args, **kwargs):
-        """Custom module environement from JuliaPackage"""
+        """Custom module environment from JuliaPackage"""
         mod = super(JuliaBundle, self).make_module_extra(*args, **kwargs)
         return mod

easybuild/easyblocks/generic/juliapackage.py

@@ -66,7 +66,7 @@ class JuliaPackage(ExtensionEasyBlock):
     """
     Builds and installs Julia Packages.
 
-    Julia environement setup during installation:
+    Julia environment setup during installation:
         - initialize new Julia environment in 'environments' subdir in installation directory
         - remove paths in user depot '~/.julia' from DEPOT_PATH and LOAD_PATH
         - put installation directory as top DEPOT_PATH, the target depot for installations with Pkg

easybuild/easyblocks/generic/pythonbundle.py

@@ -134,6 +134,18 @@ def make_module_extra(self, *args, **kwargs):
 
         return txt
 
+    def load_module(self, *args, **kwargs):
+        """
+        Make sure that $PYTHONNOUSERSITE is defined after loading module file for this software."""
+
+        super(PythonBundle, self).load_module(*args, **kwargs)
+
+        # Don't add user site directory to sys.path (equivalent to python -s),
+        # to avoid that any Python packages installed in $HOME/.local/lib affect the sanity check.
+        # Required here to ensure that it is defined for sanity check commands of the bundle
+        # because the environment is reset to the initial environment right before loading the module
+        env.setvar('PYTHONNOUSERSITE', '1', verbose=False)
+
     def sanity_check_step(self, *args, **kwargs):
         """Custom sanity check for bundle of Python package."""