Merge pull request #37 from ecmwf-ifs/develop

Merge v1.3.0 to main

.github/scripts/bootstrap-nvhpc.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+set -euo pipefail
+set -x
+
+nvhpc_version=21.9
+
+# Use Atlas' nvhpc installation script
+wget https://raw.githubusercontent.com/ecmwf/atlas/develop/tools/install-nvhpc.sh
+chmod +x install-nvhpc.sh
+
+# Install nvhpc
+./install-nvhpc.sh --version $nvhpc_version --prefix "${GITHUB_WORKSPACE}/nvhpc-install" --tmpdir "${RUNNER_TEMP}"
+
+exit 0

.github/scripts/install-hdf5.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+set -euo pipefail
+set -x
+
+hdf5_version=1.10.8
+
+# Choose hdf5
+version_parts=($(echo ${hdf5_version} | tr "." "\n"))
+major_version=${version_parts[0]}.${version_parts[1]}
+temporary_files="${RUNNER_TEMP}/hdf5"
+url=https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-${major_version}/hdf5-${hdf5_version}/src/hdf5-${hdf5_version}.tar.gz
+
+# Download hdf5
+mkdir -p "${temporary_files}"
+curl --location "$url" | tar zx -C "${temporary_files}"
+
+# Build hdf5
+cd "${temporary_files}/hdf5-${hdf5_version}"
+prefix="${GITHUB_WORKSPACE}/hdf5-install"
+mkdir -p "${prefix}"
+./configure --prefix="${prefix}" --enable-shared --enable-fortran --enable-hl
+make -j
+make install
+
+exit 0
+

.github/scripts/run-targets.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-set -euo pipefail
+set -eu
 set -x
 
 # These targets don't have an MPI-parallel driver routine
@@ -8,11 +8,23 @@ non_mpi_targets=(dwarf-P-cloudMicrophysics-IFSScheme dwarf-cloudsc-c)
 # These targets currently cause issues and are therefore not tested
 skipped_targets=(dwarf-cloudsc-gpu-claw)
 
+if [[ "$arch" == *"nvhpc"* ]]
+then
+  # Skip GPU targets if built with nvhpc (don't have GPU in test runner)
+  skipped_targets+=(dwarf-cloudsc-gpu-scc dwarf-cloudsc-gpu-scc-hoist dwarf-cloudsc-gpu-omp-scc-hoist)
+
+  # Skip GPU targets from Loki if built with nvhpc (don't have GPU in test runner)
+  skipped_targets+=(dwarf-cloudsc-loki-claw-gpu dwarf-cloudsc-loki-scc dwarf-cloudsc-loki-scc-hoist)
+
+  # Skip C target if built with nvhpc, segfaults for unknown reasons
+  skipped_targets+=(dwarf-cloudsc-c)
+fi
+
 exit_code=0
 cd build
 
 #
-# Run each of the binaries with default NPROMA and validate exit codes
+# Run each of the binaries with a safe NPROMA value and validate exit codes
 #
 
 for target in $(ls bin)
@@ -25,11 +37,12 @@ do
 
   if [[ "$mpi_flag" == "--with-mpi" && ! " ${non_mpi_targets[*]} " =~ " $target " ]]
   then
-    # Two ranks with one thread each, default NPROMA
-    mpirun -np 2 bin/$target 1 100
+    # Two ranks with one thread each, safe NPROMA
+    # NB: Use oversubscribe to run, even if we end up on a single core agent
+    mpirun --oversubscribe -np 2 bin/$target 1 100 64
   else
-    # Single thread, default NPROMA
-    bin/$target 1 100
+    # Single thread, safe NPROMA
+    bin/$target 1 100 64
   fi
   exit_code=$((exit_code + $?))
 done

.github/scripts/verify-targets.sh

@@ -10,9 +10,32 @@ exit_code=0
 
 targets=(dwarf-P-cloudMicrophysics-IFSScheme dwarf-cloudsc-fortran)
 
+if [[ "$io_library_flag" == "--with-serialbox" ]]
+then
+  targets+=(dwarf-cloudsc-c)
+fi
+
 if [[ "$gpu_flag" == "--with-gpu" ]]
 then
-  targets+=(dwarf-cloudsc-gpu-claw dwarf-cloudsc-gpu-scc dwarf-cloudsc-gpu-scc-hoist)
+  targets+=(dwarf-cloudsc-gpu-scc dwarf-cloudsc-gpu-scc-hoist dwarf-cloudsc-gpu-omp-scc-hoist)
+  if [[ "$claw_flag" == "--with-claw" ]]
+  then
+    targets+=(dwarf-cloudsc-gpu-claw)
+  fi
+fi
+
+if [[ "$loki_flag" == "--with-loki" ]]
+then
+  targets+=(dwarf-cloudsc-loki-idem dwarf-cloudsc-loki-sca)
+  targets+=(dwarf-cloudsc-loki-scc dwarf-cloudsc-loki-scc-hoist)
+  if [[ "$prec_flag" != "--single-precision" ]]
+  then
+    targets+=(dwarf-cloudsc-loki-c)
+  fi
+  if [[ "$claw_flag" == "--with-claw" ]]
+  then
+    targets+=(dwarf-cloudsc-loki-claw-cpu dwarf-cloudsc-loki-claw-gpu)
+  fi
 fi
 
 #

.github/workflows/build.yml

@@ -4,7 +4,7 @@ name: build
 on:
   # Triggers the workflow on push events
   push:
-    branches: [ '**' ]
+    branches: [ 'main', 'develop' ]
     tags-ignore: [ '**' ]
 
   # Triggers the workflow on pull request events
@@ -17,61 +17,105 @@ on:
 jobs:
   # This workflow contains a single job called "build"
   build:
-    name: Test on ${{ matrix.arch }} ${{ matrix.io_library_flag }} ${{ matrix.mpi_flag }} ${{ matrix.prec_flag }} ${{ matrix.gpu_flag }}
-    
+    name: Test on ${{ matrix.arch }} ${{ matrix.io_library_flag }} ${{ matrix.mpi_flag }} ${{ matrix.prec_flag }} ${{ matrix.gpu_flag }} ${{ matrix.loki_flag }} ${{ matrix.claw_flag }}
+
     # The type of runner that the job will run on
-    runs-on: ubuntu-latest
-  
+    runs-on: ubuntu-20.04
+
     strategy:
       fail-fast: false  # false: try to complete all jobs
-      
+
       matrix:
-      
+
         arch:
-          - github/ubuntu/gnu/9.3.0
-
-        io_library_flag: ['']  # Switch between Serialbox and HDF5
-        # FIXME: serialbox builds are currently disabled until a compatible serialbox version is available to Github actions
-
+          - github/ubuntu/gnu/9.4.0
+
+        io_library_flag: ['', '--with-serialbox']  # Switch between Serialbox and HDF5
+
         mpi_flag: ['', '--with-mpi']  # Enable MPI-parallel build
 
         prec_flag: ['', '--single-precision']  # Switch single/double precision
 
         gpu_flag: ['', '--with-gpu']  # GPU-variants enabled
 
+        loki_flag: ['', '--with-loki']  # Loki source-to-source translation enabled
+
+        claw_flag: ['']  # Flag to enable CLAW-generated variants
+
+        include:
+          # Add nvhpc build configurations with serialbox and HDF5
+          - arch: github/ubuntu/nvhpc/21.9
+            io_library_flag: ''
+            mpi_flag: ''
+            prec_flag: ''
+            gpu_flag: '--with-gpu'
+          - arch: github/ubuntu/nvhpc/21.9
+            io_library_flag: '--with-serialbox'
+            mpi_flag: ''
+            prec_flag: ''
+            gpu_flag: '--with-gpu'
+
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
-      # Sets-up environment and installs required packages
-      - name: Environment setup
+      # Installs required packages
+      - name: Package installation
         run: |
-          [[ "${{ matrix.mpi_flag }}" == "--with-mpi" ]] && sudo apt install libopenmpi-dev || true
-          [[ "${{ matrix.io_library_flag }}" != "--with-serialbox" ]] && sudo apt install libhdf5-dev || true
-      
+          sudo apt-get install libc-dev-bin gfortran-9 gcc-9 g++-9
+
+      # Install MPI
+      - name: Install MPI via Apt
+        if: contains( matrix.mpi_flag, 'with-mpi' )
+        run: sudo apt-get install libopenmpi-dev
+
+      # Install Compiler
+      - name: Install nvhpc
+        if: contains( matrix.arch, 'nvhpc' )
+        run: .github/scripts/bootstrap-nvhpc.sh
+
+      # Install HDF5
+      - name: Install HDF5 via Apt
+        if: ${{ ! contains( matrix.arch, 'nvhpc' ) && ! contains( matrix.io_library_flag, 'with-serialbox' ) }}
+        run: sudo apt-get install libhdf5-dev
+
+      - name: Install HDF5 from source
+        if: contains( matrix.arch, 'nvhpc' ) && ! contains( matrix.io_library_flag, 'with-serialbox' )
+        run: source arch/${{ matrix.arch }}/env.sh && .github/scripts/install-hdf5.sh
+
+      # Install Boost
+      - name: Install Boost libraries
+        if: contains( matrix.io_library_flag, 'with-serialbox' )
+        run: sudo apt install libboost-filesystem-dev libboost-system-dev
+
       # Check-out dependencies as part of the bundle creation
       - name: Bundle create
         run: ./cloudsc-bundle create
-        
+
       # Build the targets
       - name: Bundle build
         run: |
           ./cloudsc-bundle build --retry-verbose \
           --arch=arch/${{ matrix.arch }} ${{ matrix.prec_flag }} \
-          ${{ matrix.mpi_flag }} ${{ matrix.io_library_flag }} ${{ matrix.gpu_flag }}
+          ${{ matrix.mpi_flag }} ${{ matrix.io_library_flag }} ${{ matrix.gpu_flag }} \
+          ${{ matrix.claw_flag}} ${{ matrix.loki_flag }}
 
       # Verify targets exist
       - name: Verify targets
         env:
           io_library_flag: ${{ matrix.io_library_flag }}
+          prec_flag: ${{ matrix.prec_flag }}
           gpu_flag: ${{ matrix.gpu_flag }}
+          loki_flag: ${{ matrix.loki_flag }}
+          claw_flag: ${{ matrix.claw_flag }}
         run: .github/scripts/verify-targets.sh
 
       # Run double-precision targets
       # (Mind the exclusions inside the script!)
       - name: Run targets
         env:
           mpi_flag: ${{ matrix.mpi_flag }}
+          arch: ${{ matrix.arch }}
         if: ${{ matrix.prec_flag == '' }}
         run: .github/scripts/run-targets.sh

.gitignore

@@ -1,7 +1,19 @@
 *~
+.vscode
 ecbundle
 ecbundle/*
 build/*
 source/*
 serialbox2hdf5/venv
 serialbox2hdf5/serialbox
+benchmark/venv
+benchmark/rundir_*
+benchmark/jube-debug.log
+venv/
+__pycache__/
+.DS_Store
+.dace.conf
+.gt_cache/
+.idea/
+.python-version
+*.egg-info/

AUTHORS.md

@@ -9,13 +9,16 @@
 - M. Hamrud (ECMWF)
 - M. Koehler (ECMWF)
 - M. Lange (ECMWF)
+- L. Lucido (Atos)
 - O. Marsden (ECMWF)
 - G. Mengaldo (ECMWF)
 - G. Mozdzynski (ECMWF)
+- Z. Piotrowski (ECMWF)
 - B. Reuter (ECMWF)
 - D. Salmond (ECMWF)
 - M. Tiedtke (ECMWF)
 - A. Tompkins (ECMWF)
+- S. Ubbiali (ETH Zuerich)
 - F. Vana (ECMWF)
 
 If you have contributed to this project,