Merge pull request #72 from yoctoyotta1024/installyac

Include yac enable/disable in helper bash scripts for building CLEO
yoctoyotta1024 · Jun 6, 2024 · 0923f3a · 0923f3a
2 parents 8310f70 + a1c9a31
commit 0923f3a
Show file tree

Hide file tree

Showing 27 changed files with 344 additions and 77 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -25,13 +25,13 @@ All notable changes to this project will be documented in this file. See [conven
 #### Miscellaneous Chores
 - correct typo in comment - ([85f8072](https://github.com/yoctoyotta1024/CLEO/commit/85f8072041452b12087686dd1d13b7e4939bf79e)) - clara.bayley
 - correct comment and formatting - ([b46a0fc](https://github.com/yoctoyotta1024/CLEO/commit/b46a0fc137d3a975a4af21c925d3eec6a5aa09f8)) - clara.bayley
-- delete todos - ([03cfd09](https://github.com/yoctoyotta1024/CLEO/commit/03cfd099097f8e8aff153868eb101ca0f8af58b9)) - clara.bayley
+- delete excess notes - ([03cfd09](https://github.com/yoctoyotta1024/CLEO/commit/03cfd099097f8e8aff153868eb101ca0f8af58b9)) - clara.bayley
 - rename const tstep observer - ([b40e55b](https://github.com/yoctoyotta1024/CLEO/commit/b40e55b7f0e3d65ddb0956495936234c81666d28)) - clara.bayley
 - rename - ([4c289ee](https://github.com/yoctoyotta1024/CLEO/commit/4c289eefab5b2b2d994399c615b0867abecdae52)) - clara.bayley
 - rename - ([a0a21d1](https://github.com/yoctoyotta1024/CLEO/commit/a0a21d14978987a3dfbaf4fd71587e5bd6e09af3)) - clara.bayley
 - rename file - ([d159b90](https://github.com/yoctoyotta1024/CLEO/commit/d159b90888d56a80fbce53a52215d8eb429e9d41)) - clara.bayley
 #### Refactoring
-- WIP to test monitor microphysics in condensation - ([b65bacd](https://github.com/yoctoyotta1024/CLEO/commit/b65bacdce397e7a9436d3c4b9fbe0f4b51a6879e)) - clara.bayley
+- work to test monitor microphysics in condensation - ([b65bacd](https://github.com/yoctoyotta1024/CLEO/commit/b65bacdce397e7a9436d3c4b9fbe0f4b51a6879e)) - clara.bayley
 - revert to commit e3041238475c3d4ac848ce8bb34b6f9f95ddbf07 and then move sdmmonitor to superdrops library - ([9336266](https://github.com/yoctoyotta1024/CLEO/commit/933626698b94d875b3bdd8bfeade888e7d3770f4)) - clara.bayley
 - Revert "refactor: attempt to include monitor in condensation" - ([147cc9d](https://github.com/yoctoyotta1024/CLEO/commit/147cc9d5bc1a78a28d31876737c447933db39266)) - clara.bayley
 - attempt to include monitor in condensation - ([7ac5712](https://github.com/yoctoyotta1024/CLEO/commit/7ac5712e3a70fc8184fd1e54b73a110b62eb69a6)) - clara.bayley

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1,6 +1,6 @@
 # run cmake with " cmake -S [source dir of targerts] -B [build dir for makefile] "
 # e.g. could run cmake with these compiler options:
-# cmake -S ./ -B ./build -DCMAKE_CXX_COMPILER=g++ -DCMAKE_CC_COMPILER=gcc -DKokkos_ARCH_NATIVE=ON -DKokkos_ENABLE_SERIAL=ON -DKokkos_ENABLE_OPENMP=ON
+# cmake -S ./ -B ./build -DCMAKE_CXX_COMPILER=g++ -DCMAKE_C_COMPILER=gcc -DKokkos_ARCH_NATIVE=ON -DKokkos_ENABLE_SERIAL=ON -DKokkos_ENABLE_OPENMP=ON
 
 # set cmake version
 cmake_minimum_required(VERSION 3.18.0)

diff --git a/docs/source/usage/examples.rst b/docs/source/usage/examples.rst
@@ -42,6 +42,10 @@ You will need to configure ```examples/run_example.sh``` in the following ways:
 
   replace the path in the line stating ``python=[...]`` with the path to your Python interpreter.
 
+* Set the path to your YAC and YAXT installations
+
+  replace ``yacyaxtroot=[...]`` with the path to the directory containing your yac and yaxt
+  directories, or to ``yacyaxtroot=""`` if you do not intend to run an example that requires YAC.
 
 You can optionally configure the bash script specific to each example in the following ways:
 

diff --git a/docs/source/usage/extern.rst b/docs/source/usage/extern.rst
@@ -1,31 +1,50 @@
 External Libraries
 ==================
 
-CLEO depends upon Kokkos and may depend upon some additional external libraries such as YAC and
-``yaml-cpp`` depending on your setup. These are automatically built using CMAKE and compiled if
-required.
-
-.. note::
-  The installation of YAC for CLEO is currently in development and may require some manual installation.
+CLEO depends upon Kokkos and may depend upon some additional external libraries such as ``YAC`` and
+``yaml-cpp`` depending on your setup.
 
 Kokkos
 ------
 All builds of CLEO require Kokkos in order to implement thread parallelism. You can read more about
 how we use Kokkos on :doc:`our page about Kokkos<../intro/kokkos>`.
 
+The Kokkos libaries for CLEO are automatically built using CMAKE and compiled if required.
+
+.. _extern_yac:
+
 YAC
 ---
 YAC is required if CLEO couples to dynamics using YAC and/or uses MPI domain decompoisiton. You can
 find more information about it from `its documentation: <https://dkrz-sw.gitlab-pages.dkrz.de/yac>`_.
 
-TODO(all): Detail how to install YAC for CLEO.
+To build CLEO with dependency on YAC, you will first need to install YAXT and YAC manually.
+(YAXT is a dependency of YAC.)
+
+.. note::
+  The installation of YAC for CLEO is currently in active development and so may not be exactly as written here.
+
+How to Install YAC (and YAXT)
+#############################
+
+The easiest way to install YAXT and YAC is to run the ``install_yac.sh`` bash script found in
+``scripts/bash/``. Note you will need to provide the path to the directory where you want
+to put the installations.
+
+Alternatively you can download `YAXT <https://swprojects.dkrz.de/redmine/>`_ and
+`YAC <https://gitlab.dkrz.de/dkrz-sw/yac/>`_ as compressed files and then configure and compile
+them yourself.
 
 yaml-cpp
 --------
 CLEO's ``initialise`` library depends on the ```yaml-cpp``` package to read and write YAML files. You
 can find more information about it from `its repository: <https://github.com/jbeder/yaml-cpp>`_.
 
+The yaml-cpp library for CLEO is automatically built using CMAKE and compiled if required.
+
 CVODE
 -----
 CLEO's ``coupldyn_cvode`` library requires the SUNDIALS CVODE package. You can find more information
 about it from `its webpage: <https://computing.llnl.gov/projects/sundials/cvode>`_.
+
+The CVODE libraries for CLEO are automatically built using CMAKE and compiled if required.
diff --git a/docs/source/usage/requirements.rst b/docs/source/usage/requirements.rst
@@ -63,16 +63,24 @@ You can install Python packages to an existing Conda (or Mamba) environment via:
 YAC
 ---
 
-.. note::
-  The installation of YAC for CLEO is currently in development and may require some manual installation.
-
 YAC is one of the :doc:`external libraries<extern>` which CLEO may require in order to
 couple to dynamics and/or have MPI domain decomposition.
 
-YAC requires some additional MPI, NetCDF and yaml libraries alongside the gcc compiler. You can
-load them on Levante via:
+Note that YAC (and its YAXT dependency) need to be installed manually before you can build
+CLEO with them. You can find instructions on how to do install YAC (and YAXT) :ref:`here <_extern_yac>`.
+
+YAC also requires some additional MPI, NetCDF and yaml libraries alongside the compatible gcc
+compiler which you can load on Levante via:
+
+.. code-block:: console
+
+  $ module load gcc/11.2.0-gcc-11.2.0 openmpi/4.1.2-gcc-11.2.0 netcdf-c/4.8.1-openmpi-4.1.2-gcc-11.2.0
+  $ spack load openblas@0.3.18%gcc@=11.2.0
+
+When you want to run CLEO with YAC, you will also need to export some additional paths:
 
 .. code-block:: console
 
-  $ module load openmpi/4.1.2-gcc-11.2.0 netcdf-c/4.8.1-openmpi-4.1.2-gcc-11.2.0
-  $ spack load openblas@0.3.18%gcc@=11.2.0 libfyaml /fvbhgoq py-mpi4py
+  $ export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/sw/spack-levante/libfyaml-0.7.12-fvbhgo/lib
+  $ export PYTHONPATH=/your/path/to/yac/python/
+  $ spack load py-numpy
diff --git a/examples/adiabaticparcel/as2017.sh b/examples/adiabaticparcel/as2017.sh
@@ -20,6 +20,7 @@
 buildtype="openmp"
 path2CLEO=${HOME}/CLEO/
 path2build=${HOME}/CLEO/build_adia0D/
+enableyac=false
 executables="adia0D"
 
 pythonscript=${path2CLEO}/examples/adiabaticparcel/as2017.py
@@ -31,6 +32,6 @@ script_args="${configfile}"
 
 ### ---------- build, compile and run example ---------- ###
 ${path2CLEO}/examples/run_example.sh \
-  ${buildtype} ${path2CLEO} ${path2build} \
+  ${buildtype} ${path2CLEO} ${path2build} ${enableyac} \
   "${executables}" ${pythonscript} "${script_args}"
 ### ---------------------------------------------------- ###
diff --git a/examples/adiabaticparcel/cuspbifurc.sh b/examples/adiabaticparcel/cuspbifurc.sh
@@ -20,6 +20,7 @@
 buildtype="serial"
 path2CLEO=${HOME}/CLEO/
 path2build=${HOME}/CLEO/build_adia0D/
+enableyac=false
 executables="adia0D"
 
 pythonscript=${path2CLEO}/examples/adiabaticparcel/cuspbifurc.py
@@ -31,6 +32,6 @@ script_args="${configfile}"
 
 ### ---------- build, compile and run example ---------- ###
 ${path2CLEO}/examples/run_example.sh \
-  ${buildtype} ${path2CLEO} ${path2build} \
+  ${buildtype} ${path2CLEO} ${path2build} ${enableyac} \
   "${executables}" ${pythonscript} "${script_args}"
 ### ---------------------------------------------------- ###
diff --git a/examples/boxmodelcollisions/shima2009.sh b/examples/boxmodelcollisions/shima2009.sh
@@ -21,6 +21,7 @@
 buildtype="cuda"
 path2CLEO=${HOME}/CLEO/
 path2build=${HOME}/CLEO/build_shima2009/
+enableyac=false
 executables="golcolls longcolls lowlistcolls"
 
 pythonscript=${path2CLEO}/examples/boxmodelcollisions/shima2009.py
@@ -32,6 +33,6 @@ script_args="${configfile} golovin long lowlist"
 
 ### ---------- build, compile and run example ---------- ###
 ${path2CLEO}/examples/run_example.sh \
-  ${buildtype} ${path2CLEO} ${path2build} \
+  ${buildtype} ${path2CLEO} ${path2build} ${enableyac} \
   "${executables}" ${pythonscript} "${script_args}"
 ### ---------------------------------------------------- ###
diff --git a/examples/constthermo2d/constthermo2d.sh b/examples/constthermo2d/constthermo2d.sh
@@ -21,6 +21,7 @@
 buildtype="cuda"
 path2CLEO=${HOME}/CLEO/
 path2build=${HOME}/CLEO/build_const2D/
+enableyac=false
 executables="const2D"
 
 pythonscript=${path2CLEO}/examples/constthermo2d/constthermo2d.py
@@ -32,6 +33,6 @@ script_args="${configfile}"
 
 ### ---------- build, compile and run example ---------- ###
 ${path2CLEO}/examples/run_example.sh \
-  ${buildtype} ${path2CLEO} ${path2build} \
+  ${buildtype} ${path2CLEO} ${path2build} ${enableyac} \
   "${executables}" ${pythonscript} "${script_args}"
 ### ---------------------------------------------------- ###
diff --git a/examples/divfreemotion/divfree2d.sh b/examples/divfreemotion/divfree2d.sh
@@ -20,6 +20,7 @@
 buildtype="cuda"
 path2CLEO=${HOME}/CLEO/
 path2build=${HOME}/CLEO/build_divfree2D/
+enableyac=false
 executables="divfree2D"
 
 pythonscript=${path2CLEO}/examples/divfreemotion/divfree2d.py
@@ -31,6 +32,6 @@ script_args="${configfile}"
 
 ### ---------- build, compile and run example ---------- ###
 ${path2CLEO}/examples/run_example.sh \
-  ${buildtype} ${path2CLEO} ${path2build} \
+  ${buildtype} ${path2CLEO} ${path2build} ${enableyac} \
   "${executables}" ${pythonscript} "${script_args}"
 ### ---------------------------------------------------- ###
diff --git a/examples/eurec4a1d/eurec4a1d.sh b/examples/eurec4a1d/eurec4a1d.sh
@@ -20,6 +20,7 @@
 buildtype="cuda"
 path2CLEO=${HOME}/CLEO/
 path2build=${HOME}/CLEO/build_eurec4a1D/
+enableyac=false
 executables="eurec4a1D"
 
 configfile=${path2CLEO}/examples/eurec4a1d/src/config/eurec4a1d_config.yaml
@@ -31,6 +32,6 @@ script_args=""
 
 ### ---------- build, compile and run example ---------- ###
 ${path2CLEO}/examples/run_example.sh \
-  ${buildtype} ${path2CLEO} ${path2build} \
+  ${buildtype} ${path2CLEO} ${path2build} ${enableyac} \
   "${executables}" ${pythonscript} "${script_args}"
 ### ---------------------------------------------------- ###
diff --git a/examples/rainshaft1d/rainshaft1d.sh b/examples/rainshaft1d/rainshaft1d.sh
@@ -20,6 +20,7 @@
 buildtype="cuda"
 path2CLEO=${HOME}/CLEO/
 path2build=${HOME}/CLEO/build_rshaft1D/
+enableyac=false
 executables="rshaft1D"
 
 pythonscript=${path2CLEO}/examples/rainshaft1d/rainshaft1d.py
@@ -31,6 +32,6 @@ script_args="${configfile}"
 
 ### ---------- build, compile and run example ---------- ###
 ${path2CLEO}/examples/run_example.sh \
-  ${buildtype} ${path2CLEO} ${path2build} \
+  ${buildtype} ${path2CLEO} ${path2build} ${enableyac} \
   "${executables}" ${pythonscript} "${script_args}"
 ### ---------------------------------------------------- ###
diff --git a/examples/run_example.sh b/examples/run_example.sh
@@ -25,12 +25,14 @@
 buildtype=$1
 path2CLEO=$2
 path2build=$3
-executables="$4"
-pythonscript=$5
-script_args="$6"
+enableyac=$4      # required "true" or otherwise
+executables="$5"
+pythonscript=$6
+script_args="$7"
 
 cleoenv=/work/mh1126/m300950/cleoenv
 python=${cleoenv}/bin/python3
+yacyaxtroot=/work/mh1126/m300950/yac
 spack load cmake@3.23.1%gcc
 module load python3/2022.01-gcc-11.2.0
 source activate ${cleoenv}
@@ -43,14 +45,15 @@ echo "----- Running Example -----"
 echo "buildtype:  ${buildtype}"
 echo "path2CLEO: ${path2CLEO}"
 echo "path2build: ${path2build}"
+echo "enableyac: ${enableyac}"
 echo "executables: ${executables}"
 echo "pythonscript: ${pythonscript}"
 echo "script_args: ${script_args}"
 echo "---------------------------"
 ### ---------------------------------------------------- ###
 
 ### ---------------------- build CLEO ------------------ ###
-${path2CLEO}/scripts/bash/build_cleo.sh ${buildtype} ${path2CLEO} ${path2build}
+${path2CLEO}/scripts/bash/build_cleo.sh ${buildtype} ${path2CLEO} ${path2build} ${enableyac} ${yacyaxtroot}
 ### ---------------------------------------------------- ###
 
 ### --------- compile executable(s) from scratch ---------- ###
@@ -62,5 +65,8 @@ ${path2CLEO}/scripts/bash/compile_cleo.sh ${cleoenv} ${buildtype} ${path2build}
 ### --------- run model through Python script ---------- ###
 export OMP_PROC_BIND=spread
 export OMP_PLACES=threads
+
+# TODO(all): add exports to paths required if YAC is enabled
+
 ${python}  ${pythonscript} ${path2CLEO} ${path2build} ${script_args}
 ### ---------------------------------------------------- ###
diff --git a/examples/speedtest/speedtest.sh b/examples/speedtest/speedtest.sh
@@ -19,6 +19,7 @@
 ### ---------------------------------------------------- ###
 path2CLEO=${HOME}/CLEO/
 path2build=${HOME}/CLEO/build_spdtest/
+enableyac=false
 executables="spdtest"
 
 pythonscript=${path2CLEO}/examples/speedtest/speedtest.py
@@ -46,7 +47,7 @@ do
 
   ### ---------- build, compile and run example ---------- ###
   ${path2CLEO}/examples/run_example.sh \
-    ${buildtype} ${path2CLEO} ${path2build_test} \
+    ${buildtype} ${path2CLEO} ${path2build} ${enableyac} \
     "${executables}" ${pythonscript} "${script_args}"
   ### ---------------------------------------------------- ###
 done

diff --git a/examples/yac/fromfile/yac1_fromfile.sh b/examples/yac/fromfile/yac1_fromfile.sh
@@ -20,6 +20,7 @@
 buildtype="openmp"
 path2CLEO=${HOME}/CLEO/
 path2build=${HOME}/CLEO/build_yac1/
+enableyac=false
 executables="yac1"
 
 pythonscript=${path2CLEO}/examples/yac/fromfile/yac1_fromfile.py
@@ -31,6 +32,6 @@ script_args="${configfile}"
 
 ### ---------- build, compile and run example ---------- ###
 ${path2CLEO}/examples/run_example.sh \
-  ${buildtype} ${path2CLEO} ${path2build} \
+  ${buildtype} ${path2CLEO} ${path2build} ${enableyac} \
   "${executables}" ${pythonscript} "${script_args}"
 ### ---------------------------------------------------- ###
diff --git a/examples/yac/yac_3d/yac_3d_fromfile.sh b/examples/yac/yac_3d/yac_3d_fromfile.sh
@@ -0,0 +1,37 @@
+#!/bin/bash
+#SBATCH --job-name=yac_3d
+#SBATCH --partition=compute
+#SBATCH --nodes=1
+#SBATCH --ntasks-per-node=128
+#SBATCH --mem=30G
+#SBATCH --time=00:05:00
+#SBATCH --mail-user=clara.bayley@mpimet.mpg.de
+#SBATCH --mail-type=FAIL
+#SBATCH --account=mh1126
+#SBATCH --output=./yac_3d_out.%j.out
+#SBATCH --error=./yac_3d_err.%j.out
+
+### ---------------------------------------------------- ###
+### ------------------ Input Parameters ---------------- ###
+### ------ You MUST edit these lines to set your ------- ###
+### ---- build type, directories, the executable(s) ---- ###
+### -------- to compile, and your python script -------- ###
+### ---------------------------------------------------- ###
+buildtype="openmp"
+path2CLEO=${HOME}/CLEO/
+path2build=${HOME}/CLEO/build_yac3d/
+enableyac=true
+executables="yac_3d"
+
+pythonscript=${path2CLEO}/examples/yac/yac_3d/yac_3d_fromfile.py
+configfile=${path2CLEO}/examples/yac/yac_3d/src/config/yac1_fromfile_config.yaml
+script_args="${configfile}"
+### ---------------------------------------------------- ###
+### ---------------------------------------------------- ###
+### ---------------------------------------------------- ###
+
+### ---------- build, compile and run example ---------- ###
+${path2CLEO}/examples/run_example.sh \
+  ${buildtype} ${path2CLEO} ${path2build} ${enableyac} \
+  "${executables}" ${pythonscript} "${script_args}"
+### ---------------------------------------------------- ###
diff --git a/extern/kokkos/CMakeLists.txt b/extern/kokkos/CMakeLists.txt
@@ -23,10 +23,10 @@ if(${Kokkos_ENABLE_CUDA})
   message(STATUS "Using Kokkos nvcc wrapper (see: \
 https://kokkos.org/kokkos-core-wiki/ProgrammingGuide/Compiling.html?highlight=wrapper#building-for-cuda)")
   message(STATUS "CXX compiler: ${CMAKE_CXX_COMPILER}")
-  message(STATUS "CC compiler: ${CMAKE_CC_COMPILER}")
+  message(STATUS "CC compiler: ${CMAKE_C_COMPILER}")
   message(STATUS "wrapper default (C++) compiler: ${NVCC_WRAPPER_DEFAULT_COMPILER}")
   message(STATUS "wrapper CUDA compiler: ${CUDA_ROOT}/bin/nvcc")
 else()
   message(STATUS "CXX compiler: ${CMAKE_CXX_COMPILER}")
-  message(STATUS "CC compiler: ${CMAKE_CC_COMPILER}")
+  message(STATUS "CC compiler: ${CMAKE_C_COMPILER}")
 endif()
diff --git a/libs/coupldyn_cvode/CMakeLists.txt b/libs/coupldyn_cvode/CMakeLists.txt
@@ -10,6 +10,11 @@ message("${LIBNAME} LIBRARY_SOURCE_DIR: ${CMAKE_CURRENT_SOURCE_DIR}")
 # explicitly set library executables path to /lib in top level of build tree
 set(LIBRARY_OUTPUT_PATH ${CMAKE_BINARY_DIR}/lib)
 
+# Enable but don't use Sundials' LAPACK option
+option(ENABLE_LAPACK "enable LAPACK" OFF)
+unset(LAPACK_WORKS)
+message("Enable and use LAPACK: ${ENABLE_LAPACK} ${LAPACK_WORKS}")
+
 # fetch CVODE libraries and make available
 include(FetchContent)
 FetchContent_Declare(

diff --git a/libs/coupldyn_yac/CMakeLists.txt b/libs/coupldyn_yac/CMakeLists.txt
@@ -10,8 +10,11 @@ message("${LIBNAME} LIBRARY_SOURCE_DIR: ${CMAKE_CURRENT_SOURCE_DIR}")
 # explicitly set library executables path to /lib in top level of build tree
 set(LIBRARY_OUTPUT_PATH ${CMAKE_BINARY_DIR}/lib)
 
-find_package(MPI)
-find_package(YAC)
+message(STATUS "Using YAC installation from: ${YAC_ROOT}")
+message(STATUS "Using YAXT installation from: ${YAXT_ROOT}")
+
+find_package(MPI REQUIRED)
+find_package(YAC REQUIRED)
 
 # Add executables and create library target
 set(SOURCES

diff --git a/roughpaper/build_compile_test.sh b/roughpaper/build_compile_test.sh
@@ -81,7 +81,7 @@ rm ${path2build}/roughpaper/test
 if [[ $1 == "build" ]];
 then
   cmake -DCMAKE_CXX_COMPILER=${CXX} \
-      -DCMAKE_CC_COMPILER=${CC} \
+      -DCMAKE_C_COMPILER=${CC} \
       -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}" \
       -S ${path2CLEO} -B ${path2build} \
       ${kokkosflags} ${kokkosdevice} ${kokkoshost} && \