Add more build options for linear solvers (#4181)

Add two new CMake build options, AMReX_LINEAR_SOLVERS_INCFLO and
AMReX_LINEAR_SOLVERS_EM. Both options depend on AMReX_LINEAR_SOLVERS,
and are enabled by default when AMReX_LINEAR_SOLVERS is enabled. By
default, the cell-centered MLABecLaplacian and MLPoisson solvers are
always included. However, these new build options provide finer control
over which solvers to build.

* AMReX_LINEAR_SOLVERS_INCFLO can be used to control the inclusion of
solvers commonly used by incompressible flow codes, such as nodal
projection and Navier-Stokes tensor solvers.

* AMReX_LINEAR_SOLVERS_EM can be used to control the inclusion of
solvers commonly used by electromagnetic applications, such as the curl
curl and tensor Laplacian solvers used by WarpX.

These options are also available in GNU Make as
USE_LINEAR_SOLVERS_INCFLO and USE_LINEAR_SOLVER_EM.

.github/workflows/apps.yml

@@ -52,7 +52,9 @@ jobs:
         export AMREX_HOME=${PWD}
         export MICROPHYSICS_HOME=${PWD}/Microphysics
         cd Castro/Exec/hydro_tests/Sedov/
-        make -j4 CCACHE=ccache USE_MPI=FALSE
+        make -j4 CCACHE=ccache USE_MPI=FALSE \
+            USE_LINEAR_SOLVERS_INCFLO=FALSE \
+            USE_LINEAR_SOLVERS_EM=FALSE
 
         ccache -s
         du -hs ~/.cache/ccache
@@ -92,7 +94,8 @@ jobs:
             -DWarpX_QED=OFF                       \
             -DWarpX_OPENPMD=OFF                   \
             -DCMAKE_VERBOSE_MAKEFILE=ON           \
-            -DCMAKE_CXX_COMPILER_LAUNCHER=ccache
+            -DCMAKE_CXX_COMPILER_LAUNCHER=ccache  \
+            -DAMReX_LINEAR_SOLVER_INCFLO=OFF
         cmake --build WarpX/build -j 4
 
         ccache -s

Docs/sphinx_documentation/source/BuildingAMReX.rst

@@ -463,6 +463,12 @@ The list of available options is reported in the :ref:`table <tab:cmakevar>` bel
    +------------------------------+-------------------------------------------------+-------------------------+-----------------------+
    | AMReX_LINEAR_SOLVERS         |  Build AMReX linear solvers                     | YES                     | YES, NO               |
    +------------------------------+-------------------------------------------------+-------------------------+-----------------------+
+   | AMReX_LINEAR_SOLVERS_INCFLO  |  Build AMReX linear solvers for incompressible  | YES                     | YES, NO               |
+   |                              |  flow                                           |                         |
+   +------------------------------+-------------------------------------------------+-------------------------+-----------------------+
+   | AMReX_LINEAR_SOLVERS_EM      |  Build AMReX linear solvers for electromagnetic | YES                     | YES, NO               |
+   |                              |  solvers                                        |                         |
+   +------------------------------+-------------------------------------------------+-------------------------+-----------------------+
    | AMReX_AMRDATA                |  Build data services                            | NO                      | YES, NO               |
    +------------------------------+-------------------------------------------------+-------------------------+-----------------------+
    | AMReX_AMRLEVEL               |  Build AmrLevel class                           | YES                     | YES, NO               |
@@ -681,6 +687,10 @@ A list of AMReX component names and related configure options are shown in the t
    +------------------------------+-----------------+
    | AMReX_LINEAR_SOLVERS         | LSOLVERS        |
    +------------------------------+-----------------+
+   | AMReX_LINEAR_SOLVERS_INCFLO  | LSOLVERS_INCFLO |
+   +------------------------------+-----------------+
+   | AMReX_LINEAR_SOLVERS_EM      | LSOLVERS_EM     |
+   +------------------------------+-----------------+
    | AMReX_AMRDATA                | AMRDATA         |
    +------------------------------+-----------------+
    | AMReX_AMRLEVEL               | AMRLEVEL        |

Src/Extern/HYPRE/AMReX_Habec_2D_K.H

@@ -6,7 +6,7 @@
 #include <AMReX_EBMultiFabUtil.H>
 #include <AMReX_MultiCutFab.H>
 #include <AMReX_EBFabFactory.H>
-#include <AMReX_MLEBABecLap_K.H>
+#include <AMReX_MLLinOp_K.H>
 #endif
 
 namespace amrex {

Src/Extern/HYPRE/AMReX_Habec_3D_K.H

@@ -6,7 +6,7 @@
 #include <AMReX_EBMultiFabUtil.H>
 #include <AMReX_MultiCutFab.H>
 #include <AMReX_EBFabFactory.H>
-#include <AMReX_MLEBABecLap_K.H>
+#include <AMReX_MLLinOp_K.H>
 #endif
 
 namespace amrex {

Src/LinearSolvers/CMakeLists.txt

@@ -16,6 +16,8 @@ foreach(D IN LISTS AMReX_SPACEDIM)
        MLMG/AMReX_MLLinOp_K.H
        MLMG/AMReX_MLCellLinOp.H
        MLMG/AMReX_MLNodeLinOp.H
+       MLMG/AMReX_MLNodeLinOp_K.H
+       MLMG/AMReX_MLNodeLinOp_${D}D_K.H
        MLMG/AMReX_MLNodeLinOp.cpp
        MLMG/AMReX_MLCellABecLap.H
        MLMG/AMReX_MLCellABecLap_K.H
@@ -31,30 +33,6 @@ foreach(D IN LISTS AMReX_SPACEDIM)
        MLMG/AMReX_MLPoisson.H
        MLMG/AMReX_MLPoisson_K.H
        MLMG/AMReX_MLPoisson_${D}D_K.H
-       MLMG/AMReX_MLNodeLaplacian.H
-       MLMG/AMReX_MLNodeLaplacian.cpp
-       MLMG/AMReX_MLNodeLaplacian_sync.cpp
-       MLMG/AMReX_MLNodeLaplacian_sten.cpp
-       MLMG/AMReX_MLNodeLaplacian_misc.cpp
-       MLMG/AMReX_MLNodeLap_K.H
-       MLMG/AMReX_MLNodeLap_${D}D_K.H
-       MLMG/AMReX_MLNodeTensorLaplacian.H
-       MLMG/AMReX_MLNodeTensorLaplacian.cpp
-       MLMG/AMReX_MLNodeTensorLap_K.H
-       MLMG/AMReX_MLNodeTensorLap_${D}D_K.H
-       MLMG/AMReX_MLTensorOp.H
-       MLMG/AMReX_MLTensorOp.cpp
-       MLMG/AMReX_MLTensorOp_grad.cpp
-       MLMG/AMReX_MLTensor_K.H
-       MLMG/AMReX_MLTensor_${D}D_K.H
-       MLMG/AMReX_MLEBNodeFDLaplacian.H
-       MLMG/AMReX_MLEBNodeFDLaplacian.cpp
-       MLMG/AMReX_MLEBNodeFDLap_K.H
-       MLMG/AMReX_MLEBNodeFDLap_${D}D_K.H
-       MLMG/AMReX_MLNodeABecLaplacian.H
-       MLMG/AMReX_MLNodeABecLaplacian.cpp
-       MLMG/AMReX_MLNodeABecLap_K.H
-       MLMG/AMReX_MLNodeABecLap_${D}D_K.H
        AMReX_GMRES.H
        AMReX_GMRES_MLMG.H
        )
@@ -68,30 +46,72 @@ foreach(D IN LISTS AMReX_SPACEDIM)
           )
     endif ()
 
-    if (NOT D EQUAL 1)
+    if (AMReX_LINEAR_SOLVERS_EM)
+       if (NOT D EQUAL 1 AND AMReX_LINEAR_SOLVERS_EM)
+          target_sources(amrex_${D}d
+             PRIVATE
+             MLMG/AMReX_MLCurlCurl.H
+             MLMG/AMReX_MLCurlCurl.cpp
+             MLMG/AMReX_MLCurlCurl_K.H
+             )
+       endif ()
+
        target_sources(amrex_${D}d
           PRIVATE
-          MLMG/AMReX_MLCurlCurl.H
-          MLMG/AMReX_MLCurlCurl.cpp
-          MLMG/AMReX_MLCurlCurl_K.H
-          )
+          MLMG/AMReX_MLEBNodeFDLaplacian.H
+          MLMG/AMReX_MLEBNodeFDLaplacian.cpp
+          MLMG/AMReX_MLEBNodeFDLap_K.H
+          MLMG/AMReX_MLEBNodeFDLap_${D}D_K.H
+          MLMG/AMReX_MLNodeTensorLaplacian.H
+          MLMG/AMReX_MLNodeTensorLaplacian.cpp
+          MLMG/AMReX_MLNodeTensorLap_K.H
+          MLMG/AMReX_MLNodeTensorLap_${D}D_K.H
+       )
+    endif ()
+
+    if (AMReX_LINEAR_SOLVERS_INCFLO)
+       target_sources(amrex_${D}d
+          PRIVATE
+          MLMG/AMReX_MLNodeABecLaplacian.H
+          MLMG/AMReX_MLNodeABecLaplacian.cpp
+          MLMG/AMReX_MLNodeABecLap_K.H
+          MLMG/AMReX_MLNodeABecLap_${D}D_K.H
+          MLMG/AMReX_MLNodeLaplacian.H
+          MLMG/AMReX_MLNodeLaplacian.cpp
+          MLMG/AMReX_MLNodeLaplacian_sync.cpp
+          MLMG/AMReX_MLNodeLaplacian_sten.cpp
+          MLMG/AMReX_MLNodeLaplacian_misc.cpp
+          MLMG/AMReX_MLNodeLap_K.H
+          MLMG/AMReX_MLNodeLap_${D}D_K.H
+          MLMG/AMReX_MLTensorOp.H
+          MLMG/AMReX_MLTensorOp.cpp
+          MLMG/AMReX_MLTensorOp_grad.cpp
+          MLMG/AMReX_MLTensor_K.H
+          MLMG/AMReX_MLTensor_${D}D_K.H
+       )
     endif ()
 
     if (AMReX_EB AND NOT D EQUAL 1)
        target_sources(amrex_${D}d
           PRIVATE
-          MLMG/AMReX_MLNodeLaplacian_eb.cpp
           MLMG/AMReX_MLEBABecLap.H
           MLMG/AMReX_MLEBABecLap.cpp
           MLMG/AMReX_MLEBABecLap_F.cpp
           MLMG/AMReX_MLEBABecLap_K.H
           MLMG/AMReX_MLEBABecLap_${D}D_K.H
-          MLMG/AMReX_MLEBTensorOp.H
-          MLMG/AMReX_MLEBTensorOp.cpp
-          MLMG/AMReX_MLEBTensorOp_bc.cpp
-          MLMG/AMReX_MLEBTensor_K.H
-          MLMG/AMReX_MLEBTensor_${D}D_K.H
           )
+
+       if (AMReX_LINEAR_SOLVERS_INCFLO)
+          target_sources(amrex_${D}d
+             PRIVATE
+             MLMG/AMReX_MLNodeLaplacian_eb.cpp
+             MLMG/AMReX_MLEBTensorOp.H
+             MLMG/AMReX_MLEBTensorOp.cpp
+             MLMG/AMReX_MLEBTensorOp_bc.cpp
+             MLMG/AMReX_MLEBTensor_K.H
+             MLMG/AMReX_MLEBTensor_${D}D_K.H
+             )
+       endif ()
     endif ()
 
     if (AMReX_FORTRAN)
@@ -102,7 +122,7 @@ foreach(D IN LISTS AMReX_SPACEDIM)
           )
     endif ()
 
-    if (AMReX_HYPRE)
+    if (AMReX_HYPRE AND AMReX_LINEAR_SOLVERS_INCFLO)
        target_sources(amrex_${D}d
           PRIVATE
           MLMG/AMReX_MLNodeLaplacian_hypre.cpp

Src/LinearSolvers/MLMG/AMReX_MLEBABecLap_K.H

@@ -6,13 +6,6 @@
 
 #include <AMReX_EBCellFlag.H>
 
-namespace amrex {
-    AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
-    Real get_dx_eb (Real kappa) noexcept {
-        return amrex::max(Real(0.3),(kappa*kappa-Real(0.25))/(Real(2.0)*kappa));
-    }
-}
-
 #if (AMREX_SPACEDIM == 2)
 #include <AMReX_MLEBABecLap_2D_K.H>
 #else

Src/LinearSolvers/MLMG/AMReX_MLEBNodeFDLaplacian.cpp

@@ -1,6 +1,6 @@
 #include <AMReX_MLEBNodeFDLaplacian.H>
 #include <AMReX_MLEBNodeFDLap_K.H>
-#include <AMReX_MLNodeLap_K.H>
+#include <AMReX_MLNodeLinOp_K.H>
 #include <AMReX_MLNodeTensorLap_K.H>
 #include <AMReX_MultiFabUtil.H>
 

Src/LinearSolvers/MLMG/AMReX_MLLinOp_K.H

@@ -1053,6 +1053,15 @@ void mllinop_apply_innu_zhi (int i, int j, int k,
     }
 }
 
+#ifdef AMREX_USE_EB
+
+AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
+Real get_dx_eb (Real kappa) noexcept {
+    return amrex::max(Real(0.3),(kappa*kappa-Real(0.25))/(Real(2.0)*kappa));
+}
+
+#endif
+
 }
 
 #endif