fixed merge

.clang-tidy

@@ -1,56 +1,8 @@
 Checks: '-*,
-    bugprone-branch-clone,
-    bugprone-exception-escape,
-    bugprone-fold-init-type,
-    bugprone-forward-declaration-namespace,
-    bugprone-forwarding-reference-overload,
+    bugprone-*
+    -bugprone-easily-swappable-parameters,
     -bugprone-implicit-widening-of-multiplication-result,
-    bugprone-inaccurate-erase,
-    bugprone-incorrect-roundings,
-    bugprone-infinite-loop,
-    bugprone-integer-division,
-    bugprone-lambda-function-name,
-    bugprone-macro-parentheses,
-    bugprone-macro-repeated-side-effects,
-    bugprone-misplaced-operator-in-strlen-in-alloc,
-    bugprone-misplaced-pointer-arithmetic-in-alloc,
     -bugprone-misplaced-widening-cast,
-    bugprone-move-forwarding-reference,
-    bugprone-multiple-statement-macro,
-    bugprone-no-escape,
-    bugprone-not-null-terminated-result,
-    bugprone-parent-virtual-call,
-    bugprone-posix-return,
-    bugprone-redundant-branch-condition,
-    bugprone-reserved-identifier,
-    bugprone-signal-handler,
-    bugprone-signed-char-misuse,
-    bugprone-sizeof-container,
-    bugprone-sizeof-expression,
-    bugprone-spuriously-wake-up-functions,
-    bugprone-string-constructor,
-    bugprone-string-integer-assignment,
-    bugprone-string-literal-with-embedded-nul,
-    bugprone-stringview-nullptr,
-    bugprone-suspicious-enum-usage,
-    bugprone-suspicious-include,
-    bugprone-suspicious-memory-comparison,
-    bugprone-suspicious-memset-usage,
-    bugprone-suspicious-missing-comma,
-    bugprone-suspicious-semicolon,
-    bugprone-suspicious-string-compare,
-    bugprone-swapped-arguments,
-    bugprone-terminating-continue,
-    bugprone-throw-keyword-missing,
-    bugprone-too-small-loop-variable,
-    bugprone-undefined-memory-manipulation,
-    bugprone-undelegated-constructor,
-    bugprone-unhandled-exception-at-new,
-    bugprone-unhandled-self-assignment,
-    bugprone-unused-raii,
-    bugprone-unused-return-value,
-    bugprone-use-after-move,
-    bugprone-virtual-near-miss,
     cppcoreguidelines-avoid-goto,
     misc-const-correctness,
     modernize-avoid-bind,

.github/workflows/dependencies/dpcpp.sh

@@ -14,10 +14,13 @@ echo 'Acquire::Retries "3";' | sudo tee /etc/apt/apt.conf.d/80-retries
 
 # Ref.: https://github.com/rscohn2/oneapi-ci
 # intel-basekit intel-hpckit are too large in size
-wget -q -O - https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB \
-  | sudo apt-key add -
-echo "deb https://apt.repos.intel.com/oneapi all main" \
-  | sudo tee /etc/apt/sources.list.d/oneAPI.list
+
+# download the key to system keyring
+wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB \
+| gpg --dearmor | sudo tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null
+
+# add signed entry to apt sources and configure the APT client to use Intel repository:
+echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list
 
 sudo apt-get update
 
@@ -33,7 +36,7 @@ do
         build-essential \
         ccache          \
         cmake           \
-        intel-oneapi-dpcpp-cpp-compiler intel-oneapi-mkl-devel \
+        intel-oneapi-compiler-dpcpp-cpp intel-oneapi-mkl-devel \
         g++ gfortran    \
         libopenmpi-dev  \
         openmpi-bin     \

.github/workflows/insitu.yml

@@ -39,7 +39,7 @@ jobs:
       CC: gcc
       CMAKE_PREFIX_PATH: /ascent/install/lib/cmake/
     container:
-      image: alpinedav/ascent:0.9.1
+      image: alpinedav/ascent:0.9.2
     steps:
     - uses: actions/checkout@v3
     - name: Configure

Docs/source/install/hpc.rst

@@ -39,6 +39,7 @@ This section documents quick-start guides for a selection of supercomputers that
    hpc/fugaku
    hpc/hpc3
    hpc/juwels
+   hpc/karolina
    hpc/lassen
    hpc/lawrencium
    hpc/lumi

Docs/source/install/hpc/karolina.rst

@@ -0,0 +1,209 @@
+.. _building-karolina:
+
+Karolina (IT4I)
+===============
+
+The `Karolina cluster <https://docs.it4i.cz/karolina/introduction/>`_ is located at `IT4I, Technical University of Ostrava <https://www.it4i.cz/en>`__.
+
+
+Introduction
+------------
+
+If you are new to this system, **please see the following resources**:
+
+* `IT4I user guide <https://docs.it4i.cz>`__
+* Batch system: `PBS <https://docs.it4i.cz/general/job-submission-and-execution/>`__
+* Jupyter service: not provided/documented (yet)
+* `Filesystems <https://docs.it4i.cz/karolina/storage/>`__:
+
+  * ``$HOME``: per-user directory, use only for inputs, source and scripts; backed up (25GB default quota)
+  * ``/scatch/``: `production directory <https://docs.it4i.cz/karolina/storage/#scratch-file-system>`__; very fast for parallel jobs (20TB default)
+
+
+.. _building-karolina-preparation:
+
+Preparation
+-----------
+
+Use the following commands to download the WarpX source code:
+
+.. code-block:: bash
+
+   git clone https://github.com/ECP-WarpX/WarpX.git $HOME/src/warpx
+
+On Karolina, you can run either on GPU nodes with fast A100 GPUs (recommended) or CPU nodes.
+
+.. tab-set::
+
+   .. tab-item:: A100 GPUs
+
+      We use system software modules, add environment hints and further dependencies via the file ``$HOME/karolina_gpu_warpx.profile``.
+      Create it now:
+
+      .. code-block:: bash
+
+         cp $HOME/src/warpx/Tools/machines/karolina-it4i/karolina_gpu_warpx.profile.example $HOME/karolina_gpu_warpx.profile
+
+      .. dropdown:: Script Details
+         :color: light
+         :icon: info
+         :animate: fade-in-slide-down
+
+         .. literalinclude:: ../../../../Tools/machines/karolina-it4i/karolina_gpu_warpx.profile.example
+            :language: bash
+
+      Edit the 2nd line of this script, which sets the ``export proj=""`` variable.
+      For example, if you are member of the project ``DD-23-83``, then run ``vi $HOME/karolina_gpu_warpx.profile``.
+      Enter the edit mode by typing ``i`` and edit line 2 to read:
+
+      .. code-block:: bash
+
+         export proj="DD-23-83"
+
+      Exit the ``vi`` editor with ``Esc`` and then type ``:wq`` (write & quit).
+
+      .. important::
+
+         Now, and as the first step on future logins to Karolina, activate these environment settings:
+
+         .. code-block:: bash
+
+            source $HOME/karolina_gpu_warpx.profile
+
+      Finally, since Karolina does not yet provide software modules for some of our dependencies, install them once:
+
+      .. code-block:: bash
+
+         bash $HOME/src/warpx/Tools/machines/karolina-it4i/install_gpu_dependencies.sh
+         source $HOME/sw/karolina/gpu/venvs/warpx-gpu/bin/activate
+
+      .. dropdown:: Script Details
+         :color: light
+         :icon: info
+         :animate: fade-in-slide-down
+
+         .. literalinclude:: ../../../../Tools/machines/karolina-it4i/install_gpu_dependencies.sh
+            :language: bash
+
+
+   .. tab-item:: CPU Nodes
+
+      CPU usage is documentation is TODO.
+
+
+.. _building-karolina-compilation:
+
+Compilation
+-----------
+
+Use the following :ref:`cmake commands <building-cmake>` to compile:
+
+.. tab-set::
+
+   .. tab-item:: A100 GPUs
+
+      .. code-block:: bash
+
+         cd $HOME/src/warpx
+         rm -rf build_gpu
+
+         cmake -S . -B build_gpu -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_LIB=ON -DWarpX_DIMS="1;2;RZ;3"
+         cmake --build build_gpu -j 12
+         cmake --build build_gpu -j 12 --target pip_install
+
+      **That's it!**
+      The WarpX application executables are now in ``$HOME/src/warpx/build_gpu/bin/`` and we installed the ``pywarpx`` Python module.
+
+   .. tab-item:: CPU Nodes
+
+      .. code-block:: bash
+
+         cd $HOME/src/warpx
+         rm -rf build_cpu
+
+         cmake -S . -B build_cpu -DWarpX_COMPUTE=OMP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_LIB=ON -DWarpX_DIMS="1;2;RZ;3"
+         cmake --build build_cpu -j 12
+         cmake --build build_cpu -j 12 --target pip_install
+
+      **That's it!**
+      The WarpX application executables are now in ``$HOME/src/warpx/build_cpu/bin/`` and we installed the ``pywarpx`` Python module.
+
+Now, you can :ref:`submit Karolina compute jobs <running-cpp-karolina>` for WarpX :ref:`Python (PICMI) scripts <usage-picmi>` (:ref:`example scripts <usage-examples>`).
+Or, you can use the WarpX executables to submit Karolina jobs (:ref:`example inputs <usage-examples>`).
+For executables, you can reference their location in your :ref:`job script <running-cpp-karolina>` or copy them to a location in ``/scatch/``.
+
+
+.. _building-karolina-update:
+
+Update WarpX & Dependencies
+---------------------------
+
+If you already installed WarpX in the past and want to update it, start by getting the latest source code:
+
+.. code-block:: bash
+
+   cd $HOME/src/warpx
+
+   # read the output of this command - does it look ok?
+   git status
+
+   # get the latest WarpX source code
+   git fetch
+   git pull
+
+   # read the output of these commands - do they look ok?
+   git status
+   git log # press q to exit
+
+And, if needed,
+
+- :ref:`update the karolina_gpu_warpx.profile or karolina_cpu_warpx.profile files <building-karolina-preparation>`,
+- log out and into the system, activate the now updated environment profile as usual,
+- :ref:`execute the dependency install scripts <building-karolina-preparation>`.
+
+As a last step, clean the build directory ``rm -rf $HOME/src/warpx/build_*`` and rebuild WarpX.
+
+
+.. _running-cpp-karolina:
+
+Running
+-------
+
+.. tab-set::
+
+   .. tab-item:: A100 (40GB) GPUs
+
+      The batch script below can be used to run a WarpX simulation on multiple GPU nodes (change ``#PBS -l select=`` accordingly) on the supercomputer Karolina at IT4I.
+      This partition as up to `72 nodes <https://docs.it4i.cz/karolina/hardware-overview/>`__.
+      Every node has 8x A100 (40GB) GPUs and 2x AMD EPYC 7763, 64-core, 2.45 GHz processors.
+
+      Replace descriptions between chevrons ``<>`` by relevant values, for instance ``<proj>`` could be ``DD-23-83``.
+      Note that we run one MPI rank per GPU.
+
+      .. literalinclude:: ../../../../Tools/machines/karolina-it4i/karolina_gpu.qsub
+         :language: bash
+         :caption: You can copy this file from ``$HOME/src/warpx/Tools/machines/karolina-it4i/karolina_gpu.qsub``.
+
+      To run a simulation, copy the lines above to a file ``karolina_gpu.qsub`` and run
+
+      .. code-block:: bash
+
+         qsub karolina_gpu.qsub
+
+      to submit the job.
+
+
+   .. tab-item:: CPU Nodes
+
+      CPU usage is documentation is TODO.
+
+
+.. _post-processing-karolina:
+
+Post-Processing
+---------------
+
+.. note::
+
+   This section was not yet written.
+   Usually, we document here how to use a Jupyter service.

Docs/source/usage/parameters.rst

@@ -2659,7 +2659,8 @@ Reduced Diagnostics
         to file. The electromagnetic field components are interpolated to the measurement point
         by default, but can they be saved as non-averaged by setting
         ``<reduced_diags_name>.raw_fields = true``, in which case the raw fields for the cell
-        containing the measurement point are saved.
+        containing the measurement point are saved. In RZ geometry, this only saves the
+        0'th azimuthal mode component of the fields.
         The interpolation order can be set by specifying ``<reduced_diags_name>.interp_order``,
         otherwise it is set to ``1``.
         Integrated electric and magnetic field components can instead be obtained by specifying
@@ -2983,6 +2984,9 @@ Reduced Diagnostics
     The separator between row values in the output file.
     The default separator is a whitespace.
 
+* ``<reduced_diags_name>.precision`` (`integer`) optional (default `14`)
+    The precision used when writing out the data to the text files.
+
 Lookup tables and other settings for QED modules
 ------------------------------------------------
 

Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp

@@ -302,7 +302,12 @@ FlushFormatPlotfile::WriteWarpXHeader(
 
         warpx.GetPartContainer().WriteHeader(HeaderFile);
 
-        HeaderFile << warpx.getcurrent_injection_position() << "\n";
+        MultiParticleContainer& mypc = warpx.GetPartContainer();
+        const int n_species = mypc.nSpecies();
+        for (int i=0; i<n_species; i++)
+        {
+             HeaderFile << mypc.GetParticleContainer(i).m_current_injection_position << "\n";
+        }
 
         HeaderFile << warpx.getdo_moving_window() << "\n";
 

Source/Diagnostics/ReducedDiags/FieldProbe.H

@@ -66,7 +66,7 @@ public:
      */
 
     //! noutputs is 11 for particle id + (x, y, z, Ex, Ey, Ez, Bx, By, Bz, S)
-    static constexpr int noutputs = FieldProbePIdx::nattribs + 3 + 1;
+    static const int noutputs = 11;
 
 private:
     amrex::Real x_probe = 0._rt;

Source/Diagnostics/ReducedDiags/FieldProbe.cpp

@@ -51,12 +51,6 @@ FieldProbe::FieldProbe (std::string rd_name)
 : ReducedDiags{rd_name}, m_probe(&WarpX::GetInstance())
 {
 
-    // RZ coordinate is not working
-#if (defined WARPX_DIM_RZ)
-    WARPX_ALWAYS_ASSERT_WITH_MESSAGE(false,
-        "FieldProbe reduced diagnostics does not work for RZ coordinate.");
-#endif
-
     // read number of levels
     int nLevel = 0;
     const amrex::ParmParse pp_amr("amr");

Source/Diagnostics/ReducedDiags/FieldProbeParticleContainer.H

@@ -19,13 +19,20 @@
  * This enumerated struct is used to index the field probe particle
  * values that are being stored as SoA data. Nattribs
  * is enumerated to give the number of attributes stored.
+ * The strange insertion of `theta` below is due to the use of
+ * GetParticlePosition for the field probe locations, which reads the probe
+ * theta value from PIdx::theta = 4.
  */
 struct FieldProbePIdx
 {
     enum
     {
         Ex = 0, Ey, Ez,
-        Bx, By, Bz,
+        Bx,
+#ifdef WARPX_DIM_RZ
+        theta,      ///< RZ needs all three position components
+#endif
+        By, Bz,
         S, //!< the Poynting vector
         nattribs
     };