diff --git a/.clang-tidy b/.clang-tidy
index 239e1e8e52a..c7d43e1f907 100644
--- a/.clang-tidy
+++ b/.clang-tidy
@@ -1,56 +1,8 @@
Checks: '-*,
- bugprone-branch-clone,
- bugprone-exception-escape,
- bugprone-fold-init-type,
- bugprone-forward-declaration-namespace,
- bugprone-forwarding-reference-overload,
+ bugprone-*
+ -bugprone-easily-swappable-parameters,
-bugprone-implicit-widening-of-multiplication-result,
- bugprone-inaccurate-erase,
- bugprone-incorrect-roundings,
- bugprone-infinite-loop,
- bugprone-integer-division,
- bugprone-lambda-function-name,
- bugprone-macro-parentheses,
- bugprone-macro-repeated-side-effects,
- bugprone-misplaced-operator-in-strlen-in-alloc,
- bugprone-misplaced-pointer-arithmetic-in-alloc,
-bugprone-misplaced-widening-cast,
- bugprone-move-forwarding-reference,
- bugprone-multiple-statement-macro,
- bugprone-no-escape,
- bugprone-not-null-terminated-result,
- bugprone-parent-virtual-call,
- bugprone-posix-return,
- bugprone-redundant-branch-condition,
- bugprone-reserved-identifier,
- bugprone-signal-handler,
- bugprone-signed-char-misuse,
- bugprone-sizeof-container,
- bugprone-sizeof-expression,
- bugprone-spuriously-wake-up-functions,
- bugprone-string-constructor,
- bugprone-string-integer-assignment,
- bugprone-string-literal-with-embedded-nul,
- bugprone-stringview-nullptr,
- bugprone-suspicious-enum-usage,
- bugprone-suspicious-include,
- bugprone-suspicious-memory-comparison,
- bugprone-suspicious-memset-usage,
- bugprone-suspicious-missing-comma,
- bugprone-suspicious-semicolon,
- bugprone-suspicious-string-compare,
- bugprone-swapped-arguments,
- bugprone-terminating-continue,
- bugprone-throw-keyword-missing,
- bugprone-too-small-loop-variable,
- bugprone-undefined-memory-manipulation,
- bugprone-undelegated-constructor,
- bugprone-unhandled-exception-at-new,
- bugprone-unhandled-self-assignment,
- bugprone-unused-raii,
- bugprone-unused-return-value,
- bugprone-use-after-move,
- bugprone-virtual-near-miss,
cppcoreguidelines-avoid-goto,
misc-const-correctness,
modernize-avoid-bind,
diff --git a/.github/workflows/dependencies/dpcpp.sh b/.github/workflows/dependencies/dpcpp.sh
index 65e3f36477f..9ecc5e4ca19 100755
--- a/.github/workflows/dependencies/dpcpp.sh
+++ b/.github/workflows/dependencies/dpcpp.sh
@@ -14,10 +14,13 @@ echo 'Acquire::Retries "3";' | sudo tee /etc/apt/apt.conf.d/80-retries
# Ref.: https://github.com/rscohn2/oneapi-ci
# intel-basekit intel-hpckit are too large in size
-wget -q -O - https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2023.PUB \
- | sudo apt-key add -
-echo "deb https://apt.repos.intel.com/oneapi all main" \
- | sudo tee /etc/apt/sources.list.d/oneAPI.list
+
+# download the key to system keyring
+wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB \
+| gpg --dearmor | sudo tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null
+
+# add signed entry to apt sources and configure the APT client to use Intel repository:
+echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list
sudo apt-get update
@@ -33,7 +36,7 @@ do
build-essential \
ccache \
cmake \
- intel-oneapi-dpcpp-cpp-compiler intel-oneapi-mkl-devel \
+ intel-oneapi-compiler-dpcpp-cpp intel-oneapi-mkl-devel \
g++ gfortran \
libopenmpi-dev \
openmpi-bin \
diff --git a/.github/workflows/insitu.yml b/.github/workflows/insitu.yml
index d6d9f338181..57a25ce7629 100644
--- a/.github/workflows/insitu.yml
+++ b/.github/workflows/insitu.yml
@@ -39,7 +39,7 @@ jobs:
CC: gcc
CMAKE_PREFIX_PATH: /ascent/install/lib/cmake/
container:
- image: alpinedav/ascent:0.9.1
+ image: alpinedav/ascent:0.9.2
steps:
- uses: actions/checkout@v3
- name: Configure
diff --git a/Docs/source/install/hpc.rst b/Docs/source/install/hpc.rst
index 2829527675c..67468cc2f02 100644
--- a/Docs/source/install/hpc.rst
+++ b/Docs/source/install/hpc.rst
@@ -39,6 +39,7 @@ This section documents quick-start guides for a selection of supercomputers that
hpc/fugaku
hpc/hpc3
hpc/juwels
+ hpc/karolina
hpc/lassen
hpc/lawrencium
hpc/lumi
diff --git a/Docs/source/install/hpc/karolina.rst b/Docs/source/install/hpc/karolina.rst
new file mode 100644
index 00000000000..67627f04e91
--- /dev/null
+++ b/Docs/source/install/hpc/karolina.rst
@@ -0,0 +1,209 @@
+.. _building-karolina:
+
+Karolina (IT4I)
+===============
+
+The `Karolina cluster `_ is located at `IT4I, Technical University of Ostrava `__.
+
+
+Introduction
+------------
+
+If you are new to this system, **please see the following resources**:
+
+* `IT4I user guide `__
+* Batch system: `PBS `__
+* Jupyter service: not provided/documented (yet)
+* `Filesystems `__:
+
+ * ``$HOME``: per-user directory, use only for inputs, source and scripts; backed up (25GB default quota)
+ * ``/scatch/``: `production directory `__; very fast for parallel jobs (20TB default)
+
+
+.. _building-karolina-preparation:
+
+Preparation
+-----------
+
+Use the following commands to download the WarpX source code:
+
+.. code-block:: bash
+
+ git clone https://github.com/ECP-WarpX/WarpX.git $HOME/src/warpx
+
+On Karolina, you can run either on GPU nodes with fast A100 GPUs (recommended) or CPU nodes.
+
+.. tab-set::
+
+ .. tab-item:: A100 GPUs
+
+ We use system software modules, add environment hints and further dependencies via the file ``$HOME/karolina_gpu_warpx.profile``.
+ Create it now:
+
+ .. code-block:: bash
+
+ cp $HOME/src/warpx/Tools/machines/karolina-it4i/karolina_gpu_warpx.profile.example $HOME/karolina_gpu_warpx.profile
+
+ .. dropdown:: Script Details
+ :color: light
+ :icon: info
+ :animate: fade-in-slide-down
+
+ .. literalinclude:: ../../../../Tools/machines/karolina-it4i/karolina_gpu_warpx.profile.example
+ :language: bash
+
+ Edit the 2nd line of this script, which sets the ``export proj=""`` variable.
+ For example, if you are member of the project ``DD-23-83``, then run ``vi $HOME/karolina_gpu_warpx.profile``.
+ Enter the edit mode by typing ``i`` and edit line 2 to read:
+
+ .. code-block:: bash
+
+ export proj="DD-23-83"
+
+ Exit the ``vi`` editor with ``Esc`` and then type ``:wq`` (write & quit).
+
+ .. important::
+
+ Now, and as the first step on future logins to Karolina, activate these environment settings:
+
+ .. code-block:: bash
+
+ source $HOME/karolina_gpu_warpx.profile
+
+ Finally, since Karolina does not yet provide software modules for some of our dependencies, install them once:
+
+ .. code-block:: bash
+
+ bash $HOME/src/warpx/Tools/machines/karolina-it4i/install_gpu_dependencies.sh
+ source $HOME/sw/karolina/gpu/venvs/warpx-gpu/bin/activate
+
+ .. dropdown:: Script Details
+ :color: light
+ :icon: info
+ :animate: fade-in-slide-down
+
+ .. literalinclude:: ../../../../Tools/machines/karolina-it4i/install_gpu_dependencies.sh
+ :language: bash
+
+
+ .. tab-item:: CPU Nodes
+
+ CPU usage is documentation is TODO.
+
+
+.. _building-karolina-compilation:
+
+Compilation
+-----------
+
+Use the following :ref:`cmake commands ` to compile:
+
+.. tab-set::
+
+ .. tab-item:: A100 GPUs
+
+ .. code-block:: bash
+
+ cd $HOME/src/warpx
+ rm -rf build_gpu
+
+ cmake -S . -B build_gpu -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_LIB=ON -DWarpX_DIMS="1;2;RZ;3"
+ cmake --build build_gpu -j 12
+ cmake --build build_gpu -j 12 --target pip_install
+
+ **That's it!**
+ The WarpX application executables are now in ``$HOME/src/warpx/build_gpu/bin/`` and we installed the ``pywarpx`` Python module.
+
+ .. tab-item:: CPU Nodes
+
+ .. code-block:: bash
+
+ cd $HOME/src/warpx
+ rm -rf build_cpu
+
+ cmake -S . -B build_cpu -DWarpX_COMPUTE=OMP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_LIB=ON -DWarpX_DIMS="1;2;RZ;3"
+ cmake --build build_cpu -j 12
+ cmake --build build_cpu -j 12 --target pip_install
+
+ **That's it!**
+ The WarpX application executables are now in ``$HOME/src/warpx/build_cpu/bin/`` and we installed the ``pywarpx`` Python module.
+
+Now, you can :ref:`submit Karolina compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `).
+Or, you can use the WarpX executables to submit Karolina jobs (:ref:`example inputs `).
+For executables, you can reference their location in your :ref:`job script ` or copy them to a location in ``/scatch/``.
+
+
+.. _building-karolina-update:
+
+Update WarpX & Dependencies
+---------------------------
+
+If you already installed WarpX in the past and want to update it, start by getting the latest source code:
+
+.. code-block:: bash
+
+ cd $HOME/src/warpx
+
+ # read the output of this command - does it look ok?
+ git status
+
+ # get the latest WarpX source code
+ git fetch
+ git pull
+
+ # read the output of these commands - do they look ok?
+ git status
+ git log # press q to exit
+
+And, if needed,
+
+- :ref:`update the karolina_gpu_warpx.profile or karolina_cpu_warpx.profile files `,
+- log out and into the system, activate the now updated environment profile as usual,
+- :ref:`execute the dependency install scripts `.
+
+As a last step, clean the build directory ``rm -rf $HOME/src/warpx/build_*`` and rebuild WarpX.
+
+
+.. _running-cpp-karolina:
+
+Running
+-------
+
+.. tab-set::
+
+ .. tab-item:: A100 (40GB) GPUs
+
+ The batch script below can be used to run a WarpX simulation on multiple GPU nodes (change ``#PBS -l select=`` accordingly) on the supercomputer Karolina at IT4I.
+ This partition as up to `72 nodes `__.
+ Every node has 8x A100 (40GB) GPUs and 2x AMD EPYC 7763, 64-core, 2.45 GHz processors.
+
+ Replace descriptions between chevrons ``<>`` by relevant values, for instance ```` could be ``DD-23-83``.
+ Note that we run one MPI rank per GPU.
+
+ .. literalinclude:: ../../../../Tools/machines/karolina-it4i/karolina_gpu.qsub
+ :language: bash
+ :caption: You can copy this file from ``$HOME/src/warpx/Tools/machines/karolina-it4i/karolina_gpu.qsub``.
+
+ To run a simulation, copy the lines above to a file ``karolina_gpu.qsub`` and run
+
+ .. code-block:: bash
+
+ qsub karolina_gpu.qsub
+
+ to submit the job.
+
+
+ .. tab-item:: CPU Nodes
+
+ CPU usage is documentation is TODO.
+
+
+.. _post-processing-karolina:
+
+Post-Processing
+---------------
+
+.. note::
+
+ This section was not yet written.
+ Usually, we document here how to use a Jupyter service.
diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst
index 0a12dc82d6c..9c3041fce11 100644
--- a/Docs/source/usage/parameters.rst
+++ b/Docs/source/usage/parameters.rst
@@ -2659,7 +2659,8 @@ Reduced Diagnostics
to file. The electromagnetic field components are interpolated to the measurement point
by default, but can they be saved as non-averaged by setting
``.raw_fields = true``, in which case the raw fields for the cell
- containing the measurement point are saved.
+ containing the measurement point are saved. In RZ geometry, this only saves the
+ 0'th azimuthal mode component of the fields.
The interpolation order can be set by specifying ``.interp_order``,
otherwise it is set to ``1``.
Integrated electric and magnetic field components can instead be obtained by specifying
@@ -2983,6 +2984,9 @@ Reduced Diagnostics
The separator between row values in the output file.
The default separator is a whitespace.
+* ``.precision`` (`integer`) optional (default `14`)
+ The precision used when writing out the data to the text files.
+
Lookup tables and other settings for QED modules
------------------------------------------------
diff --git a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp
index 3ea631292cf..28033b3ace0 100644
--- a/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp
+++ b/Source/Diagnostics/FlushFormats/FlushFormatPlotfile.cpp
@@ -302,7 +302,12 @@ FlushFormatPlotfile::WriteWarpXHeader(
warpx.GetPartContainer().WriteHeader(HeaderFile);
- HeaderFile << warpx.getcurrent_injection_position() << "\n";
+ MultiParticleContainer& mypc = warpx.GetPartContainer();
+ const int n_species = mypc.nSpecies();
+ for (int i=0; i m_data;
diff --git a/Source/Diagnostics/ReducedDiags/ReducedDiags.cpp b/Source/Diagnostics/ReducedDiags/ReducedDiags.cpp
index 7a2cb6df2ca..38da1feff83 100644
--- a/Source/Diagnostics/ReducedDiags/ReducedDiags.cpp
+++ b/Source/Diagnostics/ReducedDiags/ReducedDiags.cpp
@@ -9,6 +9,7 @@
#include "WarpX.H"
#include "Utils/Parser/IntervalsParser.H"
+#include "Utils/Parser/ParserUtils.H"
#include "Utils/TextMsg.H"
#include
@@ -66,6 +67,9 @@ ReducedDiags::ReducedDiags (std::string rd_name)
// read separator
pp_rd_name.query("separator", m_sep);
+
+ // precision of data in the output file
+ utils::parser::queryWithParser(pp_rd_name, "precision", m_precision);
}
// end constructor
@@ -107,7 +111,7 @@ void ReducedDiags::WriteToFile (int step) const
ofs << m_sep;
// set precision
- ofs << std::fixed << std::setprecision(14) << std::scientific;
+ ofs << std::fixed << std::setprecision(m_precision) << std::scientific;
// write time
ofs << WarpX::GetInstance().gett_new(0);
diff --git a/Source/Diagnostics/WarpXIO.cpp b/Source/Diagnostics/WarpXIO.cpp
index 36f8756598a..aeaf8530438 100644
--- a/Source/Diagnostics/WarpXIO.cpp
+++ b/Source/Diagnostics/WarpXIO.cpp
@@ -211,8 +211,12 @@ WarpX::InitFromCheckpoint ()
}
mypc->ReadHeader(is);
- is >> current_injection_position;
- GotoNextLine(is);
+ const int n_species = mypc->nSpecies();
+ for (int i=0; i> mypc->GetParticleContainer(i).m_current_injection_position;
+ GotoNextLine(is);
+ }
int do_moving_window_before_restart;
is >> do_moving_window_before_restart;
diff --git a/Source/Initialization/PlasmaInjector.H b/Source/Initialization/PlasmaInjector.H
index 0f33aa06221..1c2fc453d73 100644
--- a/Source/Initialization/PlasmaInjector.H
+++ b/Source/Initialization/PlasmaInjector.H
@@ -133,8 +133,10 @@ public:
InjectorPosition* getInjectorPosition ();
InjectorDensity* getInjectorDensity ();
+
InjectorFlux* getInjectorFlux ();
- InjectorMomentum* getInjectorMomentum ();
+ InjectorMomentum* getInjectorMomentumDevice ();
+ InjectorMomentum* getInjectorMomentumHost ();
protected:
diff --git a/Source/Initialization/PlasmaInjector.cpp b/Source/Initialization/PlasmaInjector.cpp
index 865e531e7b8..1ed86864710 100644
--- a/Source/Initialization/PlasmaInjector.cpp
+++ b/Source/Initialization/PlasmaInjector.cpp
@@ -775,7 +775,13 @@ PlasmaInjector::getInjectorFlux ()
}
InjectorMomentum*
-PlasmaInjector::getInjectorMomentum ()
+PlasmaInjector::getInjectorMomentumDevice ()
{
return d_inj_mom;
}
+
+InjectorMomentum*
+PlasmaInjector::getInjectorMomentumHost ()
+{
+ return h_inj_mom.get();
+}
diff --git a/Source/Initialization/WarpXAMReXInit.H b/Source/Initialization/WarpXAMReXInit.H
index a34072f223b..06c31e35872 100644
--- a/Source/Initialization/WarpXAMReXInit.H
+++ b/Source/Initialization/WarpXAMReXInit.H
@@ -11,24 +11,28 @@
#include
-/** Call amrex::Initialize
- *
- * This function calls amrex::Initialize and overwrites AMReX' defaults.
- * Note: AMReX defines a placeholder/"mock-up" for MPI_Comm and
- * MPI_COMM_WORLD in serial builds
- *
- * @param[in] argc number of arguments from main()
- * @param[in] argv argument strings from main()
- * @param[in] build_parm_parse build the input file parser (AMReX' default: true)
- * @param[in] mpi_comm the MPI communicator to use (AMReX' default: MPI_COMM_WORLD)
- * @returns pointer to an AMReX* object, forwarded from amrex::Initialize
- */
-amrex::AMReX*
-warpx_amrex_init(
- int& argc,
- char**& argv,
- bool const build_parm_parse = true,
- MPI_Comm const mpi_comm = MPI_COMM_WORLD
-);
+namespace warpx::initialization
+{
+
+ /** Call amrex::Initialize
+ *
+ * This function calls amrex::Initialize and overwrites AMReX' defaults.
+ * Note: AMReX defines a placeholder/"mock-up" for MPI_Comm and
+ * MPI_COMM_WORLD in serial builds
+ *
+ * @param[in] argc number of arguments from main()
+ * @param[in] argv argument strings from main()
+ * @param[in] build_parm_parse build the input file parser (AMReX' default: true)
+ * @param[in] mpi_comm the MPI communicator to use (AMReX' default: MPI_COMM_WORLD)
+ * @returns pointer to an AMReX* object, forwarded from amrex::Initialize
+ */
+ amrex::AMReX*
+ amrex_init(
+ int& argc,
+ char**& argv,
+ bool const build_parm_parse = true,
+ MPI_Comm const mpi_comm = MPI_COMM_WORLD
+ );
+}
#endif
diff --git a/Source/Initialization/WarpXAMReXInit.cpp b/Source/Initialization/WarpXAMReXInit.cpp
index 7f8af142f4a..33bc1a78998 100644
--- a/Source/Initialization/WarpXAMReXInit.cpp
+++ b/Source/Initialization/WarpXAMReXInit.cpp
@@ -56,14 +56,20 @@ namespace {
}
}
-amrex::AMReX*
-warpx_amrex_init (int& argc, char**& argv, bool const build_parm_parse, MPI_Comm const mpi_comm)
+
+namespace warpx::initialization
{
- return amrex::Initialize(
- argc,
- argv,
- build_parm_parse,
- mpi_comm,
- overwrite_amrex_parser_defaults
- );
+
+ amrex::AMReX*
+ amrex_init (int& argc, char**& argv, bool const build_parm_parse, MPI_Comm const mpi_comm)
+ {
+ return amrex::Initialize(
+ argc,
+ argv,
+ build_parm_parse,
+ mpi_comm,
+ ::overwrite_amrex_parser_defaults
+ );
+ }
+
}
diff --git a/Source/Particles/PhysicalParticleContainer.H b/Source/Particles/PhysicalParticleContainer.H
index 4a9beeec24f..08682c5819d 100644
--- a/Source/Particles/PhysicalParticleContainer.H
+++ b/Source/Particles/PhysicalParticleContainer.H
@@ -66,6 +66,11 @@ public:
virtual void InitIonizationModule () override;
+ /*
+ * \brief Returns a pointer to the plasma injector.
+ */
+ virtual PlasmaInjector* GetPlasmaInjector () override;
+
/**
* \brief Evolve is the central function PhysicalParticleContainer that
* advances plasma particles for a time dt (typically one timestep).
@@ -194,7 +199,8 @@ public:
void AddPlasmaFlux (amrex::Real dt);
void MapParticletoBoostedFrame (amrex::ParticleReal& x, amrex::ParticleReal& y, amrex::ParticleReal& z,
- amrex::ParticleReal& ux, amrex::ParticleReal& uy, amrex::ParticleReal& uz);
+ amrex::ParticleReal& ux, amrex::ParticleReal& uy, amrex::ParticleReal& uz,
+ amrex::ParticleReal t_lab = 0._prt);
void AddGaussianBeam (
const amrex::Real x_m, const amrex::Real y_m, const amrex::Real z_m,
@@ -219,7 +225,8 @@ public:
amrex::Gpu::HostVector& particle_ux,
amrex::Gpu::HostVector& particle_uy,
amrex::Gpu::HostVector& particle_uz,
- amrex::Gpu::HostVector& particle_w);
+ amrex::Gpu::HostVector& particle_w,
+ amrex::ParticleReal t_lab= 0._prt);
/**
* \brief Default initialize runtime attributes in a tile. This routine does not initialize the
diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp
index 0793094ff0c..56553575c49 100644
--- a/Source/Particles/PhysicalParticleContainer.cpp
+++ b/Source/Particles/PhysicalParticleContainer.cpp
@@ -393,7 +393,7 @@ void PhysicalParticleContainer::InitData ()
}
void PhysicalParticleContainer::MapParticletoBoostedFrame (
- ParticleReal& x, ParticleReal& y, ParticleReal& z, ParticleReal& ux, ParticleReal& uy, ParticleReal& uz)
+ ParticleReal& x, ParticleReal& y, ParticleReal& z, ParticleReal& ux, ParticleReal& uy, ParticleReal& uz, ParticleReal t_lab)
{
// Map the particles from the lab frame to the boosted frame.
// This boosts the particle to the lab frame and calculates
@@ -402,8 +402,6 @@ void PhysicalParticleContainer::MapParticletoBoostedFrame (
// For now, start with the assumption that this will only happen
// at the start of the simulation.
- const ParticleReal t_lab = 0._prt;
-
const ParticleReal uz_boost = WarpX::gamma_boost*WarpX::beta_boost*PhysConst::c;
// tpr is the particle's time in the boosted frame
@@ -429,13 +427,14 @@ void PhysicalParticleContainer::MapParticletoBoostedFrame (
uz = -uz;
}
- // Move the particles to where they will be at t = 0 in the boosted frame
+ //Move the particles to where they will be at t = t0, the current simulation time in the boosted frame
+ constexpr int lev = 0;
+ const amrex::Real t0 = WarpX::GetInstance().gett_new(lev);
if (boost_adjust_transverse_positions) {
- x = xpr - tpr*vxpr;
- y = ypr - tpr*vypr;
+ x = xpr - (tpr-t0)*vxpr;
+ y = ypr - (tpr-t0)*vypr;
}
-
- z = zpr - tpr*vzpr;
+ z = zpr - (tpr-t0)*vzpr;
}
@@ -582,6 +581,7 @@ PhysicalParticleContainer::AddPlasmaFromFile(ParticleReal q_tot,
// assumption asserts: see PlasmaInjector
openPMD::Iteration it = series->iterations.begin()->second;
+ double const t_lab = it.time() * it.timeUnitSI();
std::string const ps_name = it.particles.begin()->first;
openPMD::ParticleSpecies ps = it.particles.begin()->second;
@@ -649,7 +649,7 @@ PhysicalParticleContainer::AddPlasmaFromFile(ParticleReal q_tot,
CheckAndAddParticle(x, y, z, ux, uy, uz, weight,
particle_x, particle_y, particle_z,
particle_ux, particle_uy, particle_uz,
- particle_w);
+ particle_w, t_lab);
}
}
auto const np = particle_z.size();
@@ -795,10 +795,11 @@ PhysicalParticleContainer::CheckAndAddParticle (
Gpu::HostVector& particle_ux,
Gpu::HostVector& particle_uy,
Gpu::HostVector& particle_uz,
- Gpu::HostVector& particle_w)
+ Gpu::HostVector& particle_w,
+ ParticleReal t_lab)
{
if (WarpX::gamma_boost > 1.) {
- MapParticletoBoostedFrame(x, y, z, ux, uy, uz);
+ MapParticletoBoostedFrame(x, y, z, ux, uy, uz, t_lab);
}
particle_x.push_back(x);
particle_y.push_back(y);
@@ -926,7 +927,7 @@ PhysicalParticleContainer::AddPlasma (int lev, RealBox part_realbox)
InjectorPosition* inj_pos = plasma_injector->getInjectorPosition();
InjectorDensity* inj_rho = plasma_injector->getInjectorDensity();
- InjectorMomentum* inj_mom = plasma_injector->getInjectorMomentum();
+ InjectorMomentum* inj_mom = plasma_injector->getInjectorMomentumDevice();
const Real gamma_boost = WarpX::gamma_boost;
const Real beta_boost = WarpX::beta_boost;
const Real t = WarpX::GetInstance().gett_new(lev);
@@ -1476,7 +1477,7 @@ PhysicalParticleContainer::AddPlasmaFlux (amrex::Real dt)
InjectorPosition* inj_pos = plasma_injector->getInjectorPosition();
InjectorFlux* inj_flux = plasma_injector->getInjectorFlux();
- InjectorMomentum* inj_mom = plasma_injector->getInjectorMomentum();
+ InjectorMomentum* inj_mom = plasma_injector->getInjectorMomentumDevice();
constexpr int level_zero = 0;
const amrex::Real t = WarpX::GetInstance().gett_new(level_zero);
@@ -2922,6 +2923,11 @@ PhysicalParticleContainer::getIonizationFunc (const WarpXParIter& pti,
ion_atomic_number);
}
+PlasmaInjector* PhysicalParticleContainer::GetPlasmaInjector ()
+{
+ return plasma_injector.get();
+}
+
void PhysicalParticleContainer::resample (const int timestep)
{
// In heavily load imbalanced simulations, MPI processes with few particles will spend most of
diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H
index ffeead6f1d6..6e0d3f4f927 100644
--- a/Source/Particles/WarpXParticleContainer.H
+++ b/Source/Particles/WarpXParticleContainer.H
@@ -13,6 +13,7 @@
#include "WarpXParticleContainer_fwd.H"
#include "Evolve/WarpXDtType.H"
+#include "Initialization/PlasmaInjector.H"
#include "Particles/ParticleBoundaries.H"
#include "SpeciesPhysicalProperties.H"
@@ -121,6 +122,12 @@ public:
virtual void InitIonizationModule () {}
+ /*
+ * \brief Virtual function that returns a pointer to the plasma injector,
+ * for derived classes that define one (PhysicalParticleContainer).
+ */
+ virtual PlasmaInjector* GetPlasmaInjector () { return nullptr; }
+
/**
* Evolve is the central WarpXParticleContainer function that advances
* particles for a time dt (typically one timestep). It is a pure virtual
@@ -241,10 +248,13 @@ public:
virtual void ContinuousInjection(const amrex::RealBox& /*injection_box*/) {}
// Update optional sub-class-specific injection location.
virtual void UpdateContinuousInjectionPosition(amrex::Real /*dt*/) {}
+ bool doContinuousInjection() const {return do_continuous_injection;}
// Inject a continuous flux of particles from a defined plane
virtual void ContinuousFluxInjection(amrex::Real /*t*/, amrex::Real /*dt*/) {}
+ int getSpeciesId() const {return species_id;}
+
///
/// This returns the total charge for all the particles in this ParticleContainer.
/// This is needed when solving Poisson's equation with periodic boundary conditions.
@@ -306,6 +316,9 @@ public:
int self_fields_max_iters = 200;
int self_fields_verbosity = 2;
+ //! Current injection position
+ amrex::Real m_current_injection_position;
+
// split along diagonals (0) or axes (1)
int split_type = 0;
diff --git a/Source/Python/WarpXWrappers.cpp b/Source/Python/WarpXWrappers.cpp
index afc4caa5bee..cbee65b6159 100644
--- a/Source/Python/WarpXWrappers.cpp
+++ b/Source/Python/WarpXWrappers.cpp
@@ -150,12 +150,12 @@ namespace
void amrex_init (int argc, char* argv[])
{
- warpx_amrex_init(argc, argv);
+ warpx::initialization::amrex_init(argc, argv);
}
void amrex_init_with_inited_mpi (int argc, char* argv[], MPI_Comm mpicomm)
{
- warpx_amrex_init(argc, argv, true, mpicomm);
+ warpx::initialization::amrex_init(argc, argv, true, mpicomm);
}
void amrex_finalize (int /*finalize_mpi*/)
diff --git a/Source/Utils/WarpXMovingWindow.cpp b/Source/Utils/WarpXMovingWindow.cpp
index 7e7985b98a0..e43740d87e2 100644
--- a/Source/Utils/WarpXMovingWindow.cpp
+++ b/Source/Utils/WarpXMovingWindow.cpp
@@ -56,25 +56,71 @@ void
WarpX::UpdatePlasmaInjectionPosition (amrex::Real a_dt)
{
const int dir = moving_window_dir;
- // Continuously inject plasma in new cells (by default only on level 0)
- if (WarpX::warpx_do_continuous_injection and (WarpX::gamma_boost > 1)){
- // In boosted-frame simulations, the plasma has moved since the last
- // call to this function, and injection position needs to be updated
- current_injection_position -= WarpX::beta_boost *
+
+ // Loop over species
+ const int n_species = mypc->nSpecies();
+ for (int i=0; iGetParticleContainer(i);
+
+ // Continuously inject plasma in new cells (by default only on level 0)
+ if (pc.doContinuousInjection())
+ {
+ PlasmaInjector* plasma_injector = pc.GetPlasmaInjector();
+ if (plasma_injector == nullptr) continue;
+
+ // Get bulk momentum and velocity of plasma
+ // 1D: dir=0 is z
+ // 2D: dir=0 is x, dir=1 is z
+ // 3D: dir=0 is x, dir=1 is y, dir=2 is z
+ amrex::Vector current_injection_position = {0._rt, 0._rt, 0._rt};
+#if defined(WARPX_DIM_1D_Z)
+ current_injection_position[2] = pc.m_current_injection_position;
+#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ)
+ current_injection_position[dir*2] = pc.m_current_injection_position;
+#else // 3D
+ current_injection_position[dir] = pc.m_current_injection_position;
+#endif
+ amrex::XDim3 u_bulk = plasma_injector->getInjectorMomentumHost()->getBulkMomentum(current_injection_position[0],
+ current_injection_position[1],
+ current_injection_position[2]);
+#if defined(WARPX_DIM_1D_Z)
+ amrex::Vector u_bulk_vec = {u_bulk.z};
+#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ)
+ amrex::Vector u_bulk_vec = {u_bulk.x, u_bulk.z};
+#else // 3D
+ amrex::Vector u_bulk_vec = {u_bulk.x, u_bulk.y, u_bulk.z};
+#endif
+ amrex::Real v_bulk = PhysConst::c * u_bulk_vec[dir] / std::sqrt(1._rt + u_bulk_vec[dir]*u_bulk_vec[dir]);
+
+ // In boosted-frame simulations, the plasma has moved since the last
+ // call to this function, and injection position needs to be updated.
+ // Note that the bulk velocity v, obtained from getBulkMomentum, is
+ // transformed to the boosted frame velocity v' via the formula
+ // v' = (v-c*beta)/(1-v*beta/c)
+ if (WarpX::gamma_boost > 1._rt)
+ {
+ v_bulk = (v_bulk - PhysConst::c*WarpX::beta_boost)
+ / (1._rt - v_bulk*WarpX::beta_boost/PhysConst::c);
#if defined(WARPX_DIM_3D)
- WarpX::boost_direction[dir] * PhysConst::c * a_dt;
+ v_bulk *= WarpX::boost_direction[dir];
#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ)
- // In 2D, dir=0 corresponds to x and dir=1 corresponds to z
- // This needs to be converted in order to index `boost_direction`
- // which has 3 components, for both 2D and 3D simulations.
- WarpX::boost_direction[2*dir] * PhysConst::c * a_dt;
+ // In 2D, dir=0 corresponds to x and dir=1 corresponds to z.
+ // This needs to be converted to access boost_direction,
+ // which has always 3 components.
+ v_bulk *= WarpX::boost_direction[2*dir];
#elif defined(WARPX_DIM_1D_Z)
- // In 1D, dir=0 corresponds to z
- // This needs to be converted in order to index `boost_direction`
- // which has 3 components, for 1D, 2D, and 3D simulations.
- WarpX::boost_direction[2] * PhysConst::c * a_dt;
- amrex::ignore_unused(dir);
+ // In 1D, dir=0 corresponds to z.
+ // This needs to be converted to access boost_direction,
+ // which has always 3 components.
+ v_bulk *= WarpX::boost_direction[2];
+ amrex::ignore_unused(dir);
#endif
+ }
+
+ // Update current injection position
+ pc.m_current_injection_position += v_bulk * a_dt;
+ }
}
}
@@ -96,15 +142,13 @@ WarpX::MoveWindow (const int step, bool move_j)
moving_window_x += (moving_window_v - WarpX::beta_boost * PhysConst::c)/(1 - moving_window_v * WarpX::beta_boost / PhysConst::c) * dt[0];
const int dir = moving_window_dir;
- // Update warpx.current_injection_position
+ // Update warpx.current_injection_position,
// PhysicalParticleContainer uses this injection position
UpdatePlasmaInjectionPosition( dt[0] );
- if (WarpX::warpx_do_continuous_injection){
- // Update injection position for WarpXParticleContainer in mypc.
- // Nothing to do for PhysicalParticleContainers
- // For LaserParticleContainer, need to update the antenna position.
- mypc->UpdateContinuousInjectionPosition( dt[0] );
- }
+ // Update injection position for WarpXParticleContainer in mypc,
+ // nothing to do for PhysicalParticleContainer,
+ // need to update the antenna position for LaserParticleContainer.
+ mypc->UpdateContinuousInjectionPosition( dt[0] );
// compute the number of cells to shift on the base level
amrex::Real new_lo[AMREX_SPACEDIM];
@@ -312,41 +356,54 @@ WarpX::MoveWindow (const int step, bool move_j)
}
}
- // Continuously inject plasma in new cells (by default only on level 0)
- if (WarpX::warpx_do_continuous_injection) {
-
- const int lev = 0;
-
- // particleBox encloses the cells where we generate particles
- // (only injects particles in an integer number of cells,
- // for correct particle spacing)
- amrex::RealBox particleBox = geom[lev].ProbDomain();
- amrex::Real new_injection_position;
- if (moving_window_v >= 0){
- // Forward-moving window
- const amrex::Real dx = geom[lev].CellSize(dir);
- new_injection_position = current_injection_position +
- std::floor( (geom[lev].ProbHi(dir) - current_injection_position)/dx ) * dx;
- } else {
- // Backward-moving window
- const amrex::Real dx = geom[lev].CellSize(dir);
- new_injection_position = current_injection_position -
- std::floor( (current_injection_position - geom[lev].ProbLo(dir))/dx) * dx;
- }
- // Modify the corresponding bounds of the particleBox
- if (moving_window_v >= 0) {
- particleBox.setLo( dir, current_injection_position );
- particleBox.setHi( dir, new_injection_position );
- } else {
- particleBox.setLo( dir, new_injection_position );
- particleBox.setHi( dir, current_injection_position );
- }
+ // Loop over species
+ const int n_species = mypc->nSpecies();
+ for (int i=0; iGetParticleContainer(i);
- if (particleBox.ok() and (current_injection_position != new_injection_position)){
- // Performs continuous injection of all WarpXParticleContainer
- // in mypc.
- mypc->ContinuousInjection(particleBox);
- current_injection_position = new_injection_position;
+ // Continuously inject plasma in new cells (by default only on level 0)
+ if (pc.doContinuousInjection())
+ {
+ const int lev = 0;
+
+ // particleBox encloses the cells where we generate particles
+ // (only injects particles in an integer number of cells,
+ // for correct particle spacing)
+ amrex::RealBox particleBox = geom[lev].ProbDomain();
+ amrex::Real new_injection_position = pc.m_current_injection_position;
+ if (moving_window_v > 0._rt)
+ {
+ // Forward-moving window
+ const amrex::Real dx = geom[lev].CellSize(dir);
+ new_injection_position = pc.m_current_injection_position +
+ std::floor( (geom[lev].ProbHi(dir) - pc.m_current_injection_position)/dx ) * dx;
+ }
+ else if (moving_window_v < 0._rt)
+ {
+ // Backward-moving window
+ const amrex::Real dx = geom[lev].CellSize(dir);
+ new_injection_position = pc.m_current_injection_position -
+ std::floor( (pc.m_current_injection_position - geom[lev].ProbLo(dir))/dx) * dx;
+ }
+ // Modify the corresponding bounds of the particleBox
+ if (moving_window_v > 0._rt)
+ {
+ particleBox.setLo( dir, pc.m_current_injection_position );
+ particleBox.setHi( dir, new_injection_position );
+ }
+ else if (moving_window_v < 0._rt)
+ {
+ particleBox.setLo( dir, new_injection_position );
+ particleBox.setHi( dir, pc.m_current_injection_position );
+ }
+
+ if (particleBox.ok() and (pc.m_current_injection_position != new_injection_position)){
+ // Performs continuous injection of all WarpXParticleContainer
+ // in mypc.
+ pc.ContinuousInjection(particleBox);
+ pc.m_current_injection_position = new_injection_position;
+ }
}
}
diff --git a/Source/Utils/WarpXUtil.cpp b/Source/Utils/WarpXUtil.cpp
index 7e486eb98df..c65603f4d02 100644
--- a/Source/Utils/WarpXUtil.cpp
+++ b/Source/Utils/WarpXUtil.cpp
@@ -322,14 +322,15 @@ void CheckGriddingForRZSpectral ()
Vector max_grid_size_x(max_level+1);
// Set the radial block size to be the power of 2 greater than or equal to
- // the number of grid cells. The blocking_factor must be a power of 2
- // and the max_grid_size should be a multiple of the blocking_factor.
+ // the number of grid cells. The blocking factor must be a power of 2
+ // and the max_grid_size must be a multiple of the blocking_factor unless
+ // it is less than the blocking factor.
int k = 1;
while (k < n_cell[0]) {
k *= 2;
}
blocking_factor_x[0] = k;
- max_grid_size_x[0] = k;
+ max_grid_size_x[0] = n_cell[0];
for (int lev=1 ; lev <= max_level ; lev++) {
// For this to be correct, this needs to read in any user specified refinement ratios.
diff --git a/Source/WarpX.H b/Source/WarpX.H
index fb5e403f432..dbe922812a3 100644
--- a/Source/WarpX.H
+++ b/Source/WarpX.H
@@ -868,7 +868,6 @@ public:
amrex::Real getdt (int lev) const {return dt[lev];}
int getdo_moving_window() const {return do_moving_window;}
amrex::Real getmoving_window_x() const {return moving_window_x;}
- amrex::Real getcurrent_injection_position () const {return current_injection_position;}
bool getis_synchronized() const {return is_synchronized;}
int maxStep () const {return max_step;}
@@ -1470,7 +1469,6 @@ private:
amrex::Real v_particle_pml;
amrex::Real moving_window_x = std::numeric_limits::max();
- amrex::Real current_injection_position = 0;
// Plasma injection parameters
int warpx_do_continuous_injection = 0;
diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp
index 17601175bc5..fc37ccc6523 100644
--- a/Source/WarpX.cpp
+++ b/Source/WarpX.cpp
@@ -279,16 +279,25 @@ WarpX::WarpX ()
t_old.resize(nlevs_max, std::numeric_limits::lowest());
dt.resize(nlevs_max, std::numeric_limits::max());
- // Particle Container
+ // Loop over species and set current injection position per species
mypc = std::make_unique(this);
- warpx_do_continuous_injection = mypc->doContinuousInjection();
- if (warpx_do_continuous_injection){
- if (moving_window_v >= 0){
+ const int n_species = mypc->nSpecies();
+ for (int i=0; iGetParticleContainer(i);
+
+ // Storing injection position for all species, regardless of whether
+ // they are continuously injected, since it makes looping over the
+ // elements of current_injection_position easier elsewhere in the code.
+ if (moving_window_v > 0._rt)
+ {
// Inject particles continuously from the right end of the box
- current_injection_position = geom[0].ProbHi(moving_window_dir);
- } else {
+ pc.m_current_injection_position = geom[0].ProbHi(moving_window_dir);
+ }
+ else if (moving_window_v < 0._rt)
+ {
// Inject particles continuously from the left end of the box
- current_injection_position = geom[0].ProbLo(moving_window_dir);
+ pc.m_current_injection_position = geom[0].ProbLo(moving_window_dir);
}
}
diff --git a/Source/ablastr/coarsen/average.H b/Source/ablastr/coarsen/average.H
index 269403f7b2c..8f484a18709 100644
--- a/Source/ablastr/coarsen/average.H
+++ b/Source/ablastr/coarsen/average.H
@@ -7,16 +7,13 @@
#ifndef ABLASTR_COARSEN_AVERAGE_H_
#define ABLASTR_COARSEN_AVERAGE_H_
-
#include
#include
#include
#include
#include
-#include
#include
#include
-
#include
#include
diff --git a/Source/ablastr/coarsen/average.cpp b/Source/ablastr/coarsen/average.cpp
index 021df06b4cf..a5dde3d8aa4 100644
--- a/Source/ablastr/coarsen/average.cpp
+++ b/Source/ablastr/coarsen/average.cpp
@@ -9,15 +9,16 @@
#include "ablastr/utils/TextMsg.H"
#include
-#include
#include
#include
#include
#include
+#include
#include
#include
#include
+#include
namespace ablastr::coarsen::average
{
diff --git a/Source/ablastr/coarsen/sample.H b/Source/ablastr/coarsen/sample.H
index 1390cbebb3c..0a338a1c464 100644
--- a/Source/ablastr/coarsen/sample.H
+++ b/Source/ablastr/coarsen/sample.H
@@ -10,7 +10,6 @@
#include
#include
-#include
#include
#include
#include
diff --git a/Source/ablastr/coarsen/sample.cpp b/Source/ablastr/coarsen/sample.cpp
index 5d6973b21b5..65ada612905 100644
--- a/Source/ablastr/coarsen/sample.cpp
+++ b/Source/ablastr/coarsen/sample.cpp
@@ -9,15 +9,20 @@
#include "ablastr/utils/TextMsg.H"
#include
-#include
#include
#include
+#include
+#include
+#include
#include
#include
+#include
#include
#include
#include
+#include
+
namespace ablastr::coarsen::sample
{
diff --git a/Source/ablastr/parallelization/MPIInitHelpers.cpp b/Source/ablastr/parallelization/MPIInitHelpers.cpp
index cdf45821356..65e7525c087 100644
--- a/Source/ablastr/parallelization/MPIInitHelpers.cpp
+++ b/Source/ablastr/parallelization/MPIInitHelpers.cpp
@@ -10,7 +10,6 @@
#include
#include
-#include
#if defined(AMREX_USE_MPI)
# include
diff --git a/Source/ablastr/utils/Communication.H b/Source/ablastr/utils/Communication.H
index 2c879460490..1653c0a1a0d 100644
--- a/Source/ablastr/utils/Communication.H
+++ b/Source/ablastr/utils/Communication.H
@@ -7,16 +7,13 @@
#ifndef ABLASTR_UTILS_COMMUNICATION_H_
#define ABLASTR_UTILS_COMMUNICATION_H_
-#include
-#include
+#include
#include
-#include
-#include
-#include
+#include
#include
-#include
+#include
-#include "WarpX.H"
+#include
#include
diff --git a/Source/ablastr/utils/Communication.cpp b/Source/ablastr/utils/Communication.cpp
index ac1dd1331fe..89579ba1ee5 100644
--- a/Source/ablastr/utils/Communication.cpp
+++ b/Source/ablastr/utils/Communication.cpp
@@ -6,15 +6,21 @@
*/
#include "Communication.H"
-#include
#include
#include
#include
#include
+#include
+#include
#include
#include
+#include
#include
+#include
+#include
+#include
+
namespace ablastr::utils::communication
{
diff --git a/Source/ablastr/utils/Serialization.H b/Source/ablastr/utils/Serialization.H
index 7fb10c14fde..5db3e9968b4 100644
--- a/Source/ablastr/utils/Serialization.H
+++ b/Source/ablastr/utils/Serialization.H
@@ -11,6 +11,7 @@
#include
#include
#include
+#include
#include
#include
#include
diff --git a/Source/ablastr/utils/SignalHandling.cpp b/Source/ablastr/utils/SignalHandling.cpp
index 36ab50f4c3a..c34d760be04 100644
--- a/Source/ablastr/utils/SignalHandling.cpp
+++ b/Source/ablastr/utils/SignalHandling.cpp
@@ -8,11 +8,11 @@
#include "SignalHandling.H"
#include "TextMsg.H"
-#include
#include
#include
#include
+#include
// For sigaction() et al.
#if defined(__linux__) || defined(__APPLE__)
diff --git a/Source/ablastr/utils/TextMsg.H b/Source/ablastr/utils/TextMsg.H
index 6fa7b28a982..89bbf500395 100644
--- a/Source/ablastr/utils/TextMsg.H
+++ b/Source/ablastr/utils/TextMsg.H
@@ -9,8 +9,6 @@
#define ABLASTR_TEXT_MSG_H_
#include
-#include
-
namespace ablastr::utils::TextMsg
{
diff --git a/Source/ablastr/utils/TextMsg.cpp b/Source/ablastr/utils/TextMsg.cpp
index 324b0752554..fb8e4894792 100644
--- a/Source/ablastr/utils/TextMsg.cpp
+++ b/Source/ablastr/utils/TextMsg.cpp
@@ -14,7 +14,7 @@
#include
#include
#include
-#include
+#include
namespace
diff --git a/Source/ablastr/utils/UsedInputsFile.cpp b/Source/ablastr/utils/UsedInputsFile.cpp
index 175c67619e7..dfdc4bfa192 100644
--- a/Source/ablastr/utils/UsedInputsFile.cpp
+++ b/Source/ablastr/utils/UsedInputsFile.cpp
@@ -11,7 +11,6 @@
#include
#include
-#include
#include
#include
diff --git a/Source/ablastr/utils/msg_logger/MsgLogger.H b/Source/ablastr/utils/msg_logger/MsgLogger.H
index 451f90d3dd7..d6683a1ffc1 100644
--- a/Source/ablastr/utils/msg_logger/MsgLogger.H
+++ b/Source/ablastr/utils/msg_logger/MsgLogger.H
@@ -8,7 +8,7 @@
#ifndef ABLASTR_MSG_LOGGER_H_
#define ABLASTR_MSG_LOGGER_H_
-#include
+#include
#include
#include