diff --git a/_sources/olcf-compilers.rst.txt b/_sources/olcf-compilers.rst.txt index a233083..a38c139 100644 --- a/_sources/olcf-compilers.rst.txt +++ b/_sources/olcf-compilers.rst.txt @@ -3,58 +3,6 @@ Compiling at OLCF ================= -Summit ------- - -In order to compile you will need to swap the xl module with gcc (you need to use atleast gcc/7.4.0 due to C++17 support): - -.. prompt:: bash - - module load gcc/10.2.0 - -.. note:: - - You will need to load the same module you use for compiling in your - submissions script, otherwise the code won't find the shared - libraries at runtime. - -Then load CUDA: - -.. prompt:: bash - - module load cuda/11.5.2 - -.. note:: - - Presently you will see a warning when you load a CUDA 11 module, but the packages - should work fine. - -You also need to make sure you have the python module loaded: - -.. prompt:: bash - - module load python - -Compile with ``USE_CUDA=TRUE`` (and ``COMP=gnu`` which is usually the default). -Is important to setup ``USE_OMP=FALSE``, since the ``TRUE`` option is currently disallowed by Castro. -An example compilation line is: - -.. prompt:: bash - - make COMP=gnu USE_MPI=TRUE USE_CUDA=TRUE -j 4 - -The recommended/tested version pairs are: - - * ``gcc/10.2.0`` + ``cuda/11.5.2`` - -.. note:: - - - OpenMP offloading to the GPU is controlled by - ``USE_OMP_OFFLOAD``, which will default to ``FALSE``, AMReX-Astro - doesn't use this feature. - - - Frontier -------- diff --git a/_sources/olcf-workflow.rst.txt b/_sources/olcf-workflow.rst.txt index cc18d32..dc77cf6 100644 --- a/_sources/olcf-workflow.rst.txt +++ b/_sources/olcf-workflow.rst.txt @@ -4,434 +4,6 @@ Managing Jobs at OLCF ===================== - -Summit ------- - -Summit Architecture: -^^^^^^^^^^^^^^^^^^^^ - -Let us start by reviewing the node architecture of Summit. Our goal is to provide the necessary insight to make better -decisions in the construction of our particular AMReX-Astro job scripts, and to explain how our code interacts with Summit. -All the exposed information in this section is a condensed version of the `Summit documentation guide -`_, and should not replace it. - -In Summit, a node is composed by two sockets: each one with 21 CPU physical cores (+1 reserved for the system), 3 GPUs and 1 RAM memory bank. -The sockets are connected by a bus allowing communication among them. Each CPU physical core may define up to 4 threads. -The whole structure of the node can be depicted as follows: - -.. figure:: ./figs/summit-node-description-1.png - :width: 100% - :align: center - - Figure extracted from ``https://docs.olcf.ornl.gov/systems/summit_user_guide.html#job-launcher-jsrun``. - -A resource set is a minimal collection of CPU physical cores and GPUs, on which a certain number of MPI processes and OpenMP -threads operates through the execution of the code. Therefore, for each resource set, we need to allocate: - -- A number of CPU physical cores. - -- A number of physical GPUs. - -- A number of MPI processes. - -- The number of OpenMP threads. - -where each core supports up to 4 threads; however, this option is not supported in AMReX and we will not extend our -discussion here. For now, we fix just only one thread through the whole execution of our code. The next step is to determine the maximum -number of resource sets that may fit into one node. - -In Castro we construct each resource set with: 1 CPU physical core, 1 GPU, and only 1 MPI process. -The next step is to see how many resources sets fits into one node. According to the node architecture depicted in Figure 1, -we can fit up to 6 resource sets per node as in Figure 2. - -.. figure:: ./figs/image56.png - :width: 100% - :align: center - - Figure modified and extracted from ``https://docs.olcf.ornl.gov/systems/summit_user_guide.html#job-launcher-jsrun``. - -Requesting Allocation: -^^^^^^^^^^^^^^^^^^^^^^ - -To allocate the resource sets we need to summon the command ``bsub`` in addition to some flags: - -.. list-table:: - :widths: 25 75 - :header-rows: 1 - - * - Flag - - Description - - * - ``-nnodes`` - - allocates the number of nodes we need to run our code. Is important to perform the calculation - described in the previous section to select the correct number of nodes in our setup. - - - * - ``-W`` - - allocates the walltime of the selected nodes. The format we use in Summit is [hours:]minutes, there is - no room for seconds in Summit. The maximum walltime that we can request is 03:00 (three hours). - - - * - ``-alloc_flags`` - - allocates the maximum number of threads available per CPU core. By default the option is ``smt4`` that - allows 4 threads per core. However, since we will consider only one thread through the whole execution - we will setup the option ``smt1``. Also ``-alloc_flags`` stands for more options, however we are only - interested in the one discussed before. - - - * - ``-J`` - - defines the name of the allocation. The value ``%J`` correspond to the allocation ID number. - - * - ``-o`` - - defines the **output name that contains the standard output stream**, after running all the jobs inside the requested - allocation. - - * - ``-e`` - - defines the **output file name containing the standard error stream**, similar to the ``-o`` flag. If ``-e`` is not supplied, then - the ``-o`` option is assumed by default. - - * - ``-q`` - - defines the queue on which our application will run. There are several options, however, we alternate - between two options: the standard production queue ``batch`` and the debugging queue ``debug``. The ``debug`` queue is designed - to allocate an small number of nodes in order to see that our code is running smoothly without bugs. - - * - ``-Is`` - - flags for interactive job followed by the shell name. The Unix bash shell option is ``/bin/bash``. This flag is very useful - for debugging, because the standard output can be checked as the code is running. Is important to mention that any interactive - job can only be summoned by command line and not by running a bash script. - -For example, if we want to allocate one node to run an interactive job in the debug queue for 30 minutes we may setup: - -.. prompt:: bash - - bsub -nnodes 1 -q debug -W 0:30 -P ast106 -alloc_flags smt1 -J example -o stdout_to_show.%J -e stderr_to_show.%J -Is /bin/bash - -.. note:: - - An interactive job can only be allocated by the use of the command line. No script can be defined for interactive jobs. - - -Submitting a Job: -^^^^^^^^^^^^^^^^^ - -Once our allocation is granted, is important to load the same modules used in the compilation process of the executable and -export the variable ``OMP_NUM_THREADS`` to setup the number of threads per MPI process. - -In Castro, we have used the following modules: - -.. code-block:: - - module load gcc/10.2.0 - module load cuda/11.5.2 - module load python - -and fixed only one thread per MPI process by: - -.. code-block:: - - export OMP_NUM_THREADS=1 - -The next step is to submit our job. The command `jsrun`, provided with the *total number of resource sets*, the -*number of CPU physical cores per resource set*, *the number of GPUs per resource set*, *the number of MPI processes allocated per resource set*, -works as follows: - -.. prompt:: bash - - jsrun -n[number of resource sets] -c[number of CPU physical cores] -g[number of GPUs] -a[number of MPI processes] -r[number of max resources per node] ./[executable] [executable inputs] - -In Castro we will use: - -.. prompt:: bash - - jsrun -n [number of resource sets] -a1 -c1 -g1 -r6 ./$CASTRO $INPUTS - -where the ``CASTRO`` and ``INPUTS`` environment variables are placeholders to the executable and input file names respectively. - -Now, in order to use all the resources we have allocated to run our jobs, the number of resource sets should match the number of AMReX boxes (grids) -of the corresponding level with the biggest number of them. Let us consider an extract piece from a Castro problem standard output: - -.. code-block:: - - INITIAL GRIDS - Level 0 2 grids 32768 cells 100 % of domain - smallest grid: 128 x 128 biggest grid: 128 x 128 - Level 1 8 grids 131072 cells 100 % of domain - smallest grid: 128 x 128 biggest grid: 128 x 128 - Level 2 8 grids 524288 cells 100 % of domain - smallest grid: 256 x 256 biggest grid: 256 x 256 - Level 3 32 grids 2097152 cells 100 % of domain - smallest grid: 256 x 256 biggest grid: 256 x 256 - Level 4 128 grids 7864320 cells 93.75 % of domain - smallest grid: 256 x 128 biggest grid: 256 x 256 - Level 5 480 grids 30408704 cells 90.625 % of domain - smallest grid: 256 x 128 biggest grid: 256 x 256 - -In this example, Level 5 contains the biggest number of AMReX boxes: 480. From here, we may assert that a good allocation for this problem are -480 resource sets, equivalent to 80 nodes by setting 6 resources per node. However, note that that Level 0 uses only 2 AMReX boxes, this implies that -from the 480 resources available, 398 resources will remain idle until the two working processes sweep the entire Level 0. - -.. note:: - - Therefore, is important, if possible, to keep the number of boxes - on each level balanced to maximize the use of the allocated resources. - -Writting a Job Script: -^^^^^^^^^^^^^^^^^^^^^^ - -In order to make our life easier, instead of submitting an allocation -command line, loading the modules, setting the threads/MPI process, -and writing another command line to submit our jobs, we can make an -script to pack all these command into one executable ``.sh`` file, -that can be submitted via ``bsub`` just once. - -We start our job script, summoning the shell with the statement -``!/bin/bash``. Then we add the ``bsub`` allocations flags, starting -with ``#BSUB`` as follows: - -.. code-block:: bash - - #!/bin/bash - #BSUB -P ast106 - #BSUB -W 2:00 - #BSUB -nnodes 80 - #BSUB -alloc_flags smt1 - #BSUB -J luna_script - #BSUB -o luna_output.%J - #BSUB -e luna_sniffing_output.%J - -In addition we add the modules statements, fixing only one thread per MPI process: - -.. code-block:: - - module load gcc/10.2.0 - module load cuda/11.5.2 - module load python - - export OMP_NUM_THREADS=1 - -and define the environment variables: - -.. code-block:: - - CASTRO=./Castro2d.gnu.MPI.CUDA.ex - INPUTS=inputs_luna - - n_res=480 # The max allocated number of resource sets is - n_cpu_cores_per_res=1 # nnodes * n_max_res_per_node. In this case we will - n_mpi_per_res=1 # use all the allocated resource sets to run the job - n_gpu_per_res=1 # below. - n_max_res_per_node=6 - -Once the allocation ends, the job is downgraded/killed, leaving us as we started. As we pointed out, the maximum allocation -time in Summit is 03:00 (three hours), but, we may need sometimes weeks, months, or maybe years to complete -our runs. Now is when the automatic restarting section of the script comes to our salvation. - -From here we can add an optional (or mandatory) setting to our script. As the code executes, -after a certain number of timesteps, the code creates checkpoint files of the form ``chkxxxxxxx``, ``chkxxxxxx`` -or ``chkxxxxx``. This checkpoint files can be read by our executable and run from the simulation time where -the checkpoint was created. This is implemented as follows: - -.. code-block:: - - function find_chk_file { - # find_chk_file takes a single argument -- the wildcard pattern - # for checkpoint files to look through - chk=$1 - - # find the latest 2 restart files. This way if the latest didn't - # complete we fall back to the previous one. - temp_files=$(find . -maxdepth 1 -name "${chk}" -print | sort | tail -2) - restartFile="" - for f in ${temp_files} - do - # the Header is the last thing written -- if it's there, update the restart file - if [ -f ${f}/Header ]; then - restartFile="${f}" - fi - done - } - - # look for 7-digit chk files - find_chk_file "*chk???????" - - if [ "${restartFile}" = "" ]; then - # look for 6-digit chk files - find_chk_file "*chk??????" - fi - - if [ "${restartFile}" = "" ]; then - # look for 5-digit chk files - find_chk_file "*chk?????" - fi - - # restartString will be empty if no chk files are found -- i.e. new run - if [ "${restartFile}" = "" ]; then - restartString="" - else - restartString="amr.restart=${restartFile}" - fi - -The function ``find_chk_file`` searches the submission directory for -checkpoint files. Because AMReX appends digits as the number of steps -increase (with a minimum of 5 digits), we search for files with -7-digits, 6-digits, and then finally 5-digits, to ensure we pick up -the latest file. - -We can also ask the job manager to send a warning signal some amount -of time before the allocation expires by passing ``-wa 'signal'`` and -``-wt '[hour:]minute'`` to ``bsub``. We can then have bash create a -``dump_and_stop`` file when it receives the signal, which will tell -Castro to output a checkpoint file and exit cleanly after it finishes -the current timestep. An important detail that I couldn't find -documented anywhere is that the job manager sends the signal to all -the processes in the job, not just the submission script, and we have -to use a signal that is ignored by default so Castro doesn't -immediately crash upon receiving it. SIGCHLD, SIGURG, and SIGWINCH -are the only signals that fit this requirement and of these, SIGURG is -the least likely to be triggered by other events. - -.. code-block:: bash - - #BSUB -wa URG - #BSUB -wt 2 - - ... - - function sig_handler { - touch dump_and_stop - # disable this signal handler - trap - URG - echo "BATCH: allocation ending soon; telling Castro to dump a checkpoint and stop" - } - trap sig_handler URG - -We use the ``jsrun`` command to launch Castro on the compute nodes. In -order for bash to handle the warning signal before Castro exits, we -must put ``jsrun`` in the background and use the shell builtin -``wait``: - -.. code-block:: bash - - jsrun -n$n_res -c$n_cpu_cores_per_res -a$n_mpi_per_res -g$n_gpu_per_res -r$n_max_res_per_node $CASTRO $INPUTS ${restartString} & - wait - # use jswait to wait for Castro (job step 1/1) to finish and get the exit code - jswait 1 - -Finally, once the script is completed and saved as ``luna_script.sh``, we can submit it by: - -.. prompt:: bash - - bsub luna_script.sh - - -Monitoring a Job: -^^^^^^^^^^^^^^^^^ - -You can monitor the status of your jobs using ``bjobs``. Also, a slightly nicer view of your jobs can be viewed using ``jobstat`` as: - -.. prompt:: bash - - jobstat -u username - - -Script Template: -^^^^^^^^^^^^^^^^ - -Packing all the information before, lead us to the following script template - -.. literalinclude:: ../../job_scripts/summit/summit_template.sh - :language: sh - :linenos: - - -Chaining jobs -^^^^^^^^^^^^^ - -The script ``job_scripts/summit/chain_submit.sh`` can be used to setup job dependencies, -i.e., a job chain. - -First you submit a job as usual using ``bsub``, and make note of the -job-id that it prints upon submission (the same id you would see with -``bjobs`` or ``jobstat``). Then you setup N jobs to depend on the one -you just submitted as: - -.. prompt:: bash - - chain_submit.sh job-id N submit_script.sh - -where you replace ``job-id`` with the id return from your first -submission, replace ``N`` with the number of additional jobs, and -replace ``submit_script`` with the name of the script you use to -submit the job. This will queue up N additional jobs, each depending -on the previous. Your submission script should use the automatic -restarting features discussed above. - - - -Archiving to HPSS ------------------ - -You can access HPSS from submit using the data transfer nodes by submitting a job -via SLURM: - -.. prompt:: bash - - sbatch -N 1 -t 15:00 -A ast106 --cluster dtn test_hpss.sh - -where ``test_hpss.sh`` is a SLURM script that contains the ``htar`` -commands needed to archive your data. This uses ``slurm`` as the job -manager. - -An example is provided by the ``process.xrb`` archiving script in -``job_scripts/hpss/`` and associated ``summit_hpss.submit`` submission script -in ``jobs_scripts/summit/``. Together these will detect new plotfiles as they -are generated, tar them up (using ``htar``) and archive them onto HPSS. They -will also store the inputs, probin, and other runtime generated files. If -``ftime`` is found in your path, it will also create a file called -``ftime.out`` that lists the simulation time corresponding to each plotfile. - -Once the plotfiles are archived they are moved to a subdirectory under -your run directory called ``plotfiles/``. - -By default, the files will be archived to a directory in HPSS with the same -name as the directory your plotfiles are located in. This can be changed -by editing the ``$HPSS_DIR`` variable at the top of ``process.xrb``. - - -To use this, we do the following: - -#. Copy the ``process.xrb`` and ``summit_hpss.submit`` scripts into the - directory with the plotfiles. - -#. Launch the script via: - - .. prompt:: bash - - sbatch summit_hpss.submit - - It will run for the full time you asked, searching for plotfiles as - they are created and moving them to HPSS as they are produced (it - will always leave the very last plotfile alone, since it can't tell - if it is still being written). - - -Files may be unarchived in bulk from HPSS on OLCF systems using the -``hpss_xfer.py`` script, which is available in the job_scripts -directory. It requires Python 3 to be loaded to run. The command: - -.. prompt:: bash - - ./hpss_xfer.py plt00000 -s hpss_dir -o plotfile_dir - -will fetch ``hpss_dir/plt00000.tar`` from the HPSS filesystem and -unpack it in ``plotfile_dir``. If run with no arguments in the problem -launch directory, the script will attempt to recover all plotfiles -archived by ``process.titan``. Try running :code:`./hpss_xfer.py --help` -for a description of usage and arguments. - - - Frontier -------- diff --git a/_static/js/versions.js b/_static/js/versions.js index 818bc99..4958195 100644 --- a/_static/js/versions.js +++ b/_static/js/versions.js @@ -1,6 +1,6 @@ const themeFlyoutDisplay = "hidden"; -const themeVersionSelector = "True"; -const themeLanguageSelector = "True"; +const themeVersionSelector = true; +const themeLanguageSelector = true; if (themeFlyoutDisplay === "attached") { function renderLanguages(config) { @@ -8,10 +8,14 @@ if (themeFlyoutDisplay === "attached") { return ""; } + // Insert the current language to the options on the selector + let languages = config.projects.translations.concat(config.projects.current); + languages = languages.sort((a, b) => a.language.name.localeCompare(b.language.name)); + const languagesHTML = `
Languages
- ${config.projects.translations + ${languages .map( (translation) => `
diff --git a/olcf-compilers.html b/olcf-compilers.html index 001d4f1..3588a8d 100644 --- a/olcf-compilers.html +++ b/olcf-compilers.html @@ -52,7 +52,6 @@
  • Working at NERSC
  • Working at OLCF
    • Compiling at OLCF
        -
      • Summit
      • Frontier @@ -95,54 +94,16 @@

        Compiling at OLCF

        -
        -

        Summit

        -

        In order to compile you will need to swap the xl module with gcc (you need to use atleast gcc/7.4.0 due to C++17 support):

        -
        module load gcc/10.2.0
-
        -

        Note

        -

        You will need to load the same module you use for compiling in your -submissions script, otherwise the code won’t find the shared -libraries at runtime.

        -
        -

        Then load CUDA:

        -
        module load cuda/11.5.2
-
        -

        Note

        -

        Presently you will see a warning when you load a CUDA 11 module, but the packages -should work fine.

        -
        -

        You also need to make sure you have the python module loaded:

        -
        module load python
-

        Compile with USE_CUDA=TRUE (and COMP=gnu which is usually the default). -Is important to setup USE_OMP=FALSE, since the TRUE option is currently disallowed by Castro. -An example compilation line is:

        -
        make COMP=gnu USE_MPI=TRUE USE_CUDA=TRUE -j 4
-

        The recommended/tested version pairs are:

        -
        -
          -

        • gcc/10.2.0 + cuda/11.5.2

          • -
        -
        -
        -

        Note

        -
          -

        • OpenMP offloading to the GPU is controlled by -USE_OMP_OFFLOAD, which will default to FALSE, AMReX-Astro -doesn’t use this feature.

          • -
        -
        -

        Frontier

        log into: frontier.olcf.ornl.gov

        see: https://docs.olcf.ornl.gov/systems/frontier_user_guide.html#programming-environment

        load modules:

        -
        module load PrgEnv-gnu
+
        module load PrgEnv-gnu
 module load cray-mpich/8.1.28
 module load craype-accel-amd-gfx90a
 module load amd-mixed/6.1.3
diff --git a/olcf-workflow.html b/olcf-workflow.html
index baf7618..9038c30 100644
--- a/olcf-workflow.html
+++ b/olcf-workflow.html
@@ -53,17 +53,6 @@
 
      • Working at OLCF
          
 
        • Compiling at OLCF
          • 
 
        • Managing Jobs at OLCF
            
-
          • Summit
-
            • 
-
          • Archiving to HPSS
            • 
 
          • Frontier
              
 
            • Machine details
              • 
 
            • Submitting jobs
              • 
@@ -111,443 +100,6 @@
              
   
              
 
              Managing Jobs at OLCF
              
-
              
-
              Summit
              
-
              
-
              Summit Architecture:
              
-
              Let us start by reviewing the node architecture of Summit. Our goal is to provide the necessary insight to make better
-decisions in the construction of our particular AMReX-Astro job scripts, and to explain how our code interacts with Summit.
-All the exposed information in this section is a condensed version of the Summit documentation guide, and should not replace it.
              
-
              In Summit, a node is composed by two sockets: each one with 21 CPU physical cores (+1 reserved for the system), 3 GPUs and 1 RAM memory bank.
-The sockets are connected by a bus allowing communication among them. Each CPU physical core may define up to 4 threads.
-The whole structure of the node can be depicted as follows:
              
-
              
-_images/summit-node-description-1.png
-
-
              
-
              Figure extracted from https://docs.olcf.ornl.gov/systems/summit_user_guide.html#job-launcher-jsrun.
              
-
              
-
              
-
              A resource set is a minimal collection of CPU physical cores and GPUs, on which a certain number of MPI processes and OpenMP
-threads operates through the execution of the code. Therefore, for each resource set, we need to allocate:
              
-
                
-
              • A number of CPU physical cores.
                • 
-
              • A number of physical GPUs.
                • 
-
              • A number of MPI processes.
                • 
-
              • The number of OpenMP threads.
                • 
-
              
-
              where each core supports up to 4 threads; however, this option is not supported in AMReX and we will not extend our
-discussion here. For now, we fix just only one thread through the whole execution of our code. The next step is to determine the maximum
-number of resource sets that may fit into one node.
              
-
              In Castro we construct each resource set with: 1 CPU physical core, 1 GPU, and only 1 MPI process.
-The next step is to see how many resources sets fits into one node. According to the node architecture depicted in Figure 1,
-we can fit up to 6 resource sets per node as in Figure 2.
              
-
              
-_images/image56.png
-
-
              
-
              Figure modified and extracted from https://docs.olcf.ornl.gov/systems/summit_user_guide.html#job-launcher-jsrun.
              
-
              
-
              
-
              
-
              
-
              Requesting Allocation:
              
-
              To allocate the resource sets we need to summon the command bsub in addition to some flags:
              
-
----
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
              



              Flag
              Description
              -nnodes
              allocates the number of nodes we need to run our code. Is important to perform the calculation
-described in the previous section to select the correct number of nodes in our setup.
              -W
              allocates the walltime of the selected nodes. The format we use in Summit is [hours:]minutes, there is
-no room for seconds in Summit. The maximum walltime that we can request is 03:00 (three hours).
              -alloc_flags
              allocates the maximum number of threads available per CPU core. By default the option is smt4 that
-allows 4 threads per core. However, since we will consider only one thread through the whole execution
-we will setup the option smt1. Also -alloc_flags stands for more options, however we are only
-interested in the one discussed before.
              -J
              defines the name of the allocation. The value %J correspond to the allocation ID number.
              -o
              defines the output name that contains the standard output stream, after running all the jobs inside the requested
-allocation.
              -e
              defines the output file name containing the standard error stream, similar to the -o flag. If -e is not supplied, then
-the -o option is assumed by default.
              -q
              defines the queue on which our application will run. There are several options, however, we alternate
-between two options: the standard production queue batch and the debugging queue debug. The debug queue is designed
-to allocate an small number of nodes in order to see that our code is running smoothly without bugs.
              -Is
              flags for interactive job followed by the shell name. The Unix bash shell option is /bin/bash. This flag is very useful
-for debugging, because the standard output can be checked as the code is running. Is important to mention that any interactive
-job can only be summoned by command line and not by running a bash script.
              
-
              For example, if we want to allocate one node to run an interactive job in the debug queue for 30 minutes we may setup:
              
-
              bsub -nnodes 1 -q debug -W 0:30 -P ast106 -alloc_flags smt1 -J example -o stdout_to_show.%J -e stderr_to_show.%J -Is /bin/bash
-
              
-
              Note
              
-
              An interactive job can only be allocated by the use of the command line. No script can be defined for interactive jobs.
              
-
              
-
              
-
              
-
              Submitting a Job:
              
-
              Once our allocation is granted, is important to load the same modules used in the compilation process of the executable and
-export the variable OMP_NUM_THREADS to setup the number of threads per MPI process.
              
-
              In Castro, we have used the following modules:
              
-
              module load gcc/10.2.0
-module load cuda/11.5.2
-module load python
-
              
-
              
-
              and fixed only one thread per MPI process by:
              
-
              export OMP_NUM_THREADS=1
-
              
-
              
-
              The next step is to submit our job. The command jsrun, provided with the total number of resource sets, the
-number of CPU physical cores per resource set, the number of GPUs per resource set, the number of MPI processes allocated per resource set,
-works as follows:
              
-
              jsrun -n[number of resource sets] -c[number of CPU physical cores] -g[number of GPUs] -a[number of MPI processes] -r[number of max resources per node] ./[executable] [executable inputs]
-
              In Castro we will use:
              
-
              jsrun -n [number of resource sets] -a1 -c1 -g1 -r6 ./$CASTRO $INPUTS
-
              where the CASTRO and INPUTS environment variables are placeholders to the executable and input file names respectively.
              
-
              Now, in order to use all the resources we have allocated to run our jobs, the number of resource sets should match the number of AMReX boxes (grids)
-of the corresponding level with the biggest number of them. Let us consider an extract piece from a Castro problem standard output:
              
-
              INITIAL GRIDS
-Level 0   2 grids  32768 cells  100 % of domain
-          smallest grid: 128 x 128  biggest grid: 128 x 128
-Level 1   8 grids  131072 cells  100 % of domain
-          smallest grid: 128 x 128  biggest grid: 128 x 128
-Level 2   8 grids  524288 cells  100 % of domain
-          smallest grid: 256 x 256  biggest grid: 256 x 256
-Level 3   32 grids  2097152 cells  100 % of domain
-          smallest grid: 256 x 256  biggest grid: 256 x 256
-Level 4   128 grids  7864320 cells  93.75 % of domain
-          smallest grid: 256 x 128  biggest grid: 256 x 256
-Level 5   480 grids  30408704 cells  90.625 % of domain
-          smallest grid: 256 x 128  biggest grid: 256 x 256
-
              
-
              
-
              In this example, Level 5 contains the biggest number of AMReX boxes: 480. From here, we may assert that a good allocation for this problem are
-480 resource sets, equivalent to 80 nodes by setting 6 resources per node. However, note that that Level 0 uses only 2 AMReX boxes, this implies that
-from the 480 resources available, 398 resources will remain idle until the two working processes sweep the entire Level 0.
              
-
              
-
              Note
              
-
              Therefore, is important, if possible, to keep the number of boxes
-on each level balanced to maximize the use of the allocated resources.
              
-
              
-
              
-
              
-
              Writting a Job Script:
              
-
              In order to make our life easier, instead of submitting an allocation
-command line, loading the modules, setting the threads/MPI process,
-and writing another command line to submit our jobs, we can make an
-script to pack all these command into one executable .sh file,
-that can be submitted via bsub just once.
              
-
              We start our job script, summoning the shell with the statement
-!/bin/bash. Then we add the bsub allocations flags, starting
-with #BSUB as follows:
              
-
              #!/bin/bash
-#BSUB -P ast106
-#BSUB -W 2:00
-#BSUB -nnodes 80
-#BSUB -alloc_flags smt1
-#BSUB -J luna_script
-#BSUB -o luna_output.%J
-#BSUB -e luna_sniffing_output.%J
-
              
-
              
-
              In addition we add the modules statements, fixing only one thread per MPI process:
              
-
              module load gcc/10.2.0
-module load cuda/11.5.2
-module load python
-
-export OMP_NUM_THREADS=1
-
              
-
              
-
              and define the environment variables:
              
-
              CASTRO=./Castro2d.gnu.MPI.CUDA.ex
-INPUTS=inputs_luna
-
-n_res=480                # The max allocated number of resource sets is
-n_cpu_cores_per_res=1    # nnodes * n_max_res_per_node. In this case we will
-n_mpi_per_res=1          # use all the allocated resource sets to run the job
-n_gpu_per_res=1          # below.
-n_max_res_per_node=6
-
              
-
              
-
              Once the allocation ends, the job is downgraded/killed, leaving us as we started. As we pointed out, the maximum allocation
-time in Summit is 03:00 (three hours), but, we may need sometimes weeks, months, or maybe years to complete
-our runs. Now is when the automatic restarting section of the script comes to our salvation.
              
-
              From here we can add an optional (or mandatory) setting to our script. As the code executes,
-after a certain number of timesteps, the code creates checkpoint files of the form chkxxxxxxx, chkxxxxxx
-or chkxxxxx. This checkpoint files can be read by our executable and run from the simulation time where
-the checkpoint was created. This is implemented as follows:
              
-
              function find_chk_file {
-   # find_chk_file takes a single argument -- the wildcard pattern
-   # for checkpoint files to look through
-   chk=$1
-
-   # find the latest 2 restart files.  This way if the latest didn't
-   # complete we fall back to the previous one.
-   temp_files=$(find . -maxdepth 1 -name "${chk}" -print | sort | tail -2)
-   restartFile=""
-   for f in ${temp_files}
-   do
-      # the Header is the last thing written -- if it's there, update the restart file
-      if [ -f ${f}/Header ]; then
-         restartFile="${f}"
-      fi
-   done
-}
-
-# look for 7-digit chk files
-find_chk_file "*chk???????"
-
-if [ "${restartFile}" = "" ]; then
-   # look for 6-digit chk files
-   find_chk_file "*chk??????"
-fi
-
-if [ "${restartFile}" = "" ]; then
-   # look for 5-digit chk files
-   find_chk_file "*chk?????"
-fi
-
-# restartString will be empty if no chk files are found -- i.e. new run
-if [ "${restartFile}" = "" ]; then
-   restartString=""
-else
-   restartString="amr.restart=${restartFile}"
-fi
-
              
-
              
-
              The function find_chk_file searches the submission directory for
-checkpoint files.  Because AMReX appends digits as the number of steps
-increase (with a minimum of 5 digits), we search for files with
-7-digits, 6-digits, and then finally 5-digits, to ensure we pick up
-the latest file.
              
-
              We can also ask the job manager to send a warning signal some amount
-of time before the allocation expires by passing -wa 'signal' and
--wt '[hour:]minute' to bsub.  We can then have bash create a
-dump_and_stop file when it receives the signal, which will tell
-Castro to output a checkpoint file and exit cleanly after it finishes
-the current timestep.  An important detail that I couldn’t find
-documented anywhere is that the job manager sends the signal to all
-the processes in the job, not just the submission script, and we have
-to use a signal that is ignored by default so Castro doesn’t
-immediately crash upon receiving it.  SIGCHLD, SIGURG, and SIGWINCH
-are the only signals that fit this requirement and of these, SIGURG is
-the least likely to be triggered by other events.
              
-
              #BSUB -wa URG
-#BSUB -wt 2
-
-...
-
-function sig_handler {
-   touch dump_and_stop
-   # disable this signal handler
-   trap - URG
-   echo "BATCH: allocation ending soon; telling Castro to dump a checkpoint and stop"
-}
-trap sig_handler URG
-
              
-
              
-
              We use the jsrun command to launch Castro on the compute nodes. In
-order for bash to handle the warning signal before Castro exits, we
-must put jsrun in the background and use the shell builtin
-wait:
              
-
              jsrun -n$n_res -c$n_cpu_cores_per_res -a$n_mpi_per_res -g$n_gpu_per_res -r$n_max_res_per_node $CASTRO $INPUTS ${restartString} &
-wait
-# use jswait to wait for Castro (job step 1/1) to finish and get the exit code
-jswait 1
-
              
-
              
-
              Finally, once the script is completed and saved as luna_script.sh, we can submit it by:
              
-
              bsub luna_script.sh
-
              
-
              
-
              Monitoring a Job:
              
-
              You can monitor the status of your jobs using bjobs. Also, a slightly nicer view of your jobs can be viewed using jobstat as:
              
-
              jobstat -u username
-
              
-
              
-
              Script Template:
              
-
              Packing all the information before, lead us to the following script template
              
-
               1#!/bin/bash
- 2#BSUB -P ast106
- 3#BSUB -W 2:00
- 4#BSUB -nnodes 80
- 5#BSUB -alloc_flags smt1
- 6#BSUB -J luna_script
- 7#BSUB -o luna_output.%J
- 8#BSUB -e luna_sniffing_output.%J
- 9#BSUB -wa URG
-10#BSUB -wt 2
-11
-12module load gcc/10.2.0
-13module load cuda/11.5.2
-14module load python
-15
-16export OMP_NUM_THREADS=1
-17
-18CASTRO=./Castro2d.gnu.MPI.CUDA.ex
-19INPUTS=inputs_luna
-20
-21n_res=480                # The max allocated number of resource sets is
-22n_cpu_cores_per_res=1    # nnodes * n_max_res_per_node. In this case we will
-23n_mpi_per_res=1          # use all the allocated resource sets to run the job below,
-24n_gpu_per_res=1          # however we can define more enviroment variables to allocate two jobs
-25n_max_res_per_node=6     # simultaneous jobs, where n_res = n_res_1 + n_res2 allocates for two jobs.
-26
-27function find_chk_file {
-28    # find_chk_file takes a single argument -- the wildcard pattern
-29    # for checkpoint files to look through
-30    chk=$1
-31
-32    # find the latest 2 restart files.  This way if the latest didn't
-33    # complete we fall back to the previous one.
-34    temp_files=$(find . -maxdepth 1 -name "${chk}" -print | sort | tail -2)
-35    restartFile=""
-36    for f in ${temp_files}
-37    do
-38        # the Header is the last thing written -- if it's there, update the restart file
-39        if [ -f ${f}/Header ]; then
-40            # The scratch FS sometimes gives I/O errors when trying to read
-41            # from recently-created files, which crashes Castro. Avoid this by
-42            # making sure we can read from all the data files.
-43            if head --quiet -c1 "${f}/Header" "${f}"/Level_*/* >/dev/null; then
-44                restartFile="${f}"
-45            fi
-46        fi
-47    done
-48}
-49
-50# look for 7-digit chk files
-51find_chk_file "*chk???????"
-52
-53if [ "${restartFile}" = "" ]; then
-54    # look for 6-digit chk files
-55    find_chk_file "*chk??????"
-56fi
-57
-58if [ "${restartFile}" = "" ]; then
-59    # look for 5-digit chk files
-60    find_chk_file "*chk?????"
-61fi
-62
-63# restartString will be empty if no chk files are found -- i.e. new run
-64if [ "${restartFile}" = "" ]; then
-65    restartString=""
-66else
-67    restartString="amr.restart=${restartFile}"
-68fi
-69
-70# clean up any run management files left over from previous runs
-71rm -f dump_and_stop
-72
-73warning_time=$(bjobs -noheader -o action_warning_time "$LSB_JOBID")
-74# The `-wa URG -wt <n>` options tell bsub to send SIGURG to all processes n
-75# minutes before the runtime limit, so we can exit gracefully.
-76# SIGURG is ignored by default, so it won't make Castro crash.
-77function sig_handler {
-78    touch dump_and_stop
-79    # disable this signal handler
-80    trap - URG
-81    echo "BATCH: $warning_time left in allocation; telling Castro to dump a checkpoint and stop"
-82}
-83trap sig_handler URG
-84
-85# execute jsrun in the background then use the builtin wait so the shell can
-86# handle the signal
-87jsrun -n$n_res -c$n_cpu_cores_per_res -a$n_mpi_per_res -g$n_gpu_per_res -r$n_max_res_per_node $CASTRO $INPUTS ${restartString} &
-88wait
-89# use jswait to wait for Castro (job step 1/1) to finish and get the exit code
-90jswait 1
-
              
-
              
-
              
-
              
-
              Chaining jobs
              
-
              The script job_scripts/summit/chain_submit.sh can be used to setup job dependencies,
-i.e., a job chain.
              
-
              First you submit a job as usual using bsub, and make note of the
-job-id that it prints upon submission (the same id you would see with
-bjobs or jobstat).  Then you setup N jobs to depend on the one
-you just submitted as:
              
-
              chain_submit.sh job-id N submit_script.sh
-
              where you replace job-id with the id return from your first
-submission, replace N with the number of additional jobs, and
-replace submit_script with the name of the script you use to
-submit the job.  This will queue up N additional jobs, each depending
-on the previous.  Your submission script should use the automatic
-restarting features discussed above.
              
-
              
-
              
-
              
-
              Archiving to HPSS
              
-
              You can access HPSS from submit using the data transfer nodes by submitting a job
-via SLURM:
              
-
              sbatch -N 1 -t 15:00 -A ast106 --cluster dtn test_hpss.sh
-
              where test_hpss.sh is a SLURM script that contains the htar
-commands needed to archive your data.  This uses slurm as the job
-manager.
              
-
              An example is provided by the process.xrb archiving script in
-job_scripts/hpss/ and associated summit_hpss.submit submission script
-in jobs_scripts/summit/.  Together these will detect new plotfiles as they
-are generated, tar them up (using htar) and archive them onto HPSS.  They
-will also store the inputs, probin, and other runtime generated files.  If
-ftime is found in your path, it will also create a file called
-ftime.out that lists the simulation time corresponding to each plotfile.
              
-
              Once the plotfiles are archived they are moved to a subdirectory under
-your run directory called plotfiles/.
              
-
              By default, the files will be archived to a directory in HPSS with the same
-name as the directory your plotfiles are located in.  This can be changed
-by editing the $HPSS_DIR variable at the top of process.xrb.
              
-
              To use this, we do the following:
              
-
                
-
              1. Copy the process.xrb and summit_hpss.submit scripts into the
-directory with the plotfiles.
                2. 
-
              2. Launch the script via:
                
-
                sbatch summit_hpss.submit
-
                It will run for the full time you asked, searching for plotfiles as
-they are created and moving them to HPSS as they are produced (it
-will always leave the very last plotfile alone, since it can’t tell
-if it is still being written).
                
-
                3. 
-
              
-
              Files may be unarchived in bulk from HPSS on OLCF systems using the
-hpss_xfer.py script, which is available in the job_scripts
-directory. It requires Python 3 to be loaded to run. The command:
              
-
              ./hpss_xfer.py plt00000 -s hpss_dir -o plotfile_dir
-
              will fetch hpss_dir/plt00000.tar from the HPSS filesystem and
-unpack it in plotfile_dir. If run with no arguments in the problem
-launch directory, the script will attempt to recover all plotfiles
-archived by process.titan. Try running ./hpss_xfer.py --help
-for a description of usage and arguments.
              
-
              
 
              
 
              Frontier
              
 
              
@@ -594,7 +146,11 @@ 
              Submitting jobs-n and -N on the srun line.
              
 
        • 
 
        The job is submitted as:
        
-
        sbatch frontier.slurm
+
        sbatch frontier.slurm
 
        where frontier.slurm is the name of the submission script.
        
 
        A sample job script that includes the automatic restart functions can be found here:
 https://github.com/AMReX-Astro/workflow/blob/main/job_scripts/frontier/frontier.slurm
        
diff --git a/olcf.html b/olcf.html
index d08d3d1..7023958 100644
--- a/olcf.html
+++ b/olcf.html
@@ -90,7 +90,6 @@ 
        Working at OLCF
 
          
 
        • Compiling at OLCF
            
-
          • Summit
            • 
 
          • Frontier
@@ -98,17 +97,6 @@ 
            Working at OLCFManaging Jobs at OLCF
              
-
            • Summit
-
              • 
-
            • Archiving to HPSS
              • 
 
            • Frontier
                
 
              • Machine details
                • 
 
              • Submitting jobs
                • 
diff --git a/searchindex.js b/searchindex.js
index b83071f..c9f021c 100644
--- a/searchindex.js
+++ b/searchindex.js
@@ -1 +1 @@
-Search.setIndex({"alltitles": {"AMReX Astro basics": [[2, null]], "AMReX Astrophysics Suite": [[2, null]], "AMReX setup": [[1, "amrex-setup"]], "Archiving Data to HPSS": [[5, null]], "Archiving to HPSS": [[12, "archiving-to-hpss"]], "Automatic Restarting": [[0, "automatic-restarting"]], "Batch Visualization on Andes": [[9, null]], "Chaining": [[7, "chaining"]], "Chaining jobs": [[12, "chaining-jobs"]], "Compiling": [[0, "compiling"]], "Compiling at NERSC": [[4, null]], "Compiling at OLCF": [[10, null]], "Compiling with GCC + CUDA": [[4, "compiling-with-gcc-cuda"]], "Cray compilers": [[1, "cray-compilers"]], "Creating a conda environment": [[11, "creating-a-conda-environment"]], "Debugging": [[12, "debugging"]], "Disks": [[0, "disks"]], "Filesystems": [[7, "filesystems"]], "Frontier": [[10, "frontier"], [12, "frontier"]], "GCC": [[1, "gcc"]], "GCC 10.2": [[1, "gcc-10-2"]], "GPU offloading": [[13, "gpu-offloading"]], "GPU-aware MPI": [[12, "gpu-aware-mpi"]], "HIP Function Inlining": [[10, "hip-function-inlining"]], "Hypre": [[4, "hypre"]], "Installing Python": [[0, "installing-python"]], "Job Chaining": [[0, "job-chaining"], [12, "job-chaining"]], "Job Status": [[12, "job-status"]], "Linux Workstations": [[13, null]], "Logging In": [[0, "logging-in"]], "Machine details": [[12, "machine-details"]], "Managing Jobs at NERSC": [[7, null]], "Managing Jobs at OLCF": [[12, null]], "Monitoring a Job:": [[12, "monitoring-a-job"]], "Ookami": [[1, "ookami"]], "Perlmutter": [[4, "perlmutter"], [7, "perlmutter"]], "Queues": [[0, "queues"]], "Remote vis with Jupyter": [[13, "remote-vis-with-jupyter"]], "Requesting Allocation:": [[12, "requesting-allocation"]], "Running Jupyter Remotely from OLCF": [[11, null]], "Script Template:": [[12, "script-template"]], "Submitting": [[0, "submitting"]], "Submitting a Job:": [[12, "submitting-a-job"]], "Submitting jobs": [[12, "submitting-jobs"]], "Summit": [[10, "summit"], [12, "summit"]], "Summit Architecture:": [[12, "summit-architecture"]], "Troubleshooting": [[12, "troubleshooting"]], "Visualization at NERSC": [[6, null]], "Working at ALCF": [[0, null]], "Working at IACS": [[1, null]], "Working at NERSC": [[3, null]], "Working at OLCF": [[8, null]], "Writting a Job Script:": [[12, "writting-a-job-script"]], "bender": [[13, "bender"]], "groot": [[13, "groot"]]}, "docnames": ["alcf", "iacs", "index", "nersc", "nersc-compilers", "nersc-hpss", "nersc-visualization", "nersc-workflow", "olcf", "olcf-andes", "olcf-compilers", "olcf-jupyter", "olcf-workflow", "workstations"], "envversion": {"sphinx": 64, "sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx.ext.viewcode": 1}, "filenames": ["alcf.rst", "iacs.rst", "index.rst", "nersc.rst", "nersc-compilers.rst", "nersc-hpss.rst", "nersc-visualization.rst", "nersc-workflow.rst", "olcf.rst", "olcf-andes.rst", "olcf-compilers.rst", "olcf-jupyter.rst", "olcf-workflow.rst", "workstations.rst"], "indexentries": {}, "objects": {}, "objnames": {}, "objtypes": {}, "terms": {"": [0, 7, 11, 12], "0": [0, 1, 6, 7, 10, 11, 12], "00": [0, 6, 7, 9, 12], "01": 6, "02": 7, "03": 12, "05": 12, "1": [0, 1, 6, 7, 9, 10, 12], "10": [0, 10, 12, 13], "100": 12, "11": [6, 9, 10, 12, 13], "12": 1, "120": 7, "128": [7, 12], "131072": 12, "13th": 1, "15": 12, "1536": 7, "16": 7, "17": 10, "2": [0, 7, 9, 10, 12, 13], "20": [0, 7], "2023": 12, "2097152": 12, "21": 12, "23": 7, "24": 1, "256": [7, 12], "27": 12, "28": 10, "3": [0, 1, 4, 6, 7, 9, 10, 11, 12, 13], "30": [0, 12], "30408704": 12, "32": 12, "32768": 12, "39": 10, "398": 12, "3d": [1, 12], "4": [0, 1, 4, 7, 10, 12, 13], "48": 1, "480": 12, "5": [0, 1, 7, 10, 11, 12], "524288": 12, "560": 0, "6": [0, 7, 10, 12], "625": 12, "64": 7, "7": [0, 4, 7, 9, 10, 11, 12, 13], "75": 12, "7864320": 12, "8": [0, 6, 7, 10, 12], "80": [4, 12], "8888": 13, "90": 12, "93": 12, "A": [0, 6, 7, 9, 12], "Andes": 8, "As": 12, "At": [1, 11], "By": [5, 7, 10, 12], "For": [0, 7, 9, 12], "If": [5, 7, 11, 12], "In": [7, 10, 11, 12, 13], "It": [5, 9, 12], "No": 12, "On": 13, "The": [0, 1, 5, 6, 7, 9, 10, 12], "Then": [0, 9, 10, 12], "There": [1, 7, 12, 13], "These": 2, "To": [0, 1, 7, 9, 11, 12, 13], "With": 7, "__shared__": 10, "a1": 12, "a100": [0, 7], "a64fx": 1, "abl": 5, "abort": 12, "about": [1, 2, 12], "abov": [1, 12], "accel": [10, 12], "access": [0, 1, 7, 12], "accomplish": 9, "accord": 12, "account": 7, "action_warning_tim": 12, "activ": [0, 6, 9], "actual": [1, 5, 7], "ad": 9, "add": [0, 1, 7, 11, 12, 13], "addit": [5, 12], "adjust": 7, "affinity_gpu": 0, "after": [1, 7, 12], "afterani": 0, "alcf": 2, "all": [0, 5, 7, 9, 10, 12], "alloc": [7, 8], "alloc_flag": 12, "allow": [0, 5, 7, 10, 12], "alon": [5, 12], "also": [7, 9, 10, 11, 12], "altern": [7, 12], "although": [0, 9], "alwai": [5, 12], "amd": [10, 12], "amd_log_level": 12, "amd_serialize_copi": 12, "amd_serialize_kernel": 12, "amdgpu": 10, "among": 12, "amount": 12, "amr": [0, 1, 7, 12], "amrex": [5, 7, 10, 12], "an": [1, 7, 10, 12, 13], "anaconda": 9, "anaconda3": [9, 11], "andes": 9, "ani": [0, 5, 7, 12], "anl": 0, "anoth": 12, "anyth": 0, "anywher": 12, "aout": 0, "appear": 13, "append": 12, "applic": [0, 12], "approach": 5, "ar": [0, 1, 4, 5, 7, 10, 12], "architectur": [1, 8], "archiv": [3, 8], "arena": [7, 12], "argonn": 0, "argument": [0, 7, 12], "arm": 1, "around": 13, "ask": 12, "assert": 12, "assign": 7, "associ": 12, "assum": 12, "ast106": [9, 12], "astro": [1, 10, 12, 13], "astroexplos": 0, "atleast": 10, "attempt": 12, "automat": [5, 10, 12], "avail": [7, 12], "avoid": 12, "awar": 8, "b": [7, 12], "back": [0, 7, 12], "background": [5, 7, 12], "balanc": [7, 12], "bank": 12, "base": [0, 1, 7], "basenam": 7, "bash": [0, 1, 6, 7, 9, 12], "bashrc": [0, 9], "batch": [7, 8, 12], "becaus": [7, 12], "befor": [7, 12], "behind": [5, 7], "being": 12, "below": [7, 12], "best": [5, 6, 7, 9, 10, 11, 12], "better": [10, 12], "between": [7, 12], "big": 12, "bigger": 7, "biggest": 12, "bin": [0, 6, 7, 9, 12], "bind": [0, 7, 12], "bjob": 12, "blob": 12, "blocking_factor": 7, "both": 1, "box": 12, "break": 7, "browser": 13, "bsub": 12, "bt": 12, "bu": 12, "bug": [12, 13], "build": [1, 2, 4, 10, 12, 13], "built": 4, "builtin": [7, 12], "bulk": 12, "burner": 7, "c": [6, 7, 10, 11, 12], "c1": 12, "calcul": 12, "call": [5, 7, 9, 10, 12], "can": [0, 1, 5, 6, 7, 9, 10, 11, 12, 13], "cancel": 7, "cannot": 1, "case": [7, 12], "castro": [2, 4, 7, 10, 12], "castro2d": [0, 7, 12], "castro3d": [1, 12], "castro_exec": 7, "cc": 4, "cc60": 13, "cc70": 13, "cce": 1, "cd": [0, 7, 9], "cell": 12, "center": 2, "certain": 12, "cf": 7, "chain": [3, 8], "chain_submit": 12, "chainqsub": 0, "chainslurm": [7, 12], "chang": [0, 5, 7, 12], "channel": 11, "check": [0, 7, 12], "checkpoint": [0, 5, 7, 12], "chip": 1, "chk": [0, 7, 12], "chkxxxxx": 12, "chkxxxxxx": 12, "chkxxxxxxx": 12, "choic": 7, "clean": [5, 7, 12], "cleanli": 12, "clock": 7, "clone": [0, 9], "closest": 12, "cluster": 12, "co": 5, "coars": [0, 7], "code": [2, 9, 10, 12, 13], "collect": 12, "com": [0, 1, 2, 9, 10, 12], "come": 12, "command": [5, 12, 13], "commcm": 1, "comment": 1, "common": 7, "commun": 12, "comp": [0, 1, 4, 10, 13], "compar": 7, "compat": 10, "compil": [3, 7, 8, 12, 13], "compile_cuda_path": 13, "complet": [0, 7, 12], "compos": [7, 12], "comput": [1, 7, 12], "conda": [0, 6, 8, 9], "condens": 12, "config": [1, 11], "configur": 4, "connect": [12, 13], "consid": 12, "constraint": 7, "construct": 12, "contain": [5, 7, 12], "continu": [5, 13], "control": 10, "copi": [0, 5, 12], "core": [1, 6, 7, 12], "correct": [11, 12], "correctli": 7, "correspond": [5, 12], "couldn": 12, "count": 0, "coupl": 7, "cpe": 1, "cpp": 5, "cpu": [0, 6, 7, 12], "crai": [0, 10, 12], "crash": 12, "crayp": [10, 12], "creat": [0, 5, 6, 7, 8, 9, 12], "cuda": [0, 3, 7, 10, 12, 13], "cuda_vers": 13, "cudatoolkit": [4, 7], "current": [10, 12], "custom": 0, "cxx": 4, "darshan": 10, "data": [0, 3, 12], "date": [5, 9], "deactiv": 6, "debug": [0, 7, 8], "debug1": 1, "debug2": 1, "debugg": 12, "decis": 12, "default": [5, 7, 10, 12, 13], "defin": 12, "delet": 5, "depend": [0, 5, 7, 12], "depict": 12, "depth": 0, "describ": [5, 11, 12], "descript": 12, "design": [7, 12], "destin": 5, "detail": [7, 8], "detect": [5, 10, 12], "determin": 12, "detriment": 10, "dev": 12, "develop": 6, "devic": [7, 10], "diagnost": 5, "didn": [0, 7, 12], "differ": [1, 7], "digit": [0, 7, 12], "directori": [0, 5, 7, 12], "disabl": [7, 10, 12], "disallow": 10, "discuss": 12, "distribut": [7, 11], "do": [0, 1, 6, 7, 9, 11, 12, 13], "doc": [0, 2, 6, 9, 10, 12], "document": [9, 11, 12], "doe": [1, 9], "doesn": [5, 7, 10, 12], "domain": [7, 12], "don": 5, "done": [0, 1, 5, 7, 12], "down": [7, 10], "downgrad": 12, "dtn": 12, "due": 10, "dump": [7, 12], "dump_and_stop": [7, 12], "dure": 7, "e": [0, 4, 7, 11, 12], "each": [0, 1, 7, 9, 12], "easier": 12, "echo": [0, 7, 12], "edit": [5, 12], "edu": [1, 13], "els": [0, 7, 12], "empti": [0, 7, 12], "en": [10, 12], "enabl": [0, 4, 12], "end": [0, 7, 12], "ensur": [5, 12], "enter": 13, "entir": 12, "enuc": 9, "env": [0, 9], "enviro": 12, "environ": [0, 1, 6, 8, 9, 10, 12], "eo": 0, "eq": 0, "equival": 12, "error": [1, 7, 12], "especi": 5, "estim": [7, 12], "etc": 0, "even": 1, "event": 12, "everyon": 7, "ex": [0, 1, 7, 12], "exampl": [0, 7, 10, 12], "exec": [0, 12], "execut": [5, 7, 12], "exhibit": 10, "exist": [5, 7, 12], "exit": [0, 7, 12], "expert": 0, "expir": 12, "explain": 12, "explicitli": 12, "export": [0, 1, 6, 7, 12], "expos": 12, "extend": 12, "extra": [0, 11], "extract": 12, "extracxxflag": 10, "f": [0, 7, 12], "factor": 7, "fail": 7, "fall": [0, 7, 12], "fals": [1, 4, 10, 12, 13], "faq": 1, "fc": 4, "featur": [10, 12], "fetch": 12, "fi": [0, 7, 12], "fi_mr_cache_monitor": 12, "figur": 12, "file": [0, 1, 5, 7, 10, 12], "filesystem": [3, 5, 12], "fill": 5, "final": [7, 12], "find": [0, 1, 7, 10, 12], "find_chk_fil": [0, 7, 12], "fine": 10, "finish": [7, 12], "first": [7, 11, 12], "firstcount": 0, "fit": 12, "fix": [7, 12], "fj": 1, "flag": [1, 10, 12], "flame_wave_1000hz_25cm_smallplt203204": 9, "follow": [0, 1, 5, 6, 7, 9, 12], "forg": [6, 11], "form": [1, 12], "format": 12, "former": 1, "fortran": 1, "found": [0, 7, 12], "frequent": 5, "from": [0, 4, 5, 7, 8, 9, 12], "frontier": 8, "frontier_user_guid": [10, 12], "ftime": [5, 12], "ftn": 4, "full": 12, "fulli": 0, "function": [0, 7, 8, 12], "furthermor": 7, "g": [4, 11, 12], "g1": 12, "gcc": [3, 10, 12, 13], "gener": [2, 12, 13], "get": [1, 7, 12], "gettingstart": 0, "gfx90a": [10, 12], "git": [0, 1, 9], "github": [0, 2, 4, 9, 12], "githubusercont": 1, "give": [7, 12], "given": 5, "global": 1, "gnu": [0, 1, 4, 7, 10, 12, 13], "gnumak": 1, "go": 7, "goal": 12, "good": [11, 12], "gov": [0, 4, 6, 9, 10, 12], "gpu": [0, 2, 7, 8, 9, 10], "gracefulli": [7, 12], "grand": 0, "grant": 12, "greatli": 10, "grep": 7, "grid": 12, "group": 1, "grow": 12, "gt": 0, "guid": [1, 12], "guidelin": 0, "ha": [0, 1, 7, 9, 11], "half": 7, "half_siz": 7, "handl": [7, 12], "handler": [7, 12], "hang": 12, "happen": 7, "have": [0, 1, 5, 7, 9, 10, 11, 12], "head": 12, "header": [0, 7, 12, 13], "help": 12, "here": [0, 6, 7, 9, 12], "hip": [8, 12], "hip_enable_deferred_load": 12, "hipcc": 10, "hour": [1, 12], "how": [5, 12], "howev": [7, 12], "hpc": [2, 12], "hpss": [3, 8], "hpss_dir": [5, 12], "hpss_xfer": 12, "htar": [5, 12], "html": [9, 10, 12], "http": [0, 1, 2, 6, 9, 10, 12, 13], "hydro": 4, "hyper": 7, "hypr": 3, "hypre_cuda_sm": 4, "i": [0, 1, 5, 6, 7, 9, 10, 11, 12], "iac": 2, "id": [7, 12], "idea": [11, 12], "ideal": 1, "idl": 12, "ignor": 12, "immedi": 12, "implement": 12, "impli": 12, "import": [5, 10, 12], "includ": [2, 7, 12], "increas": [7, 10, 12], "index": 9, "infinit": 1, "inform": [0, 2, 12], "init": [6, 9], "initi": [0, 7, 12], "inlin": 8, "input": [0, 1, 5, 7, 12], "inputs_2d": [0, 7], "inputs_fil": 7, "inputs_luna": 12, "inputs_nova_t7": 7, "insid": 12, "insight": 12, "instal": [4, 6, 9, 11, 12], "instanc": [5, 7], "instead": 12, "instruct": [0, 7], "interact": [1, 12], "interest": 12, "interfac": 10, "interrupt": 12, "io": 12, "ip": 13, "ipykernel": 11, "issu": 10, "j": [0, 1, 4, 6, 7, 9, 10, 12, 13], "job": [1, 3, 5, 8], "job_script": [1, 5, 12], "jobid": [0, 5, 7], "jobs_script": 12, "jobstat": 12, "jsrun": 12, "jswait": 12, "june": 12, "jupyt": 8, "jupyterhub": 11, "jupyterlab": 11, "just": [12, 13], "keep": [7, 12], "kill": 12, "know": [1, 5], "l": [0, 7, 13], "lab": 13, "languag": 6, "larg": [5, 7, 9], "larger": 12, "last": [0, 7, 12], "later": 5, "latest": [0, 1, 7, 12], "latter": 1, "launch": [12, 13], "launcher": 12, "lcf": 0, "lead": 12, "least": 12, "leav": [5, 7, 12], "left": [5, 7, 12], "let": 12, "level": [7, 12], "level_": 12, "librari": [0, 5, 7, 10, 11], "life": 12, "like": [0, 6, 7, 12], "limit": [7, 12], "line": [7, 10, 12], "link": [1, 10], "linux": 2, "list": [5, 12], "live": 5, "ll": [0, 9], "llvm": 1, "load": [0, 1, 4, 6, 7, 9, 10, 11, 12, 13], "local": [1, 13], "localhost": 13, "locat": [5, 7, 12], "lockfil": 5, "log": [1, 4, 9, 10, 12], "logic": 7, "login": 1, "long": [1, 5], "look": [0, 5, 7, 12], "loop": 10, "lot": [10, 12], "lower": 7, "lsb_jobid": 12, "lt": 0, "lu": 0, "luna_output": 12, "luna_script": 12, "luna_sniffing_output": 12, "lustr": [1, 12], "m": 0, "m3018": [6, 7], "m3018_g": 7, "machin": [1, 2, 8], "maestroex": 2, "mai": [0, 5, 7, 10, 12], "main": [1, 5, 12], "mak": 1, "make": [0, 1, 4, 5, 9, 10, 11, 12, 13], "manag": [3, 8], "mandatori": 12, "mani": [0, 12], "map_gpu": 7, "massive_star_multi": 6, "match": 12, "max": 12, "max_grid_s": 7, "max_level": 1, "max_step": 1, "maxdepth": [0, 7, 12], "maxim": 12, "maximum": 12, "mayb": 12, "me": [7, 12], "memhook": 12, "memori": [4, 7, 9, 12], "mention": 12, "michael": 7, "microphys": 10, "might": 9, "milan": 7, "miniforg": 9, "minim": 12, "minimum": 12, "minut": 12, "mix": [0, 10, 12], "mkdir": 0, "mllvm": 10, "modifi": [9, 12], "modul": [0, 1, 4, 6, 7, 9, 10, 11, 12, 13], "modulefil": [0, 1], "moment": 1, "monitor": 8, "month": 12, "more": [0, 6, 7, 9, 12], "most": [0, 5, 7], "move": [5, 12], "mpi": [0, 1, 4, 7, 8], "mpi4pi": 0, "mpich": [0, 10, 12], "mpich_gpu_support_en": [0, 12], "mpich_max_thread_safeti": 7, "mpiexec": [0, 1], "multipl": [5, 7], "must": 12, "mv2_enable_affin": 1, "mvapich2": 1, "mvapich2_nogpu": 1, "my_env": 11, "myenv": [6, 9], "mz": 12, "n": [0, 1, 6, 7, 9, 11, 12, 13], "n14": [0, 7], "n_cpu_cores_per_r": 12, "n_gpu_per_r": 12, "n_max_res_per_nod": 12, "n_mpi_per_r": 12, "n_re": 12, "n_res2": 12, "n_res_1": 12, "name": [0, 5, 6, 7, 9, 12], "nb_conda_kernel": 11, "ndepth": 0, "necessari": 12, "need": [0, 1, 6, 7, 9, 10, 11, 12, 13], "nersc": [2, 5], "network": [7, 10], "new": [0, 7, 9, 11, 12], "newer": 1, "next": [7, 12], "nicer": 12, "nmpi_per_nod": 12, "nnode": [0, 12], "no_device_launch": 13, "node": [0, 1, 6, 7, 9, 12], "nohead": 12, "non": 10, "nopattern": 1, "normal": 5, "note": [1, 5, 12], "notebook": 11, "now": 12, "nranks_per_nod": 0, "ntask": [6, 7, 12], "nthread": 0, "ntotrank": 0, "null": 12, "numa": 7, "number": [0, 1, 7, 12], "numjob": 0, "nvcc": 13, "nvhpc": 0, "nvidia": [0, 7], "nyx": 2, "o": [1, 6, 7, 12], "obtain": [4, 7], "ocompil": 1, "off": 12, "offload": 10, "olcf": [2, 9], "old": 1, "older": [1, 13], "oldjob": 0, "omp": [1, 7], "omp_num_thread": [0, 1, 6, 7, 12], "omp_plac": [0, 6, 7], "omp_proc_bind": [6, 7], "onc": [0, 12], "one": [0, 5, 7, 12], "onli": [0, 1, 5, 7, 12], "onto": 12, "openmp": [0, 1, 7, 10, 12], "oper": 12, "optim": 7, "option": [0, 1, 7, 10, 12], "order": [7, 10, 11, 12], "origin": 7, "orion": 12, "ornl": [0, 9, 10, 12], "other": [9, 12], "otherwis": [7, 10], "our": [10, 12], "out": [1, 5, 6, 7, 9, 12], "output": [5, 12, 13], "over": [7, 12], "overwrit": 5, "own": [6, 9], "p": [0, 1, 9, 12], "p1": 4, "pack": 12, "packag": [0, 9, 10], "pagin": 12, "pair": 10, "parallel": 7, "paramet": 12, "part": 11, "particular": [0, 2, 6, 7, 12], "pass": [10, 12], "password": 13, "past": 10, "path": [0, 4, 5, 12, 13], "pattern": [0, 7, 12], "pb": 0, "pbs_nodefil": 0, "pbs_o_workdir": 0, "per": [6, 7, 12], "perfect": 7, "perform": [10, 12], "period": 5, "perlmutt": [3, 5], "perlmutter_script": 7, "php": 1, "physic": [7, 12], "pick": 12, "pid": 7, "piec": 12, "pip": [0, 9], "place": [0, 5], "placehold": 12, "plot": 9, "plotfil": [5, 9, 12], "plotfile_dir": 12, "plt00000": 12, "plt19862": 6, "point": [11, 12, 13], "polari": 0, "polici": 12, "pool": 1, "possibl": 12, "ppn": 0, "prefix": 4, "present": 10, "prevent": 12, "previou": [0, 7, 12], "prgenv": [0, 4, 7, 10, 12], "principl": 7, "print": [0, 7, 12], "probin": 12, "problem": [4, 7, 12], "process": [4, 5, 7, 12], "prod": 0, "produc": [5, 12], "product": [0, 12], "program": 10, "project": [0, 1, 7, 9], "prompt": 13, "provid": [0, 2, 12], "pty": 1, "publish": 0, "purg": 5, "put": [1, 12], "py": [6, 7, 9, 12], "python": [4, 6, 9, 10, 11, 12], "q": [0, 6, 7, 12], "qo": 7, "qstat": 0, "qsub": 0, "queu": 12, "queue": [5, 7, 12], "quickli": 5, "quiet": 12, "quot": 0, "quota": 7, "r": 12, "r6": 12, "ram": 12, "rank": 0, "rate": 10, "raw": 1, "reach": 7, "reaction": 7, "read": [0, 12], "readthedoc": 12, "receiv": [7, 12], "recent": [5, 6, 12], "recogn": 1, "recommend": [0, 5, 10], "recov": 12, "refer": [7, 10], "regard": 9, "regular": [6, 7], "reload": 12, "remain": [7, 12], "remot": 8, "repeat": 7, "replac": 12, "repo": 0, "report": 12, "repositori": 10, "request": [0, 8], "requir": [0, 4, 11, 12], "reserv": [7, 12], "resourc": [7, 12], "respect": 12, "restart": [7, 12], "restartfil": [0, 7, 12], "restartstr": [0, 7, 12], "restor": 12, "ret": 7, "return": 12, "review": 12, "right": 5, "rm": [7, 12], "rocgdb": 12, "rocm": 10, "rocmcc": 10, "room": 12, "run": [0, 1, 2, 5, 6, 7, 8, 9, 10, 12, 13], "runtim": [7, 10, 12], "salloc": 12, "salvat": 12, "same": [4, 5, 7, 10, 12], "sampl": [5, 6, 12], "save": 12, "sbatch": [5, 6, 7, 9, 12], "scale": [0, 7], "scancel": 7, "scatter": 0, "schedul": [0, 12], "scienc": 0, "scope": 10, "scratch": [5, 7, 12], "script": [0, 5, 6, 7, 8, 9, 10], "search": [11, 12], "second": [5, 7, 12], "section": 12, "sedov": 4, "see": [1, 7, 10, 12], "seem": [0, 1, 10], "select": [0, 7, 12], "send": [7, 12], "seq": 0, "session": 12, "set": [0, 1, 7, 12], "set_affinity_gpu_polari": 0, "setup": [6, 9, 10, 12], "sever": [7, 12], "sh": [0, 7, 12], "share": [7, 10], "shell": [7, 12], "short": 1, "shortag": 7, "should": [1, 4, 5, 7, 10, 12], "showquota": 7, "sig_handl": [7, 12], "sigchld": 12, "signal": [7, 12], "sigurg": [7, 12], "sigwinch": 12, "similar": 12, "simul": [5, 12], "simultan": 12, "sinc": [5, 7, 10, 12], "singl": [0, 7, 12], "site": 0, "size": [7, 12], "slack_job_start": 7, "sleep": 0, "slightli": 12, "slow": 10, "slurm": [1, 5, 7, 9, 12], "slurm_cpus_per_task": 7, "slurm_job_num_nod": 12, "slurm_nnod": 7, "slurm_ntasks_per_nod": 7, "slurm_output": 7, "slurm_submit_dir": [7, 9], "small": 12, "smallest": [0, 12], "smoothli": 12, "smplsdc": [0, 7], "smt1": 12, "smt4": 12, "so": [1, 7, 9, 10, 12, 13], "socket": [7, 12], "soft": 0, "softwar": [1, 9], "solut": 9, "some": [0, 5, 12], "someth": 6, "sometim": [5, 12], "soon": [7, 12], "sort": [0, 7, 12], "sourc": [0, 9], "specif": [1, 9], "sph": [1, 12], "spread": [6, 7], "squeez": 7, "squeue": [7, 12], "srun": [1, 6, 7, 9, 12], "ssh": [0, 1, 4, 13], "stale": 5, "stand": 12, "standard": 12, "start": [0, 1, 5, 7, 12], "startup": 12, "statement": 12, "statu": [0, 7, 8], "stderr_to_show": 12, "stdout_to_show": 12, "step": [6, 7, 12], "still": [5, 12], "stonybrook": 1, "stop": [7, 12], "storag": 12, "store": [5, 12], "strang": 10, "strategi": 7, "stream": 12, "string": 5, "structur": 12, "stuff": 1, "subch_": 7, "subdirectori": [5, 12], "submiss": [0, 5, 6, 7, 9, 10, 12], "submit": [5, 7, 8, 9], "submit_script": 12, "subsequ": 11, "success": 5, "suggest": 12, "summit": [8, 11], "summit_hpss": 12, "summit_user_guid": 12, "summon": 12, "sunysb": 13, "suppli": 12, "support": [0, 1, 9, 10, 12], "sure": [10, 11, 12], "sve": 1, "swap": [0, 10], "sweep": 12, "switch": 1, "system": [0, 7, 10, 12], "t": [0, 5, 6, 7, 9, 10, 12], "tabul": 10, "tail": [0, 7, 12], "tailor": 0, "take": [0, 1, 7, 10, 12], "tape": 5, "tar": [5, 12], "task": [7, 12], "tb": 7, "tell": [1, 7, 12], "temp_fil": [0, 7, 12], "templat": [8, 10], "tempt": 0, "termin": [0, 5, 13], "test": [1, 4, 7, 10, 12], "test_hpss": 12, "than": 12, "the_arena_init_s": [7, 12], "thei": [1, 11, 12], "them": [1, 5, 12], "therefor": [7, 12], "thi": [0, 1, 5, 6, 7, 9, 10, 11, 12], "thing": [0, 7, 12], "though": 1, "thread": [0, 1, 7, 12], "three": 12, "through": [0, 7, 12], "time": [1, 5, 7, 9, 10, 12], "timestep": [7, 12], "tiny_profil": 4, "titan": 12, "togeth": 12, "token": 13, "too": [0, 12], "took": 5, "tool": [1, 5], "top": [0, 5, 7, 12], "total": 12, "total_nmpi": 12, "touch": [7, 12], "trace": 12, "transfer": 12, "trap": [7, 12], "trento": 12, "trigger": 12, "troubleshoot": 8, "true": [0, 1, 4, 10, 12, 13], "try": [5, 12], "turn": 12, "two": [0, 7, 12], "u": [0, 7, 12], "unarchiv": 12, "under": 12, "unifi": 4, "unix": 12, "unload": 10, "unpack": 12, "unrol": 10, "until": [5, 7, 12], "up": [5, 7, 12], "updat": [0, 7, 12], "upon": 12, "urg": [7, 12], "us": [0, 1, 4, 5, 7, 9, 10, 11, 12, 13], "usag": [0, 12], "use_cuda": [0, 4, 10, 13], "use_gpu_aware_mpi": 12, "use_hip": [10, 12], "use_mpi": [1, 4, 10, 12, 13], "use_omp": [1, 4, 10], "use_omp_offload": 10, "usernam": [0, 7, 12], "usr": 13, "usual": [7, 10, 12], "valu": [7, 12], "variabl": [5, 10, 12], "variou": 2, "venv": 0, "venv_dir": 0, "veri": [0, 9, 12], "version": [0, 1, 9, 10, 12, 13], "vi": 6, "via": [0, 1, 4, 7, 9, 10, 11, 12], "view": [7, 12], "virtual": [0, 7], "vis_": 6, "visual": [3, 8, 13], "vol": 9, "w": [0, 12], "wa": [5, 12], "wai": [0, 6, 7, 12], "wait": [5, 7, 12], "wall": 7, "walltim": [0, 12], "want": [0, 5, 7, 9, 12], "warn": [7, 10, 12], "warning_tim": 12, "warpx": 12, "wc": 0, "we": [0, 1, 4, 7, 9, 12, 13], "web": 13, "week": 12, "well": 13, "when": [5, 9, 10, 11, 12], "where": [7, 12], "which": [1, 5, 7, 9, 10, 11, 12], "whole": [0, 12], "why": 1, "wildcard": [0, 7, 12], "window": 13, "wish": 11, "within": 7, "without": [10, 12, 13], "won": [10, 12], "work": [2, 5, 9, 10, 12, 13], "workaround": 12, "workdir": 7, "workflow": [0, 1, 12], "workspac": 0, "workstat": 2, "would": [1, 7, 12], "writ": 8, "write": [7, 12], "written": [0, 5, 7, 12], "wt": 12, "www": 1, "x": [1, 12, 13], "x86": [7, 12], "xfer": 5, "xl": 10, "xrb": [5, 9, 12], "y": 11, "year": 12, "you": [0, 1, 4, 5, 6, 7, 9, 10, 11, 12, 13], "your": [0, 5, 6, 7, 9, 10, 11, 12, 13], "yt": [6, 9, 11]}, "titles": ["Working at ALCF", "Working at IACS", "AMReX Astrophysics Suite", "Working at NERSC", "Compiling at NERSC", "Archiving Data to HPSS", "Visualization at NERSC", "Managing Jobs at NERSC", "Working at OLCF", "Batch Visualization on Andes", "Compiling at OLCF", "Running Jupyter Remotely from OLCF", "Managing Jobs at OLCF", "Linux Workstations"], "titleterms": {"10": 1, "2": 1, "Andes": 9, "In": 0, "alcf": 0, "alloc": 12, "amrex": [1, 2], "architectur": 12, "archiv": [5, 12], "astro": 2, "astrophys": 2, "automat": 0, "awar": 12, "basic": 2, "batch": 9, "bender": 13, "chain": [0, 7, 12], "compil": [0, 1, 4, 10], "conda": 11, "crai": 1, "creat": 11, "cuda": 4, "data": 5, "debug": 12, "detail": 12, "disk": 0, "environ": 11, "filesystem": 7, "from": 11, "frontier": [10, 12], "function": 10, "gcc": [1, 4], "gpu": [12, 13], "groot": 13, "hip": 10, "hpss": [5, 12], "hypr": 4, "iac": 1, "inlin": 10, "instal": 0, "job": [0, 7, 12], "jupyt": [11, 13], "linux": 13, "log": 0, "machin": 12, "manag": [7, 12], "monitor": 12, "mpi": 12, "nersc": [3, 4, 6, 7], "offload": 13, "olcf": [8, 10, 11, 12], "ookami": 1, "perlmutt": [4, 7], "python": 0, "queue": 0, "remot": [11, 13], "request": 12, "restart": 0, "run": 11, "script": 12, "setup": 1, "statu": 12, "submit": [0, 12], "suit": 2, "summit": [10, 12], "templat": 12, "troubleshoot": 12, "vi": 13, "visual": [6, 9], "work": [0, 1, 3, 8], "workstat": 13, "writ": 12}})
\ No newline at end of file
+Search.setIndex({"alltitles": {"AMReX Astro basics": [[2, null]], "AMReX Astrophysics Suite": [[2, null]], "AMReX setup": [[1, "amrex-setup"]], "Archiving Data to HPSS": [[5, null]], "Automatic Restarting": [[0, "automatic-restarting"]], "Batch Visualization on Andes": [[9, null]], "Chaining": [[7, "chaining"]], "Compiling": [[0, "compiling"]], "Compiling at NERSC": [[4, null]], "Compiling at OLCF": [[10, null]], "Compiling with GCC + CUDA": [[4, "compiling-with-gcc-cuda"]], "Cray compilers": [[1, "cray-compilers"]], "Creating a conda environment": [[11, "creating-a-conda-environment"]], "Debugging": [[12, "debugging"]], "Disks": [[0, "disks"]], "Filesystems": [[7, "filesystems"]], "Frontier": [[10, "frontier"], [12, "frontier"]], "GCC": [[1, "gcc"]], "GCC 10.2": [[1, "gcc-10-2"]], "GPU offloading": [[13, "gpu-offloading"]], "GPU-aware MPI": [[12, "gpu-aware-mpi"]], "HIP Function Inlining": [[10, "hip-function-inlining"]], "Hypre": [[4, "hypre"]], "Installing Python": [[0, "installing-python"]], "Job Chaining": [[0, "job-chaining"], [12, "job-chaining"]], "Job Status": [[12, "job-status"]], "Linux Workstations": [[13, null]], "Logging In": [[0, "logging-in"]], "Machine details": [[12, "machine-details"]], "Managing Jobs at NERSC": [[7, null]], "Managing Jobs at OLCF": [[12, null]], "Ookami": [[1, "ookami"]], "Perlmutter": [[4, "perlmutter"], [7, "perlmutter"]], "Queues": [[0, "queues"]], "Remote vis with Jupyter": [[13, "remote-vis-with-jupyter"]], "Running Jupyter Remotely from OLCF": [[11, null]], "Submitting": [[0, "submitting"]], "Submitting jobs": [[12, "submitting-jobs"]], "Troubleshooting": [[12, "troubleshooting"]], "Visualization at NERSC": [[6, null]], "Working at ALCF": [[0, null]], "Working at IACS": [[1, null]], "Working at NERSC": [[3, null]], "Working at OLCF": [[8, null]], "bender": [[13, "bender"]], "groot": [[13, "groot"]]}, "docnames": ["alcf", "iacs", "index", "nersc", "nersc-compilers", "nersc-hpss", "nersc-visualization", "nersc-workflow", "olcf", "olcf-andes", "olcf-compilers", "olcf-jupyter", "olcf-workflow", "workstations"], "envversion": {"sphinx": 64, "sphinx.domains.c": 3, "sphinx.domains.changeset": 1, "sphinx.domains.citation": 1, "sphinx.domains.cpp": 9, "sphinx.domains.index": 1, "sphinx.domains.javascript": 3, "sphinx.domains.math": 2, "sphinx.domains.python": 4, "sphinx.domains.rst": 2, "sphinx.domains.std": 2, "sphinx.ext.viewcode": 1}, "filenames": ["alcf.rst", "iacs.rst", "index.rst", "nersc.rst", "nersc-compilers.rst", "nersc-hpss.rst", "nersc-visualization.rst", "nersc-workflow.rst", "olcf.rst", "olcf-andes.rst", "olcf-compilers.rst", "olcf-jupyter.rst", "olcf-workflow.rst", "workstations.rst"], "indexentries": {}, "objects": {}, "objnames": {}, "objtypes": {}, "terms": {"": [0, 7, 11, 12], "0": [0, 1, 6, 7, 11, 12], "00": [0, 6, 7, 9, 12], "01": 6, "02": 7, "05": 12, "1": [0, 1, 6, 7, 9, 10, 12], "10": [0, 12, 13], "11": [6, 9, 13], "12": 1, "120": 7, "128": 7, "13th": 1, "1536": 7, "16": 7, "2": [0, 7, 9, 12, 13], "20": [0, 7], "2023": 12, "23": 7, "24": 1, "256": 7, "27": 12, "28": 10, "3": [0, 1, 4, 6, 7, 9, 10, 11, 12, 13], "30": [0, 12], "39": 10, "3d": [1, 12], "4": [0, 1, 4, 7, 12, 13], "48": 1, "5": [0, 1, 7, 10, 11, 12], "560": 0, "6": [0, 7, 10, 12], "64": 7, "7": [0, 4, 7, 9, 11, 12, 13], "8": [0, 6, 7, 10, 12], "80": 4, "8888": 13, "A": [0, 6, 7, 9, 12], "Andes": 8, "As": 12, "At": [1, 11], "By": [5, 7, 10], "For": [0, 7, 9, 12], "If": [5, 7, 11, 12], "In": [7, 10, 11, 13], "It": [5, 9, 12], "On": 13, "The": [0, 1, 5, 6, 7, 9, 12], "Then": [0, 9], "There": [1, 7, 12, 13], "These": 2, "To": [0, 1, 7, 9, 11, 12, 13], "With": 7, "__shared__": 10, "a100": [0, 7], "a64fx": 1, "abl": 5, "abort": 12, "about": [1, 2, 12], "abov": 1, "accel": [10, 12], "access": [0, 1, 7], "accomplish": 9, "account": 7, "activ": [0, 6, 9], "actual": [1, 5, 7], "ad": 9, "add": [0, 1, 7, 11, 12, 13], "addit": 5, "adjust": 7, "affinity_gpu": 0, "after": [1, 7], "afterani": 0, "alcf": 2, "all": [0, 5, 7, 9, 10, 12], "alloc": 7, "allow": [0, 5, 7, 10], "alon": 5, "also": [7, 9, 10, 11, 12], "altern": 7, "although": [0, 9], "alwai": 5, "amd": [10, 12], "amd_log_level": 12, "amd_serialize_copi": 12, "amd_serialize_kernel": 12, "amdgpu": 10, "amr": [0, 1, 7], "amrex": [5, 7, 12], "an": [1, 7, 12, 13], "anaconda": 9, "anaconda3": [9, 11], "andes": 9, "ani": [0, 5, 7], "anl": 0, "anyth": 0, "aout": 0, "appear": 13, "applic": 0, "approach": 5, "ar": [0, 1, 4, 5, 7, 12], "architectur": 1, "archiv": 3, "arena": [7, 12], "argonn": 0, "argument": [0, 7], "arm": 1, "around": 13, "assign": 7, "ast106": [9, 12], "astro": [1, 12, 13], "astroexplos": 0, "automat": [5, 10, 12], "avail": 7, "awar": 8, "b": [7, 12], "back": [0, 7], "background": [5, 7], "balanc": 7, "base": [0, 1, 7], "basenam": 7, "bash": [0, 1, 6, 7, 9, 12], "bashrc": [0, 9], "batch": [7, 8, 12], "becaus": 7, "befor": 7, "behind": [5, 7], "below": 7, "best": [5, 6, 7, 9, 10, 11, 12], "better": [10, 12], "between": 7, "big": 12, "bigger": 7, "bin": [0, 6, 7, 9, 12], "bind": [0, 7, 12], "blob": 12, "blocking_factor": 7, "both": 1, "break": 7, "browser": 13, "bt": 12, "bug": 13, "build": [1, 2, 4, 10, 12, 13], "built": 4, "builtin": 7, "burner": 7, "c": [6, 7, 11], "call": [5, 7, 9, 10, 12], "can": [0, 1, 5, 6, 7, 9, 10, 11, 12, 13], "cancel": 7, "cannot": 1, "case": 7, "castro": [2, 4, 7], "castro2d": [0, 7, 12], "castro3d": [1, 12], "castro_exec": 7, "cc": 4, "cc60": 13, "cc70": 13, "cce": 1, "cd": [0, 7, 9], "center": 2, "cf": 7, "chain": [3, 8], "chainqsub": 0, "chainslurm": [7, 12], "chang": [0, 5, 7, 12], "channel": 11, "check": [0, 7, 12], "checkpoint": [0, 5, 7], "chip": 1, "chk": [0, 7], "choic": 7, "clean": [5, 7], "clock": 7, "clone": [0, 9], "closest": 12, "co": 5, "coars": [0, 7], "code": [2, 9, 10, 12, 13], "collect": 12, "com": [0, 1, 2, 9, 10, 12], "command": [5, 13], "commcm": 1, "comment": 1, "common": 7, "comp": [0, 1, 4, 13], "compar": 7, "compat": 10, "compil": [3, 7, 8, 12, 13], "compile_cuda_path": 13, "complet": [0, 7], "compos": 7, "comput": [1, 7, 12], "conda": [0, 6, 8, 9], "config": [1, 11], "configur": 4, "connect": 13, "constraint": 7, "contain": [5, 7], "continu": [5, 13], "copi": [0, 5], "core": [1, 6, 7], "correct": 11, "correctli": 7, "correspond": 5, "count": 0, "coupl": 7, "cpe": 1, "cpp": 5, "cpu": [0, 6, 7, 12], "crai": [0, 10, 12], "crash": 12, "crayp": [10, 12], "creat": [0, 5, 6, 7, 8, 9], "cuda": [0, 3, 7, 13], "cuda_vers": 13, "cudatoolkit": [4, 7], "custom": 0, "cxx": 4, "darshan": 10, "data": [0, 3, 12], "date": [5, 9], "deactiv": 6, "debug": [0, 7, 8], "debug1": 1, "debug2": 1, "debugg": 12, "default": [5, 7, 10, 13], "delet": 5, "depend": [0, 5, 7, 12], "depth": 0, "describ": [5, 11], "design": 7, "destin": 5, "detail": [7, 8], "detect": [5, 10], "detriment": 10, "develop": 6, "devic": [7, 10], "diagnost": 5, "didn": [0, 7], "differ": [1, 7], "digit": [0, 7], "directori": [0, 5, 7], "disabl": [7, 10, 12], "distribut": [7, 11], "do": [0, 1, 6, 7, 9, 11, 12, 13], "doc": [0, 2, 6, 9, 10, 12], "document": [9, 11], "doe": [1, 9], "doesn": [5, 7, 12], "domain": 7, "don": 5, "done": [0, 1, 5, 7, 12], "down": [7, 10], "dump": 7, "dump_and_stop": 7, "dure": 7, "e": [0, 4, 7, 11], "each": [0, 1, 7, 9, 12], "echo": [0, 7], "edit": 5, "edu": [1, 13], "els": [0, 7], "empti": [0, 7], "en": [10, 12], "enabl": [0, 4, 12], "end": [0, 7], "ensur": 5, "enter": 13, "enuc": 9, "env": [0, 9], "environ": [0, 1, 6, 8, 9, 10, 12], "eo": 0, "eq": 0, "error": [1, 7], "especi": 5, "estim": [7, 12], "etc": 0, "even": 1, "everyon": 7, "ex": [0, 1, 7, 12], "exampl": [0, 7, 12], "exec": [0, 12], "execut": [5, 7, 12], "exhibit": 10, "exist": [5, 7, 12], "exit": [0, 7], "expert": 0, "explicitli": 12, "export": [0, 1, 6, 7, 12], "extra": [0, 11], "extracxxflag": 10, "f": [0, 7], "factor": 7, "fail": 7, "fall": [0, 7], "fals": [1, 4, 12, 13], "faq": 1, "fc": 4, "fi": [0, 7], "fi_mr_cache_monitor": 12, "file": [0, 1, 5, 7, 10], "filesystem": [3, 5, 12], "fill": 5, "final": 7, "find": [0, 1, 7], "find_chk_fil": [0, 7], "finish": 7, "first": [7, 11], "firstcount": 0, "fix": 7, "fj": 1, "flag": [1, 10], "flame_wave_1000hz_25cm_smallplt203204": 9, "follow": [0, 1, 5, 6, 7, 9, 12], "forg": [6, 11], "form": 1, "former": 1, "fortran": 1, "found": [0, 7, 12], "frequent": 5, "from": [0, 4, 5, 7, 8, 9], "frontier": 8, "frontier_user_guid": [10, 12], "ftime": 5, "ftn": 4, "fulli": 0, "function": [0, 7, 8, 12], "furthermor": 7, "g": [4, 11], "gcc": [3, 13], "gener": [2, 13], "get": [1, 7, 12], "gettingstart": 0, "gfx90a": [10, 12], "git": [0, 1, 9], "github": [0, 2, 4, 9, 12], "githubusercont": 1, "give": 7, "given": 5, "global": 1, "gnu": [0, 1, 4, 7, 10, 12, 13], "gnumak": 1, "go": 7, "good": [11, 12], "gov": [0, 4, 6, 9, 10, 12], "gpu": [0, 2, 7, 8, 9], "gracefulli": 7, "grand": 0, "greatli": 10, "grep": 7, "group": 1, "grow": 12, "gt": 0, "guid": 1, "guidelin": 0, "ha": [0, 1, 7, 9, 11], "half": 7, "half_siz": 7, "handl": 7, "handler": 7, "hang": 12, "happen": 7, "have": [0, 1, 5, 7, 9, 11], "header": [0, 7, 13], "here": [0, 6, 7, 9, 12], "hip": [8, 12], "hip_enable_deferred_load": 12, "hipcc": 10, "hour": 1, "how": 5, "howev": 7, "hpc": [2, 12], "hpss": 3, "hpss_dir": 5, "htar": 5, "html": [9, 10, 12], "http": [0, 1, 2, 6, 9, 10, 12, 13], "hydro": 4, "hyper": 7, "hypr": 3, "hypre_cuda_sm": 4, "i": [0, 1, 5, 6, 7, 9, 10, 11, 12], "iac": 2, "id": [7, 12], "idea": [11, 12], "ideal": 1, "import": 5, "includ": [2, 7, 12], "increas": [7, 10], "index": 9, "infinit": 1, "inform": [0, 2, 12], "init": [6, 9], "initi": [0, 7, 12], "inlin": 8, "input": [0, 1, 5, 7, 12], "inputs_2d": [0, 7], "inputs_fil": 7, "inputs_nova_t7": 7, "insid": 12, "instal": [4, 6, 9, 11, 12], "instanc": [5, 7], "instruct": [0, 7], "interact": [1, 12], "interfac": 10, "interrupt": 12, "io": 12, "ip": 13, "ipykernel": 11, "issu": 10, "j": [0, 1, 4, 6, 7, 9, 12, 13], "job": [1, 3, 5, 8], "job_script": [1, 5, 12], "jobid": [0, 5, 7], "june": 12, "jupyt": 8, "jupyterhub": 11, "jupyterlab": 11, "just": 13, "keep": 7, "know": [1, 5], "l": [0, 7, 13], "lab": 13, "languag": 6, "larg": [5, 7, 9], "larger": 12, "last": [0, 7], "later": 5, "latest": [0, 1, 7, 12], "latter": 1, "launch": 13, "lcf": 0, "leav": [5, 7], "left": [5, 7], "level": 7, "librari": [0, 5, 7, 11], "like": [0, 6, 7], "limit": 7, "line": [7, 12], "link": [1, 10], "linux": 2, "list": 5, "live": 5, "ll": [0, 9], "llvm": 1, "load": [0, 1, 4, 6, 7, 9, 10, 11, 12, 13], "local": [1, 13], "localhost": 13, "locat": [5, 7], "lockfil": 5, "log": [1, 4, 9, 10, 12], "logic": 7, "login": 1, "long": [1, 5], "look": [0, 5, 7], "loop": 10, "lot": [10, 12], "lower": 7, "lt": 0, "lu": 0, "lustr": [1, 12], "m": 0, "m3018": [6, 7], "m3018_g": 7, "machin": [1, 2, 8], "maestroex": 2, "mai": [0, 5, 7, 10], "main": [1, 5, 12], "mak": 1, "make": [0, 1, 4, 5, 9, 10, 11, 12, 13], "manag": [3, 8], "mani": 0, "map_gpu": 7, "massive_star_multi": 6, "max_grid_s": 7, "max_level": 1, "max_step": 1, "maxdepth": [0, 7], "mayb": 12, "me": [7, 12], "memhook": 12, "memori": [4, 7, 9], "michael": 7, "microphys": 10, "might": 9, "milan": 7, "miniforg": 9, "mix": [0, 10, 12], "mkdir": 0, "mllvm": 10, "modifi": 9, "modul": [0, 1, 4, 6, 7, 9, 10, 11, 12, 13], "modulefil": [0, 1], "moment": 1, "more": [0, 6, 7, 9], "most": [0, 5, 7], "move": 5, "mpi": [0, 1, 4, 7, 8], "mpi4pi": 0, "mpich": [0, 10, 12], "mpich_gpu_support_en": [0, 12], "mpich_max_thread_safeti": 7, "mpiexec": [0, 1], "multipl": [5, 7], "mv2_enable_affin": 1, "mvapich2": 1, "mvapich2_nogpu": 1, "my_env": 11, "myenv": [6, 9], "mz": 12, "n": [0, 1, 6, 7, 9, 11, 12, 13], "n14": [0, 7], "name": [0, 5, 6, 7, 9, 12], "nb_conda_kernel": 11, "ndepth": 0, "necessari": 12, "need": [0, 1, 6, 7, 9, 11, 12, 13], "nersc": [2, 5], "network": [7, 10], "new": [0, 7, 9, 11], "newer": 1, "next": 7, "nmpi_per_nod": 12, "nnode": 0, "no_device_launch": 13, "node": [0, 1, 6, 7, 9, 12], "non": 10, "nopattern": 1, "normal": 5, "note": [1, 5], "notebook": 11, "now": 12, "nranks_per_nod": 0, "ntask": [6, 7, 12], "nthread": 0, "ntotrank": 0, "numa": 7, "number": [0, 1, 7, 12], "numjob": 0, "nvcc": 13, "nvhpc": 0, "nvidia": [0, 7], "nyx": 2, "o": [1, 6, 7, 12], "obtain": [4, 7], "ocompil": 1, "off": 12, "olcf": [2, 9], "old": 1, "older": [1, 13], "oldjob": 0, "omp": [1, 7], "omp_num_thread": [0, 1, 6, 7, 12], "omp_plac": [0, 6, 7], "omp_proc_bind": [6, 7], "onc": 0, "one": [0, 5, 7], "onli": [0, 1, 5, 7], "onto": 12, "openmp": [0, 1, 7], "optim": 7, "option": [0, 1, 7], "order": [7, 11], "origin": 7, "orion": 12, "ornl": [0, 9, 10, 12], "other": 9, "otherwis": 7, "our": 10, "out": [1, 5, 6, 7, 9, 12], "output": [5, 13], "over": 7, "overwrit": 5, "own": [6, 9], "p": [0, 1, 9, 12], "p1": 4, "packag": [0, 9], "pagin": 12, "parallel": 7, "paramet": 12, "part": 11, "particular": [0, 2, 6, 7], "pass": 10, "password": 13, "past": 10, "path": [0, 4, 5, 13], "pattern": [0, 7], "pb": 0, "pbs_nodefil": 0, "pbs_o_workdir": 0, "per": [6, 7, 12], "perfect": 7, "perform": 10, "period": 5, "perlmutt": [3, 5], "perlmutter_script": 7, "php": 1, "physic": 7, "pid": 7, "pip": [0, 9], "place": [0, 5], "plot": 9, "plotfil": [5, 9], "plt19862": 6, "point": [11, 13], "polari": 0, "polici": 12, "pool": 1, "possibl": 12, "ppn": 0, "prefix": 4, "prevent": 12, "previou": [0, 7, 12], "prgenv": [0, 4, 7, 10, 12], "principl": 7, "print": [0, 7], "problem": [4, 7], "process": [4, 5, 7], "prod": 0, "produc": 5, "product": 0, "program": 10, "project": [0, 1, 7, 9], "prompt": 13, "provid": [0, 2], "pty": 1, "publish": 0, "purg": 5, "put": 1, "py": [6, 7, 9], "python": [4, 6, 9, 11], "q": [0, 6, 7], "qo": 7, "qstat": 0, "qsub": 0, "queu": 12, "queue": [5, 7, 12], "quickli": 5, "quot": 0, "quota": 7, "rank": 0, "rate": 10, "raw": 1, "reach": 7, "reaction": 7, "read": 0, "readthedoc": 12, "receiv": 7, "recent": [5, 6], "recogn": 1, "recommend": [0, 5], "refer": [7, 10], "regard": 9, "regular": [6, 7], "reload": 12, "remain": 7, "remot": 8, "repeat": 7, "repo": 0, "report": 12, "repositori": 10, "request": 0, "requir": [0, 4, 11], "reserv": 7, "resourc": 7, "restart": [7, 12], "restartfil": [0, 7], "restartstr": [0, 7], "restor": 12, "ret": 7, "right": 5, "rm": 7, "rocgdb": 12, "rocm": 10, "rocmcc": 10, "run": [0, 1, 2, 5, 6, 7, 8, 9, 10, 12, 13], "runtim": [7, 10, 12], "salloc": 12, "same": [4, 5, 7, 12], "sampl": [5, 6, 12], "sbatch": [5, 6, 7, 9, 12], "scale": [0, 7], "scancel": 7, "scatter": 0, "schedul": [0, 12], "scienc": 0, "scope": 10, "scratch": [5, 7], "script": [0, 5, 6, 7, 9, 12], "search": 11, "second": [5, 7], "sedov": 4, "see": [1, 7, 10, 12], "seem": [0, 1, 10], "select": [0, 7], "send": 7, "seq": 0, "session": 12, "set": [0, 1, 7, 12], "set_affinity_gpu_polari": 0, "setup": [6, 9, 12], "sever": 7, "sh": [0, 7, 12], "share": 7, "shell": [7, 12], "short": 1, "shortag": 7, "should": [1, 4, 5, 7, 12], "showquota": 7, "sig_handl": 7, "signal": 7, "sigurg": 7, "simul": 5, "sinc": [5, 7], "singl": [0, 7], "site": 0, "size": [7, 12], "slack_job_start": 7, "sleep": 0, "slow": 10, "slurm": [1, 5, 7, 9, 12], "slurm_cpus_per_task": 7, "slurm_job_num_nod": 12, "slurm_nnod": 7, "slurm_ntasks_per_nod": 7, "slurm_output": 7, "slurm_submit_dir": [7, 9], "smallest": 0, "smplsdc": [0, 7], "so": [1, 7, 9, 10, 13], "socket": 7, "soft": 0, "softwar": [1, 9], "solut": 9, "some": [0, 5, 12], "someth": 6, "sometim": 5, "soon": 7, "sort": [0, 7], "sourc": [0, 9], "specif": [1, 9], "sph": [1, 12], "spread": [6, 7], "squeez": 7, "squeue": [7, 12], "srun": [1, 6, 7, 9, 12], "ssh": [0, 1, 4, 13], "stale": 5, "start": [0, 1, 5, 7, 12], "startup": 12, "statu": [0, 7, 8], "step": [6, 7], "still": 5, "stonybrook": 1, "stop": 7, "storag": 12, "store": 5, "strang": 10, "strategi": 7, "string": 5, "stuff": 1, "subch_": 7, "subdirectori": 5, "submiss": [0, 5, 6, 7, 9, 12], "submit": [5, 7, 8, 9], "subsequ": 11, "success": 5, "suggest": 12, "summit": 11, "sunysb": 13, "support": [0, 1, 9], "sure": 11, "sve": 1, "swap": 0, "switch": 1, "system": [0, 7, 10, 12], "t": [0, 5, 6, 7, 9, 12], "tabul": 10, "tail": [0, 7], "tailor": 0, "take": [0, 1, 7, 10], "tape": 5, "tar": 5, "task": [7, 12], "tb": 7, "tell": [1, 7], "temp_fil": [0, 7], "templat": 10, "tempt": 0, "termin": [0, 5, 13], "test": [1, 4, 7, 10, 12], "than": 12, "the_arena_init_s": [7, 12], "thei": [1, 11], "them": [1, 5], "therefor": 7, "thi": [0, 1, 5, 6, 7, 9, 10, 11, 12], "thing": [0, 7], "though": 1, "thread": [0, 1, 7], "through": [0, 7], "time": [1, 5, 7, 9, 10, 12], "timestep": 7, "tiny_profil": 4, "token": 13, "too": [0, 12], "took": 5, "tool": [1, 5], "top": [0, 5, 7], "total_nmpi": 12, "touch": 7, "trace": 12, "trap": 7, "trento": 12, "troubleshoot": 8, "true": [0, 1, 4, 10, 12, 13], "try": [5, 12], "turn": 12, "two": [0, 7], "u": [0, 7], "unifi": 4, "unload": 10, "unrol": 10, "until": [5, 7], "up": [5, 7, 12], "updat": [0, 7], "urg": 7, "us": [0, 1, 4, 5, 7, 9, 10, 11, 12, 13], "usag": 0, "use_cuda": [0, 4, 13], "use_gpu_aware_mpi": 12, "use_hip": [10, 12], "use_mpi": [1, 4, 12, 13], "use_omp": [1, 4], "usernam": [0, 7], "usr": 13, "usual": [7, 12], "valu": 7, "variabl": [5, 10, 12], "variou": 2, "venv": 0, "venv_dir": 0, "veri": [0, 9], "version": [0, 1, 9, 13], "vi": 6, "via": [0, 1, 4, 7, 9, 10, 11, 12], "view": 7, "virtual": [0, 7], "vis_": 6, "visual": [3, 8, 13], "vol": 9, "w": 0, "wa": 5, "wai": [0, 6, 7], "wait": [5, 7], "wall": 7, "walltim": 0, "want": [0, 5, 7, 9, 12], "warn": 7, "warpx": 12, "wc": 0, "we": [0, 1, 4, 7, 9, 12, 13], "web": 13, "well": 13, "when": [5, 9, 10, 11], "where": [7, 12], "which": [1, 5, 7, 9, 11], "whole": 0, "why": 1, "wildcard": [0, 7], "window": 13, "wish": 11, "within": 7, "without": [10, 13], "work": [2, 5, 9, 12, 13], "workaround": 12, "workdir": 7, "workflow": [0, 1, 12], "workspac": 0, "workstat": 2, "would": [1, 7, 12], "write": 7, "written": [0, 5, 7], "www": 1, "x": [1, 12, 13], "x86": [7, 12], "xfer": 5, "xrb": [5, 9], "y": 11, "you": [0, 1, 4, 5, 6, 7, 9, 11, 12, 13], "your": [0, 5, 6, 7, 9, 11, 12, 13], "yt": [6, 9, 11]}, "titles": ["Working at ALCF", "Working at IACS", "AMReX Astrophysics Suite", "Working at NERSC", "Compiling at NERSC", "Archiving Data to HPSS", "Visualization at NERSC", "Managing Jobs at NERSC", "Working at OLCF", "Batch Visualization on Andes", "Compiling at OLCF", "Running Jupyter Remotely from OLCF", "Managing Jobs at OLCF", "Linux Workstations"], "titleterms": {"10": 1, "2": 1, "Andes": 9, "In": 0, "alcf": 0, "amrex": [1, 2], "archiv": 5, "astro": 2, "astrophys": 2, "automat": 0, "awar": 12, "basic": 2, "batch": 9, "bender": 13, "chain": [0, 7, 12], "compil": [0, 1, 4, 10], "conda": 11, "crai": 1, "creat": 11, "cuda": 4, "data": 5, "debug": 12, "detail": 12, "disk": 0, "environ": 11, "filesystem": 7, "from": 11, "frontier": [10, 12], "function": 10, "gcc": [1, 4], "gpu": [12, 13], "groot": 13, "hip": 10, "hpss": 5, "hypr": 4, "iac": 1, "inlin": 10, "instal": 0, "job": [0, 7, 12], "jupyt": [11, 13], "linux": 13, "log": 0, "machin": 12, "manag": [7, 12], "mpi": 12, "nersc": [3, 4, 6, 7], "offload": 13, "olcf": [8, 10, 11, 12], "ookami": 1, "perlmutt": [4, 7], "python": 0, "queue": 0, "remot": [11, 13], "restart": 0, "run": 11, "setup": 1, "statu": 12, "submit": [0, 12], "suit": 2, "troubleshoot": 12, "vi": 13, "visual": [6, 9], "work": [0, 1, 3, 8], "workstat": 13}})
\ No newline at end of file