From d95ee44ff712b90e7436f9e6976b7eb02b277534 Mon Sep 17 00:00:00 2001 From: salotz Date: Wed, 24 May 2017 17:24:02 -0400 Subject: [PATCH] Major improvements, workable version I cleaned up the noweb template through a round of debugging and development. I added an example directory that shows one with things filled in that work on the HPCC. --- examples/test_template/input/.keep | 0 examples/test_template/input/test_input.txt | 1 + examples/test_template/jobs/.keep | 0 .../jobs/test_template-43144464.mgr-04.i/log | 304 ++++++++++++ .../output/43144464.mgr-04.i.SC | 321 +++++++++++++ .../output/file_0.txt | 1 + .../output/file_1.txt | 1 + .../output/file_2.txt | 1 + .../output/file_3.txt | 1 + .../output/file_4.txt | 1 + .../output/file_5.txt | 1 + .../output/file_6.txt | 1 + .../output/file_7.txt | 1 + .../output/file_8.txt | 1 + .../output/test_input.txt | 1 + examples/test_template/pbs.err/.keep | 0 .../pbs.err/test_template.e43144464 | 0 examples/test_template/pbs.out/.keep | 0 .../pbs.out/test_template.o43144464 | 1 + examples/test_template/submit.org | 454 ++++++++++++++++++ examples/test_template/submit.org~ | 454 ++++++++++++++++++ examples/test_template/submit.pbs | 321 +++++++++++++ examples/sEH_NETS.pbs => sEH_NETS.pbs~ | 4 +- submit.org | 321 +++++++++---- submit.pbs | 310 ++++++++---- 25 files changed, 2302 insertions(+), 199 deletions(-) create mode 100644 examples/test_template/input/.keep create mode 100644 examples/test_template/input/test_input.txt create mode 100644 examples/test_template/jobs/.keep create mode 100644 examples/test_template/jobs/test_template-43144464.mgr-04.i/log create mode 100755 examples/test_template/jobs/test_template-43144464.mgr-04.i/output/43144464.mgr-04.i.SC create mode 100644 examples/test_template/jobs/test_template-43144464.mgr-04.i/output/file_0.txt create mode 100644 examples/test_template/jobs/test_template-43144464.mgr-04.i/output/file_1.txt create mode 100644 examples/test_template/jobs/test_template-43144464.mgr-04.i/output/file_2.txt create mode 100644 examples/test_template/jobs/test_template-43144464.mgr-04.i/output/file_3.txt create mode 100644 examples/test_template/jobs/test_template-43144464.mgr-04.i/output/file_4.txt create mode 100644 examples/test_template/jobs/test_template-43144464.mgr-04.i/output/file_5.txt create mode 100644 examples/test_template/jobs/test_template-43144464.mgr-04.i/output/file_6.txt create mode 100644 examples/test_template/jobs/test_template-43144464.mgr-04.i/output/file_7.txt create mode 100644 examples/test_template/jobs/test_template-43144464.mgr-04.i/output/file_8.txt create mode 100644 examples/test_template/jobs/test_template-43144464.mgr-04.i/output/test_input.txt create mode 100644 examples/test_template/pbs.err/.keep create mode 100644 examples/test_template/pbs.err/test_template.e43144464 create mode 100644 examples/test_template/pbs.out/.keep create mode 100644 examples/test_template/pbs.out/test_template.o43144464 create mode 100644 examples/test_template/submit.org create mode 100644 examples/test_template/submit.org~ create mode 100755 examples/test_template/submit.pbs rename examples/sEH_NETS.pbs => sEH_NETS.pbs~ (99%) diff --git a/examples/test_template/input/.keep b/examples/test_template/input/.keep new file mode 100644 index 0000000..e69de29 diff --git a/examples/test_template/input/test_input.txt b/examples/test_template/input/test_input.txt new file mode 100644 index 0000000..fae8ec6 --- /dev/null +++ b/examples/test_template/input/test_input.txt @@ -0,0 +1 @@ +This is the test input file diff --git a/examples/test_template/jobs/.keep b/examples/test_template/jobs/.keep new file mode 100644 index 0000000..e69de29 diff --git a/examples/test_template/jobs/test_template-43144464.mgr-04.i/log b/examples/test_template/jobs/test_template-43144464.mgr-04.i/log new file mode 100644 index 0000000..01f35db --- /dev/null +++ b/examples/test_template/jobs/test_template-43144464.mgr-04.i/log @@ -0,0 +1,304 @@ +Starting Log file +---------------------------------------- + +---------------------------------------- +PBS Resource requests +---------------------------------------- + +walltime 5:00 +memory 4gb +num-nodes 1 +num-processors 1 +num-gpus +feature + +---------------------------------------- +Check Paths +---------------------------------------- + +LOG /mnt/research/DicksonLab/lotzsamu/pbs_templates/examples/test_template/jobs/test_template-43144464.mgr-04.i/log +PBS_JOBID 43144464.mgr-04.i +TASK test_template +TASKDIR /mnt/research/DicksonLab/lotzsamu/pbs_templates/examples/test_template +INPUTDIR /mnt/research/DicksonLab/lotzsamu/pbs_templates/examples/test_template/input +JOBSDIR /mnt/research/DicksonLab/lotzsamu/pbs_templates/examples/test_template/jobs +JOBNAME test_template-43144464.mgr-04.i +JOBDIR /mnt/research/DicksonLab/lotzsamu/pbs_templates/examples/test_template/jobs/test_template-43144464.mgr-04.i +OUTPUTDIR /mnt/research/DicksonLab/lotzsamu/pbs_templates/examples/test_template/jobs/test_template-43144464.mgr-04.i/output +EXECDIR /mnt/research/DicksonLab/lotzsamu/pbs_templates/examples/test_template/jobs/test_template-43144464.mgr-04.i/exec +STDOUT /mnt/research/DicksonLab/lotzsamu/pbs_templates/examples/test_template/jobs/test_template-43144464.mgr-04.i/pbs.out +STDERR /mnt/research/DicksonLab/lotzsamu/pbs_templates/examples/test_template/jobs/test_template-43144464.mgr-04.i/pbs.err +PBS_STDOUT /mnt/research/DicksonLab/lotzsamu/pbs_templates/examples/test_template/pbs.out +PBS_STDERR /mnt/research/DicksonLab/lotzsamu/pbs_templates/examples/test_template/pbs.err +PBS_NODEFILE +csn-005 + +---------------------------------------- +PBS Environmental Variables +---------------------------------------- + +PBS ENV: +PBS_VERSION=TORQUE-6.1.1.1 +PBS_JOBNAME=test_template +PBS_ENVIRONMENT=PBS_BATCH +PBS_O_WORKDIR=/mnt/research/DicksonLab/lotzsamu/pbs_templates/examples/test_template +PBS_TASKNUM=1 +PBS_O_HOME=/mnt/home/lotzsamu +PBS_WALLTIME=300 +PBS_MOMPORT=15003 +PBS_GPUFILE=/var/spool/torque/aux//43144464.mgr-04.igpu +PBS_O_QUEUE=main +PBS_O_LOGNAME=lotzsamu +PBS_O_LANG=en_US.UTF-8 +PBS_JOBCOOKIE=20F5CF25F4AFC0D0320BCE3436D2B1E6 +PBS_NODENUM=0 +PBS_NUM_NODES=1 +PBS_O_SHELL=/bin/bash +PBS_JOBID=43144464.mgr-04.i +PBS_O_HOST=dev-intel16-k80.i +PBS_VNODENUM=0 +PBS_QUEUE=main +PBS_O_MAIL=/var/spool/mail/lotzsamu +PBS_MICFILE=/var/spool/torque/aux//43144464.mgr-04.imic +PBS_NP=1 +PBS_O_SERVER=mgr-04 +PBS_NUM_PPN=1 +PBS_NODEFILE=/var/spool/torque/aux//43144464.mgr-04.i +PBS_O_PATH=/mnt/home/lotzsamu/anaconda3/bin:/mnt/research/DicksonLab/programs/mmtsb/perl:/mnt/research/DicksonLab/programs/charmm/exec/gnu:/opt/software/cudatoolkit/8.0/bin:/opt/software/OpenMPI/1.6.5--GCC-4.8.2/bin:/opt/software/GCC/4.8.3/bin:/opt/software/powertools/bin:/opt/software/Stata/13.0:/opt/software/MATLAB/R2014a/bin:/opt/software/Python/2.7.2--GCC-4.4.5/bin:/opt/software/cmake/2.8.5--GCC-4.4.5/bin:/opt/moab/bin:/mnt/home/lotzsamu/anaconda3/bin:/mnt/research/DicksonLab/programs/mmtsb/perl:/mnt/research/DicksonLab/programs/charmm/exec/gnu:/usr/lib64/qt-3.3/bin:/opt/software/lmod/bin:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/usr/local/hpcc/bin:/opt/ibutils/bin +PBS_JOBID number +PBS_JOBID_NUM: 43144464 +PBS_NODEFILE: +csn-005 +PBS_GPUFILE: +cat: /var/spool/torque/aux//43144464.mgr-04.igpu: No such file or directory +PBS_MICFILE: +cat: /var/spool/torque/aux//43144464.mgr-04.imic: No such file or directory + + +---------------------------------------- +Initialization +---------------------------------------- +------------ +RUNNING: source /etc/profile +------------ + +------------ +RUNNING: source /opt/software/modulefiles/setup_modules.sh +------------ + +Activating Modules: + 1) Boost/1.47.0 2) OpenMPI/1.4.3 3) R/2.15.1 4) TBB/4.1.0 + + +------------ +RUNNING: module load GNU/4.8.3 +------------ + +Inactive Modules: + 1) R + +Due to MODULEPATH changes the following have been reloaded: + 1) TBB/4.1.0 + +The following have been reloaded with a version change: + 1) Boost/1.47.0 => Boost/1.58.0 3) OpenMPI/1.4.3 => OpenMPI/1.6.5 + 2) GNU/4.4.5 => GNU/4.8.3 + + +------------ +RUNNING: module load CUDA/7.0 +------------ + +The following have been reloaded with a version change: + 1) CUDA/6.0 => CUDA/7.0 + + +------------ +Setting environmental variables +------------ +------------ +moving to EXECDIR: /mnt/research/DicksonLab/lotzsamu/pbs_templates/examples/test_template/jobs/test_template-43144464.mgr-04.i/exec +------------ + +------------ +Removing existing files if they exist in EXECDIR: /mnt/research/DicksonLab/lotzsamu/pbs_templates/examples/test_template/jobs/test_template-43144464.mgr-04.i/exec +------------ + +------------ +Copying input files from INPUTDIR: /mnt/research/DicksonLab/lotzsamu/pbs_templates/examples/test_template/input to EXECDIR: /mnt/research/DicksonLab/lotzsamu/pbs_templates/examples/test_template/jobs/test_template-43144464.mgr-04.i/exec +------------ + +------------ +Copying submission script /mnt/research/DicksonLab/lotzsamu/pbs_templates/examples/test_template/test_template to EXECDIR: /mnt/research/DicksonLab/lotzsamu/pbs_templates/examples/test_template/jobs/test_template-43144464.mgr-04.i/exec +------------ + +------------ +listing of EXECDIR: /mnt/research/DicksonLab/lotzsamu/pbs_templates/examples/test_template/jobs/test_template-43144464.mgr-04.i/exec +------------ +43144464.mgr-04.i.SC +test_input.txt + +------------ +Environmental variables before execution: +------------ +MKLROOT=/opt/software/ClusterStudio/2013sp1/composer_xe_2013_sp1.0.080/mkl +DL=/mnt/research/DicksonLab +MANPATH=/opt/software/OpenMPI/1.6.5--GCC-4.8.2/share/man:/opt/software/GCC/4.8.3/share/man:/opt/software/ClusterStudio/2013sp1/composer_xe_2013_sp1.0.080/man/en_US:/opt/software/Python/2.7.2--GCC-4.4.5/share/man:/opt/software/cmake/2.8.5--GCC-4.4.5/man:/usr/share/man +HOSTNAME=csn-005 +_ModuleTable003_=Il09ImFjdGl2ZSIsfSxHTlU9e1siRk4iXT0iL29wdC9zb2Z0d2FyZS9tb2R1bGVmaWxlcy9jb21waWxlcnMvR05VLzQuOC4zLmx1YSIsWyJkZWZhdWx0Il09MCxbImZ1bGxOYW1lIl09IkdOVS80LjguMyIsWyJsb2FkT3JkZXIiXT03LFsibVR5cGUiXT0idyIscHJvcFQ9e30sWyJzaG9ydCJdPSJHTlUiLFsic3RhdHVzIl09ImFjdGl2ZSIsfSxNQVRMQUI9e1siRk4iXT0iL29wdC9zb2Z0d2FyZS9tb2R1bGVmaWxlcy9iaW5hcmllcy9NQVRMQUIvUjIwMTRhLmx1YSIsWyJkZWZhdWx0Il09MSxbImZ1bGxOYW1lIl09Ik1BVExBQi9SMjAxNGEiLFsibG9hZE9yZGVyIl09NCxbIm1UeXBlIl09InciLHByb3BUPXt9LFsic2hvcnQiXT0iTUFUTEFCIixbInN0YXR1cyJdPSJhY3RpdmUi +PBS_VERSION=TORQUE-6.1.1.1 +SHELL=/bin/bash +HPCC_MODULES_PATH=/opt/software/modulefiles/ +HISTSIZE=1000 +PROFILEREAD=true +PBS_JOBNAME=test_template +TMPDIR=/mnt/local/43144464.mgr-04.i +MODULEPATH_ROOT=/opt/software/modulefiles +LMOD_DEFAULT_MODULEPATH=/opt/software/modulefiles/compilers:/opt/software/modulefiles/binaries +CPPFLAGS=-I/opt/software/cudatoolkit/7.0/include -I/opt/software/boost/1.58.0--GCC-4.8.2/include -I/opt/software/ClusterStudio/2011.0/composerxe-2011.5.220/tbb/include -I/opt/software/GCC/4.8.3/include -I/opt/software/ClusterStudio/2013sp1/composer_xe_2013_sp1.0.080/mkl/include/fftw -I/opt/software/ClusterStudio/2013sp1/composer_xe_2013_sp1.0.080/mkl/include -I/opt/software/Python/2.7.2--GCC-4.4.5/include +OPENMM_BIN=/mnt/research/DicksonLab/programs/OpenMM6.3/bin +PERL5LIB=/opt/moab/lib/perl5:/opt/moab/lib/perl5:/opt/moab/lib/perl5:/opt/moab/lib/perl5:/opt/moab/lib/perl5 +LIBRARY_PATH=/opt/software/cudatoolkit/7.0/lib64:/opt/software/cudatoolkit/7.0/lib:/opt/software/boost/1.58.0--GCC-4.8.2/lib:/opt/software/ClusterStudio/2011.0/composerxe-2011.5.220/tbb/lib/intel64/cc4.1.0_libc2.4_kernel2.6.16.21/irml:/opt/software/ClusterStudio/2011.0/composerxe-2011.5.220/tbb/lib/intel64/cc4.1.0_libc2.4_kernel2.6.16.21:/opt/software/OpenMPI/1.6.5--GCC-4.8.2/lib:/opt/software/GCC/4.8.3/lib64:/opt/software/GCC/4.8.3/lib:/opt/software/ClusterStudio/2013sp1/composer_xe_2013_sp1.0.080/mkl/lib/intel64:/opt/software/Python/2.7.2--GCC-4.4.5/lib +MPICHARMM=/mnt/research/DicksonLab/programs/charmm/exec/gnu_M/charmm +MMTSBDIR=/mnt/research/DicksonLab/programs/mmtsb +PBS_ENVIRONMENT=PBS_BATCH +OLDPWD=/mnt/home/lotzsamu +QTDIR=/usr/lib64/qt-3.3 +QTINC=/usr/lib64/qt-3.3/include +PBS_O_WORKDIR=/mnt/research/DicksonLab/lotzsamu/pbs_templates/examples/test_template +MIC_LD_LIBRARY_PATH=/opt/software/ClusterStudio/2013sp1/composer_xe_2013_sp1.0.080/mkl/lib/mic +Matlab_ROOT_DIR=/opt/software/MATLAB/R2014a/bin +_ModuleTable007_=cy8xLjIiLFsibG9hZE9yZGVyIl09MSxbIm1UeXBlIl09InciLHByb3BUPXt9LFsic2hvcnQiXT0icG93ZXJ0b29scyIsWyJzdGF0dXMiXT0iYWN0aXZlIix9LH0sbXBhdGhBPXsiL29wdC9zb2Z0d2FyZS9tb2R1bGVmaWxlcy9DVURBLzcuMC9tb2R1bGVmaWxlcyIsIi9vcHQvc29mdHdhcmUvbW9kdWxlZmlsZXMvR05VLzQuOC4zL09wZW5NUEkvMS42LjUiLCIvb3B0L3NvZnR3YXJlL21vZHVsZWZpbGVzL0dOVS80LjguMy9tb2R1bGVmaWxlcyIsIi9vcHQvc29mdHdhcmUvbW9kdWxlZmlsZXMvR05VLzQuOC4yL21vZHVsZWZpbGVzIiwiL29wdC9zb2Z0d2FyZS9tb2R1bGVmaWxlcy9QeXRob24vMi43LjIiLCIvb3B0L3NvZnR3YXJlL21vZHVsZWZpbGVzL2NvbXBpbGVycyIsIi9v +ANACONDA_BIN=/mnt/research/DicksonLab/programs/anaconda2/bin +USER=lotzsamu +PBS_TASKNUM=1 +LMOD_sys=Linux +LD_LIBRARY_PATH=/mnt/research/DicksonLab/programs/lib:/opt/software/cudatoolkit/7.0/lib:/mnt/research/DicksonLab/programs/OpenMM6.3/lib:/mnt/research/DicksonLab/programs/OpenMM6.3/lib/plugins:/opt/software/cudatoolkit/7.0/computeprof/bin:/opt/software/cudatoolkit/7.0/lib64:/opt/software/cudatoolkit/7.0/lib:/opt/software/boost/1.58.0--GCC-4.8.2/lib:/opt/software/ClusterStudio/2011.0/composerxe-2011.5.220/tbb/lib/intel64/cc4.1.0_libc2.4_kernel2.6.16.21/irml:/opt/software/ClusterStudio/2011.0/composerxe-2011.5.220/tbb/lib/intel64/cc4.1.0_libc2.4_kernel2.6.16.21:/opt/software/OpenMPI/1.6.5--GCC-4.8.2/lib:/opt/software/GMP/5.1.3--GCC-4.4.5/lib:/opt/software/MPFR/3.1.2--GCC-4.4.5/lib:/opt/software/MPC/1.0.1--GCC-4.4.5/lib:/opt/software/GCC/4.8.3/lib64:/opt/software/GCC/4.8.3/lib:/opt/software/ClusterStudio/2013sp1/composer_xe_2013_sp1.0.080/mkl/lib/intel64:/opt/software/MATLAB/R2014a/sys/opengl/lib/glnxa64:/opt/software/Python/2.7.2--GCC-4.4.5/lib/python2.7/lib-dynload:/opt/software/Python/2.7.2--GCC-4.4.5/lib:/mnt/research/DicksonLab/programs/lib +MY_DICKSON_LAB=/mnt/research/DicksonLab/lotzsamu +PBS_O_HOME=/mnt/home/lotzsamu +DICKSON_LAB_PROGS=/mnt/research/DicksonLab/programs +PTOOLS_ROOT=/opt/software/powertools +_ModuleTable004_=LH0sTUtMPXtbIkZOIl09Ii9vcHQvc29mdHdhcmUvbW9kdWxlZmlsZXMvYmluYXJpZXMvTUtMLzExLjEubHVhIixbImRlZmF1bHQiXT0wLFsiZnVsbE5hbWUiXT0iTUtMLzExLjEiLFsibG9hZE9yZGVyIl09NixbIm1UeXBlIl09InciLHByb3BUPXt9LFsic2hvcnQiXT0iTUtMIixbInN0YXR1cyJdPSJhY3RpdmUiLH0sT3Blbk1QST17WyJGTiJdPSIvb3B0L3NvZnR3YXJlL21vZHVsZWZpbGVzL0dOVS80LjguMy9tb2R1bGVmaWxlcy9PcGVuTVBJLzEuNi41Lmx1YSIsWyJkZWZhdWx0Il09MCxbImZ1bGxOYW1lIl09Ik9wZW5NUEkvMS42LjUiLFsibG9hZE9yZGVyIl09OCxbIm1UeXBlIl09InciLHByb3BUPXt9LFsic2hvcnQiXT0iT3Blbk1QSSIsWyJzdGF0dXMiXT0iYWN0aXZl +SALOTZ_SCRATCH=/mnt/ls15/scratch/users/lotzsamu/users/lotzsamu +PBS_WALLTIME=300 +PBS_MOMPORT=15003 +PBS_GPUFILE=/var/spool/torque/aux//43144464.mgr-04.igpu +DICKSON_LAB=/mnt/research/DicksonLab +PBS_O_QUEUE=main +SCRATCHDIR=/mnt/ls15/scratch +OPENMM_PLATFORM=CUDA +PATH=/mnt/research/DicksonLab/programs/anaconda2/bin:/mnt/research/DicksonLab/programs/bin:/mnt/research/DicksonLab/programs/OpenMM6.3/bin:/mnt/research/DicksonLab/programs/mmtsb/perl:/mnt/research/DicksonLab/programs/charmm/exec/gnu:/opt/software/cudatoolkit/7.0/computeprof/bin:/opt/software/cudatoolkit/7.0/bin:/opt/software/OpenMPI/1.6.5--GCC-4.8.2/bin:/opt/software/GCC/4.8.3/bin:/opt/software/Stata/13.0:/opt/software/MATLAB/R2014a/bin:/opt/software/Python/2.7.2--GCC-4.4.5/bin:/opt/software/cmake/2.8.5--GCC-4.4.5/bin:/opt/moab/bin:/mnt/home/lotzsamu/anaconda3/bin:/mnt/research/DicksonLab/programs/mmtsb/perl:/mnt/research/DicksonLab/programs/charmm/exec/gnu:/opt/software/powertools/bin:/usr/lib64/qt-3.3/bin:/opt/software/lmod/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/usr/local/hpcc/bin:/usr/local/bin:/opt/ibutils/bin +PBS_O_LOGNAME=lotzsamu +MAIL=/var/spool/mail/lotzsamu +CUDA_INSTALL_PATH=/opt/software/cudatoolkit/7.0 +_ModuleTable001_=X01vZHVsZVRhYmxlXz17WyJhY3RpdmVTaXplIl09MTEsYmFzZU1wYXRoQT17Ii9vcHQvc29mdHdhcmUvbW9kdWxlZmlsZXMvL2NvbXBpbGVycyIsIi9vcHQvc29mdHdhcmUvbW9kdWxlZmlsZXMvL2JpbmFyaWVzIix9LFsiY19yZWJ1aWxkVGltZSJdPWZhbHNlLGZhbWlseT17WyJNUEkiXT0iT3Blbk1QSSIsWyJjb21waWxlciJdPSJHTlUiLH0saW5hY3RpdmU9e30sbVQ9e0Jvb3N0PXtbIkZOIl09Ii9vcHQvc29mdHdhcmUvbW9kdWxlZmlsZXMvR05VLzQuOC4yL21vZHVsZWZpbGVzL0Jvb3N0LzEuNTguMC5sdWEiLFsiZGVmYXVsdCJdPTAsWyJmdWxsTmFtZSJdPSJCb29zdC8xLjU4LjAiLFsibG9hZE9yZGVyIl09MTAsWyJtVHlwZSJdPSJ3Iixwcm9wVD17fSxbInNob3J0Il09 +PBS_O_LANG=en_US.UTF-8 +PBS_JOBCOOKIE=20F5CF25F4AFC0D0320BCE3436D2B1E6 +MKL_NUM_THREADS=1 +OPENMM_PLUGIN=/mnt/research/DicksonLab/programs/OpenMM6.3/lib/lib/plugins +PWD=/mnt/research/DicksonLab/lotzsamu/pbs_templates/examples/test_template/jobs/test_template-43144464.mgr-04.i/exec +MLM_LICENSE_FILE=27000@lm-01.i +DICKSON=/mnt/research/DicksonLab +MY_DICKSON_SCRATCH=/mnt/ls15/scratch/groups/DicksonLab/lotzsamu +LANG=en_US.UTF-8 +PBS_NODENUM=0 +MOABHOMEDIR=/opt/moab +MODULEPATH=/opt/software/modulefiles/CUDA/7.0/modulefiles:/opt/software/modulefiles/GNU/4.8.3/OpenMPI/1.6.5:/opt/software/modulefiles/GNU/4.8.3/modulefiles:/opt/software/modulefiles/GNU/4.8.2/modulefiles:/opt/software/modulefiles/Python/2.7.2:/opt/software/modulefiles/compilers:/opt/software/modulefiles/compat:/opt/software/modulefiles/binaries +OPENMM_ROOT_DIR=/mnt/research/DicksonLab/programs/OpenMM6.3 +PBS_NUM_NODES=1 +KDEDIRS=/usr +CUDADIR=/opt/software/cudatoolkit/7.0 +_ModuleTable_Sz_=8 +MPC=/mpc +OPENMM_LIB=/mnt/research/DicksonLab/programs/OpenMM6.3/lib +PYTHONHOME=/mnt/research/DicksonLab/programs/anaconda2 +CUDA_SDK_ROOT_DIR=/opt/software/cudatoolkit/7.0 +PBS_O_SHELL=/bin/bash +F77=/opt/software/GCC/4.8.3/bin/gfortran +LMOD_CMD=/opt/software/lmod/4.1.4icer5/libexec/lmod +_ModuleTable005_=Iix9LFB5dGhvbj17WyJGTiJdPSIvb3B0L3NvZnR3YXJlL21vZHVsZWZpbGVzL2JpbmFyaWVzL1B5dGhvbi8yLjcuMi5sdWEiLFsiZGVmYXVsdCJdPTEsWyJmdWxsTmFtZSJdPSJQeXRob24vMi43LjIiLFsibG9hZE9yZGVyIl09MyxbIm1UeXBlIl09InciLHByb3BUPXt9LFsic2hvcnQiXT0iUHl0aG9uIixbInN0YXR1cyJdPSJhY3RpdmUiLH0sUj17WyJmdWxsTmFtZSJdPSJSIixbImxvYWRPcmRlciJdPTE0LHByb3BUPXt9LFsic2hvcnQiXT0iUiIsWyJzdGF0dXMiXT0iaW5hY3RpdmUiLH0sU3RhdGE9e1siRk4iXT0iL29wdC9zb2Z0d2FyZS9tb2R1bGVmaWxlcy9iaW5hcmllcy9TdGF0YS8xMy4wLmx1YSIsWyJkZWZhdWx0Il09MSxbImZ1bGxOYW1lIl09IlN0YXRhLzEzLjAi +DICKSON_LAB_LIB=/mnt/research/DicksonLab/programs/lib +PBS_JOBID=43144464.mgr-04.i +DICKSON_SCRATCH=/mnt/ls15/scratch/groups/DicksonLab +ENVIRONMENT=BATCH +HISTCONTROL=ignoredups +CXX=/opt/software/GCC/4.8.3/bin/g++ +OPENMM_CUDA_COMPILER=/opt/software/cudatoolkit/8.0/bin/nvcc +BOOST_INCLUDEDIR=/opt/software/boost/1.58.0--GCC-4.8.2/include/ +HOME=/mnt/home/lotzsamu +SHLVL=2 +SETENV_DIR=/mnt/research/DicksonLab/.setenv_hpcc.d +TACC_FAMILY_COMPILER=GNU +CHARMM=/mnt/research/DicksonLab/programs/charmm/exec/gnu/charmm +ANACONDA_DIR=/mnt/research/DicksonLab/programs/anaconda2 +PBS_O_HOST=dev-intel16-k80.i +_ModuleTable002_=IkJvb3N0IixbInN0YXR1cyJdPSJhY3RpdmUiLH0sQ01ha2U9e1siRk4iXT0iL29wdC9zb2Z0d2FyZS9tb2R1bGVmaWxlcy9iaW5hcmllcy9DTWFrZS8yLjguNS5sdWEiLFsiZGVmYXVsdCJdPTEsWyJmdWxsTmFtZSJdPSJDTWFrZS8yLjguNSIsWyJsb2FkT3JkZXIiXT0yLFsibVR5cGUiXT0idyIscHJvcFQ9e30sWyJzaG9ydCJdPSJDTWFrZSIsWyJzdGF0dXMiXT0iYWN0aXZlIix9LENVREE9e1siRk4iXT0iL29wdC9zb2Z0d2FyZS9tb2R1bGVmaWxlcy9jb21waWxlcnMvQ1VEQS83LjAubHVhIixbImRlZmF1bHQiXT0wLFsiZnVsbE5hbWUiXT0iQ1VEQS83LjAiLFsibG9hZE9yZGVyIl09MTEsWyJtVHlwZSJdPSJ3Iixwcm9wVD17fSxbInNob3J0Il09IkNVREEiLFsic3RhdHVz +FC=/opt/software/GCC/4.8.3/bin/gfortran +BOOST_LIBRARYDIR=/opt/software/boost/1.58.0--GCC-4.8.2/lib/ +PBS_VNODENUM=0 +BASH_ENV=/opt/software/lmod/lmod/init/bash +_ModuleTable008_=cHQvc29mdHdhcmUvbW9kdWxlZmlsZXMvY29tcGF0IiwiL29wdC9zb2Z0d2FyZS9tb2R1bGVmaWxlcy9iaW5hcmllcyIsfSxbInZlcnNpb24iXT0yLH0= +TACC_FAMILY_MPI=OpenMPI +LOGNAME=lotzsamu +LMOD_arch=x86_64 +PYTHONPATH=:/mnt/home/lotzsamu:/mnt/home/lotzsamu +OPENMM_PLUGIN_DIR=/mnt/research/DicksonLab/programs/OpenMM6.3/lib/plugins +BOOST_ROOT=/opt/software/boost/1.58.0--GCC-4.8.2 +CVS_RSH=ssh +QTLIB=/usr/lib64/qt-3.3/lib +PBS_QUEUE=main +MODULESHOME= +PBS_O_MAIL=/var/spool/mail/lotzsamu +PBS_MICFILE=/var/spool/torque/aux//43144464.mgr-04.imic +LESSOPEN=||/usr/bin/lesspipe.sh %s +OMP_NUM_THREADS=1 +PKG_CONFIG_PATH=/opt/software/GCC/4.8.3/lib/pkgconfig:/opt/software/Python/2.7.2--GCC-4.4.5/lib/pkgconfig +DICKSON_LAB_BIN=/mnt/research/DicksonLab/programs/bin +HPCC_CLUSTER_FLAVOR=intel14-k20 +is=bash +EMAIL=$USER@msu.edu +OPENMMCHARMM=/mnt/research/DicksonLab/programs/charmm/exec/gnu/charmm_OpenMM6.3 +LMOD_FAMILY_COMPILER=GNU +SYS_SOFTWARE=/opt/software +PBS_NP=1 +CC=/opt/software/GCC/4.8.3/bin/gcc +SS=/mnt/ls15/scratch/users/lotzsamu/users/lotzsamu +CHARMM_DIR=/mnt/research/DicksonLab/programs/charmm +PBS_O_SERVER=mgr-04 +PBS_NUM_PPN=1 +LMOD_DIR=/opt/software/lmod/4.1.4icer5/libexec/ +_ModuleTable006_=LFsibG9hZE9yZGVyIl09NSxbIm1UeXBlIl09InciLHByb3BUPXt9LFsic2hvcnQiXT0iU3RhdGEiLFsic3RhdHVzIl09ImFjdGl2ZSIsfSxUQkI9e1siRk4iXT0iL29wdC9zb2Z0d2FyZS9tb2R1bGVmaWxlcy9HTlUvNC44LjIvbW9kdWxlZmlsZXMvVEJCLzQuMS4wLmx1YSIsWyJkZWZhdWx0Il09MCxbImZ1bGxOYW1lIl09IlRCQi80LjEuMCIsWyJsb2FkT3JkZXIiXT05LFsibVR5cGUiXT0idyIscHJvcFQ9e30sWyJzaG9ydCJdPSJUQkIiLFsic3RhdHVzIl09ImFjdGl2ZSIsfSxwb3dlcnRvb2xzPXtbIkZOIl09Ii9vcHQvc29mdHdhcmUvbW9kdWxlZmlsZXMvYmluYXJpZXMvcG93ZXJ0b29scy8xLjIubHVhIixbImRlZmF1bHQiXT0xLFsiZnVsbE5hbWUiXT0icG93ZXJ0b29s +SCRATCH_HOME=/mnt/ls15/scratch/users/lotzsamu +LMOD_FAMILY_MPI=OpenMPI +PBS_NODEFILE=/var/spool/torque/aux//43144464.mgr-04.i +G_BROKEN_FILENAMES=1 +SCRATCH=/mnt/ls15/scratch/users/lotzsamu +CHARMMDATA=/mnt/research/DicksonLab/toppar +PBS_O_PATH=/mnt/home/lotzsamu/anaconda3/bin:/mnt/research/DicksonLab/programs/mmtsb/perl:/mnt/research/DicksonLab/programs/charmm/exec/gnu:/opt/software/cudatoolkit/8.0/bin:/opt/software/OpenMPI/1.6.5--GCC-4.8.2/bin:/opt/software/GCC/4.8.3/bin:/opt/software/powertools/bin:/opt/software/Stata/13.0:/opt/software/MATLAB/R2014a/bin:/opt/software/Python/2.7.2--GCC-4.4.5/bin:/opt/software/cmake/2.8.5--GCC-4.4.5/bin:/opt/moab/bin:/mnt/home/lotzsamu/anaconda3/bin:/mnt/research/DicksonLab/programs/mmtsb/perl:/mnt/research/DicksonLab/programs/charmm/exec/gnu:/usr/lib64/qt-3.3/bin:/opt/software/lmod/bin:/usr/local/bin:/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/sbin:/usr/local/hpcc/bin:/opt/ibutils/bin +FFLAGS=-I/opt/software/ClusterStudio/2013sp1/composer_xe_2013_sp1.0.080/mkl/include/intel64/lp64 -I/opt/software/ClusterStudio/2013sp1/composer_xe_2013_sp1.0.080/mkl/include/fftw -I/opt/software/ClusterStudio/2013sp1/composer_xe_2013_sp1.0.080/mkl/include +_=/bin/env + +------------ +Setting Script parameters +------------ + + +------------ +Running script +=============================================================================== +This is the test input file +=============================================================================== +done with script +------------ + +------------ +moving EXECDIR /mnt/research/DicksonLab/lotzsamu/pbs_templates/examples/test_template/jobs/test_template-43144464.mgr-04.i/exec to OUTPUTDIR /mnt/research/DicksonLab/lotzsamu/pbs_templates/examples/test_template/jobs/test_template-43144464.mgr-04.i/output +------------ + +------------ +PBS STDOUT is in /mnt/research/DicksonLab/lotzsamu/pbs_templates/examples/test_template/pbs.out/test_template.o43144464 +------------ + +------------ +PBS STDERR /mnt/research/DicksonLab/lotzsamu/pbs_templates/examples/test_template/pbs.err/test_template.e43144464 +------------ diff --git a/examples/test_template/jobs/test_template-43144464.mgr-04.i/output/43144464.mgr-04.i.SC b/examples/test_template/jobs/test_template-43144464.mgr-04.i/output/43144464.mgr-04.i.SC new file mode 100755 index 0000000..ef2bc6e --- /dev/null +++ b/examples/test_template/jobs/test_template-43144464.mgr-04.i/output/43144464.mgr-04.i.SC @@ -0,0 +1,321 @@ +#!/bin/sh -login + +#PBS -N test_template +#PBS -l walltime=5:00 +#PBS -l mem=4gb +#PBS -l nodes=1:ppn=1 + + +#PBS -e pbs.err +#PBS -o pbs.out +#PBS -M samuel.lotz@gmail.com +#PBS -m abe + +# ------------------------------ +# Script for running the task test_template with PBS on hpcc + +# task name, this is the name of the directory for the task +TASK="test_template" +# directory path for this task +TASKDIR="$MY_DICKSON_LAB/pbs_templates/examples/$TASK" + +# the directory with all of the input +INPUTDIR=$TASKDIR/input +# directory to put job results in +JOBSDIR=$TASKDIR/jobs +# we should redirect our own output streams but we will catch the PBS +# files as well +PBS_STDOUT=$PBS_O_WORKDIR/pbs.out +PBS_STDERR=$PBS_O_WORKDIR/pbs.err + +# the job name +JOBNAME=${TASK}-${PBS_JOBID} +# the directory for the output of this job +JOBDIR=$JOBSDIR/$JOBNAME +# the directory output will be put into after completion +OUTPUTDIR=$JOBDIR/output +# log files +LOG=$JOBDIR/log +# directories to copy the stdout and stderr from scripts +STDOUT=$JOBDIR/pbs.out +STDERR=$JOBDIR/pbs.err + +# directory to execute code in +# EXECDIR=$TMPDIR +# EXECDIR=$MY_DICKSON_SCRATCH/$JOBNAME +# Problems with Scratch so I will just run in the jobs dir +# TODO make an if for different options +EXECDIR=$JOBDIR/exec + +# make these directories if they do not exist +mkdir -p $JOBDIR +mkdir -p $EXECDIR +mkdir -p $PBS_STDOUT +mkdir -p $PBS_STDERR + +# Starting the LOG file +echo "Starting Log file" 1> $LOG 2> $LOG +echo "----------------------------------------" 1>> $LOG 2>> $LOG +echo "" 1>> $LOG 2>> $LOG + +# write out explicitly the run parameters +echo "----------------------------------------" 1>> $LOG 2>> $LOG +echo "PBS Resource requests" 1>> $LOG 2>> $LOG +echo "----------------------------------------" 1>> $LOG 2>> $LOG + +echo "" 1>> $LOG 2>> $LOG +echo "walltime 5:00" 1>> $LOG 2>> $LOG +echo "memory 4gb" 1>> $LOG 2>> $LOG +echo "num-nodes 1" 1>> $LOG 2>> $LOG +echo "num-processors 1" 1>> $LOG 2>> $LOG +echo "num-gpus " 1>> $LOG 2>> $LOG +echo "feature " 1>> $LOG 2>> $LOG +echo "" 1>> $LOG 2>> $LOG + +# print the paths for all the variables constructed in this script +echo "----------------------------------------" 1>> $LOG 2>> $LOG +echo "Check Paths" 1>> $LOG 2>> $LOG +echo "----------------------------------------" 1>> $LOG 2>> $LOG + +echo "" 1>> $LOG 2>> $LOG +echo "LOG $LOG" 1>> $LOG 2>> $LOG +echo "PBS_JOBID $PBS_JOBID" 1>> $LOG 2>> $LOG +echo "TASK $TASK" 1>> $LOG 2>> $LOG +echo "TASKDIR $TASKDIR" 1>> $LOG 2>> $LOG +echo "INPUTDIR $INPUTDIR" 1>> $LOG 2>> $LOG +echo "JOBSDIR $JOBSDIR" 1>> $LOG 2>> $LOG +echo "JOBNAME $JOBNAME" 1>> $LOG 2>> $LOG +echo "JOBDIR $JOBDIR" 1>> $LOG 2>> $LOG +echo "OUTPUTDIR $OUTPUTDIR" 1>> $LOG 2>> $LOG +echo "EXECDIR $EXECDIR" 1>> $LOG 2>> $LOG +echo "STDOUT $STDOUT" 1>> $LOG 2>> $LOG +echo "STDERR $STDERR" 1>> $LOG 2>> $LOG +echo "PBS_STDOUT $PBS_STDOUT" 1>> $LOG 2>> $LOG +echo "PBS_STDERR $PBS_STDERR" 1>> $LOG 2>> $LOG +echo "PBS_NODEFILE" 1>> $LOG 2>> $LOG +cat $PBS_NODEFILE 1>> $LOG 2>> $LOG +echo "" 1>> $LOG 2>> $LOG + +# saving PBS environmental variables +echo "----------------------------------------" 1>> $LOG 2>> $LOG +echo "PBS Environmental Variables" 1>> $LOG 2>> $LOG +echo "----------------------------------------" 1>> $LOG 2>> $LOG + +echo "" 1>> $LOG 2>> $LOG +# PBS env vars +echo "PBS ENV:" 1>> $LOG 2>> $LOG +env | grep PBS 1>> $LOG 2>> $LOG + +# PBS JOBID number +echo "PBS_JOBID number" 1>> $LOG 2>> $LOG +PBS_JOBID_NUM=(${PBS_JOBID//./ }) +echo "PBS_JOBID_NUM: $PBS_JOBID_NUM" 1>> $LOG 2>> $LOG + +# PBS stdout and stderr file paths +PBS_JOB_STDOUT="$PBS_STDOUT/$TASK.o$PBS_JOBID_NUM" +PBS_JOB_STDERR="$PBS_STDERR/$TASK.e$PBS_JOBID_NUM" + +# NODEFILE +echo "PBS_NODEFILE:" 1>> $LOG 2>> $LOG +cat $PBS_NODEFILE 1>> $LOG 2>> $LOG +# GPUFILE +echo "PBS_GPUFILE:" 1>> $LOG 2>> $LOG +cat $PBS_GPUFILE 1>> $LOG 2>> $LOG +# MICFILE +echo "PBS_MICFILE:" 1>> $LOG 2>> $LOG +cat $PBS_MICFILE 1>> $LOG 2>> $LOG +echo "" 1>> $LOG 2>> $LOG + +# initial +echo "" 1>> $LOG 2>> $LOG +echo "----------------------------------------" 1>> $LOG 2>> $LOG +echo "Initialization" 1>> $LOG 2>> $LOG +echo "----------------------------------------" 1>> $LOG 2>> $LOG + +# load profile +echo "------------" 1>> $LOG 2>> $LOG +echo "RUNNING: source /etc/profile" 1>> $LOG 2>> $LOG +echo "------------" 1>> $LOG 2>> $LOG +source /etc/profile 1>> $LOG 2>> $LOG +echo "" 1>> $LOG 2>> $LOG + +# load hpcc modules +echo "------------" 1>> $LOG 2>> $LOG +echo "RUNNING: source /opt/software/modulefiles/setup_modules.sh" 1>> $LOG 2>> $LOG +echo "------------" 1>> $LOG 2>> $LOG +source /opt/software/modulefiles/setup_modules.sh 1>> $LOG 2>> $LOG +echo "" 1>> $LOG 2>> $LOG + +# load specific library modules +# GNU Compilers +echo "------------" 1>> $LOG 2>> $LOG +echo "RUNNING: module load GNU/4.8.3" 1>> $LOG 2>> $LOG +echo "------------" 1>> $LOG 2>> $LOG +module load GNU/4.8.3 1>> $LOG 2>> $LOG +echo "" 1>> $LOG 2>> $LOG + +# CUDA compilers +echo "------------" 1>> $LOG 2>> $LOG +echo "RUNNING: module load CUDA/7.0" 1>> $LOG 2>> $LOG +echo "------------" 1>> $LOG 2>> $LOG +module load CUDA/7.0 1>> $LOG 2>> $LOG +echo "" 1>> $LOG 2>> $LOG + + +# set environmental variables and other local variables that are used for +# many types of scripts +# =============================================================================== +echo "------------" 1>> $LOG 2>> $LOG +echo "Setting environmental variables" 1>> $LOG 2>> $LOG +echo "------------" 1>> $LOG 2>> $LOG + +# scratch directory +export SCRATCHDIR=/mnt/ls15/scratch +# your home scratch +export SCRATCH_HOME=$SCRATCH + +# the Dickson lab directory +export DICKSON_LAB=/mnt/research/DicksonLab +# your Dickson lab dir +export MY_DICKSON_LAB=$DICKSON_LAB/$USER +# locally installed programs +export DICKSON_LAB_PROGS=${DICKSON_LAB}/programs +# the installed binaries for Dickson lab +export DICKSON_LAB_BIN=${DICKSON_LAB_PROGS}/bin +# directory with installed libraries +export DICKSON_LAB_LIB=${DICKSON_LAB_PROGS}/lib +# Dickson lab scratch directory +export DICKSON_SCRATCH=$SCRATCHDIR/groups/DicksonLab +# your Dickson lab scratch dir +export MY_DICKSON_SCRATCH=$DICKSON_SCRATCH/$USER + +# set environmental variables to programs needed +#---------------------------------------- +# charmm executables +export CHARMM_DIR=${DICKSON_LAB_PROGS}/charmm +export CHARMM=${CHARMM_DIR}/exec/gnu/charmm +export MPICHARMM=${CHARMM_DIR}/exec/gnu_M/charmm +export OPENMMCHARMM=${CHARMM_DIR}/exec/gnu/charmm_OpenMM6.3 + +# charmm data +export CHARMMDATA=${DICKSON_LAB}/toppar + +# set the path for mmtsb +export MMTSBDIR=$DICKSON_LAB_PROGS/mmtsb + +# OpenMM +export OPENMM_ROOT_DIR=/mnt/research/DicksonLab/programs/OpenMM6.3 +export OPENMM_PLUGIN_DIR=$OPENMM_ROOT_DIR/lib/plugins +export OPENMM_PLATFORM=CUDA +unset OPENMM_DEVICE + + +# Anaconda Python paths +export ANACONDA_DIR=$DICKSON_LAB_PROGS/anaconda2 +export ANACONDA_BIN=$ANACONDA_DIR/bin +export PYTHONHOME=$ANACONDA_DIR + +# Library paths +export LD_LIBRARY_PATH=${DICKSON_LAB_LIB}:${CUDADIR}/lib:${OPENMM_ROOT_DIR}/lib:${OPENMM_PLUGIN_DIR}:${LD_LIBRARY_PATH} + +# PATH +export PATH=$ANACONDA_BIN:${DICKSON_LAB_BIN}:${OPENMM_ROOT_DIR}/bin:${MMTSBDIR}/perl:${DICKSON_LAB_PROGS}/charmm/exec/gnu:${PATH} + +# =============================================================================== + +# change to the exec dir +echo "------------" 1>> $LOG 2>> $LOG +echo "moving to EXECDIR: $EXECDIR" 1>> $LOG 2>> $LOG +echo "------------" 1>> $LOG 2>> $LOG +cd $EXECDIR 1>> $LOG 2>> $LOG +echo "" 1>> $LOG 2>> $LOG + +# remove current contents of the execdir, useful for if running +# interactive job which writes to same dir, harmless if not +echo "------------" 1>> $LOG 2>> $LOG +echo "Removing existing files if they exist in EXECDIR: $EXECDIR" 1>> $LOG 2>> $LOG +echo "------------" 1>> $LOG 2>> $LOG +rm -rf $EXECDIR/* 1>> $LOG 2>> $LOG +echo "" 1>> $LOG 2>> $LOG + +# copy the input files to the execution directory +echo "------------" 1>> $LOG 2>> $LOG +echo "Copying input files from INPUTDIR: $INPUTDIR to EXECDIR: $EXECDIR" 1>> $LOG 2>> $LOG +echo "------------" 1>> $LOG 2>> $LOG +cp -rf $INPUTDIR/* $EXECDIR/ 1>> $LOG 2>> $LOG +echo "" 1>> $LOG 2>> $LOG + +# copy the actual submission script used +echo "------------" 1>> $LOG 2>> $LOG +echo "Copying submission script ${TASKDIR}/${PBS_JOBNAME} to EXECDIR: $EXECDIR" 1>> $LOG 2>> $LOG +echo "------------" 1>> $LOG 2>> $LOG +cp "${0}" $EXECDIR/ 1>> $LOG 2>> $LOG +echo "" 1>> $LOG 2>> $LOG + +# write file names in $EXECDIR to log +echo "------------" 1>> $LOG 2>> $LOG +echo "listing of EXECDIR: $EXECDIR" 1>> $LOG 2>> $LOG +echo "------------" 1>> $LOG 2>> $LOG +ls $EXECDIR 1>> $LOG 2>> $LOG +echo "" 1>> $LOG 2>> $LOG + +# print out the environmental variables after modifications +echo "------------" 1>> $LOG 2>> $LOG +echo "Environmental variables before execution:" 1>> $LOG 2>> $LOG +echo "------------" 1>> $LOG 2>> $LOG +env 1>> $LOG 2>> $LOG +echo "" 1>> $LOG 2>> $LOG + +# ------------------------------ +# set the parameters that will be used in this script +# =============================================================================== +echo "------------" 1>> $LOG 2>> $LOG +echo "Setting Script parameters" 1>> $LOG 2>> $LOG +echo "------------" 1>> $LOG 2>> $LOG +echo "" 1>> $LOG 2>> $LOG + +TEST_PARAM="HELLO" +TEST_VAL=8 +TEST_FILE="test_input.txt" + +echo "" 1>> $LOG 2>> $LOG +# =============================================================================== + + +# ------------------------------ +# The code for this script +# =============================================================================== +echo "------------" 1>> $LOG 2>> $LOG +echo "Running script" 1>> $LOG 2>> $LOG +echo "===============================================================================" 1>> $LOG 2>> $LOG + +echo "HELLO beginning the script body" +cat $TEST_FILE 1>> $LOG 2>> $LOG +for ((x=0; x<=${TEST_VAL}; x++)); do + echo ${TEST_PARAM} > "file_${x}.txt" 2>> $LOG; +done + +echo "===============================================================================" 1>> $LOG 2>> $LOG +echo "done with script" 1>> $LOG 2>> $LOG +echo "------------" 1>> $LOG 2>> $LOG +echo "" 1>> $LOG 2>> $LOG + +# =============================================================================== + +# move the output files in EXECDIR back to the job output dir +echo "------------" 1>> $LOG 2>> $LOG +echo "moving EXECDIR $EXECDIR to OUTPUTDIR $OUTPUTDIR" 1>> $LOG 2>> $LOG +echo "------------" 1>> $LOG 2>> $LOG +mv $EXECDIR $OUTPUTDIR 1>> $LOG 2>> $LOG +echo "" 1>> $LOG 2>> $LOG + + +# move the PBS stdout and stderr files to the jobdir +echo "------------" 1>> $LOG 2>> $LOG +echo "PBS STDOUT is in $PBS_JOB_STDOUT" 1>> $LOG 2>> $LOG +echo "------------" 1>> $LOG 2>> $LOG +echo "" 1>> $LOG 2>> $LOG + +echo "------------" 1>> $LOG 2>> $LOG +echo "PBS STDERR $PBS_JOB_STDERR" 1>> $LOG 2>> $LOG +echo "------------" 1>> $LOG 2>> $LOG diff --git a/examples/test_template/jobs/test_template-43144464.mgr-04.i/output/file_0.txt b/examples/test_template/jobs/test_template-43144464.mgr-04.i/output/file_0.txt new file mode 100644 index 0000000..e427984 --- /dev/null +++ b/examples/test_template/jobs/test_template-43144464.mgr-04.i/output/file_0.txt @@ -0,0 +1 @@ +HELLO diff --git a/examples/test_template/jobs/test_template-43144464.mgr-04.i/output/file_1.txt b/examples/test_template/jobs/test_template-43144464.mgr-04.i/output/file_1.txt new file mode 100644 index 0000000..e427984 --- /dev/null +++ b/examples/test_template/jobs/test_template-43144464.mgr-04.i/output/file_1.txt @@ -0,0 +1 @@ +HELLO diff --git a/examples/test_template/jobs/test_template-43144464.mgr-04.i/output/file_2.txt b/examples/test_template/jobs/test_template-43144464.mgr-04.i/output/file_2.txt new file mode 100644 index 0000000..e427984 --- /dev/null +++ b/examples/test_template/jobs/test_template-43144464.mgr-04.i/output/file_2.txt @@ -0,0 +1 @@ +HELLO diff --git a/examples/test_template/jobs/test_template-43144464.mgr-04.i/output/file_3.txt b/examples/test_template/jobs/test_template-43144464.mgr-04.i/output/file_3.txt new file mode 100644 index 0000000..e427984 --- /dev/null +++ b/examples/test_template/jobs/test_template-43144464.mgr-04.i/output/file_3.txt @@ -0,0 +1 @@ +HELLO diff --git a/examples/test_template/jobs/test_template-43144464.mgr-04.i/output/file_4.txt b/examples/test_template/jobs/test_template-43144464.mgr-04.i/output/file_4.txt new file mode 100644 index 0000000..e427984 --- /dev/null +++ b/examples/test_template/jobs/test_template-43144464.mgr-04.i/output/file_4.txt @@ -0,0 +1 @@ +HELLO diff --git a/examples/test_template/jobs/test_template-43144464.mgr-04.i/output/file_5.txt b/examples/test_template/jobs/test_template-43144464.mgr-04.i/output/file_5.txt new file mode 100644 index 0000000..e427984 --- /dev/null +++ b/examples/test_template/jobs/test_template-43144464.mgr-04.i/output/file_5.txt @@ -0,0 +1 @@ +HELLO diff --git a/examples/test_template/jobs/test_template-43144464.mgr-04.i/output/file_6.txt b/examples/test_template/jobs/test_template-43144464.mgr-04.i/output/file_6.txt new file mode 100644 index 0000000..e427984 --- /dev/null +++ b/examples/test_template/jobs/test_template-43144464.mgr-04.i/output/file_6.txt @@ -0,0 +1 @@ +HELLO diff --git a/examples/test_template/jobs/test_template-43144464.mgr-04.i/output/file_7.txt b/examples/test_template/jobs/test_template-43144464.mgr-04.i/output/file_7.txt new file mode 100644 index 0000000..e427984 --- /dev/null +++ b/examples/test_template/jobs/test_template-43144464.mgr-04.i/output/file_7.txt @@ -0,0 +1 @@ +HELLO diff --git a/examples/test_template/jobs/test_template-43144464.mgr-04.i/output/file_8.txt b/examples/test_template/jobs/test_template-43144464.mgr-04.i/output/file_8.txt new file mode 100644 index 0000000..e427984 --- /dev/null +++ b/examples/test_template/jobs/test_template-43144464.mgr-04.i/output/file_8.txt @@ -0,0 +1 @@ +HELLO diff --git a/examples/test_template/jobs/test_template-43144464.mgr-04.i/output/test_input.txt b/examples/test_template/jobs/test_template-43144464.mgr-04.i/output/test_input.txt new file mode 100644 index 0000000..fae8ec6 --- /dev/null +++ b/examples/test_template/jobs/test_template-43144464.mgr-04.i/output/test_input.txt @@ -0,0 +1 @@ +This is the test input file diff --git a/examples/test_template/pbs.err/.keep b/examples/test_template/pbs.err/.keep new file mode 100644 index 0000000..e69de29 diff --git a/examples/test_template/pbs.err/test_template.e43144464 b/examples/test_template/pbs.err/test_template.e43144464 new file mode 100644 index 0000000..e69de29 diff --git a/examples/test_template/pbs.out/.keep b/examples/test_template/pbs.out/.keep new file mode 100644 index 0000000..e69de29 diff --git a/examples/test_template/pbs.out/test_template.o43144464 b/examples/test_template/pbs.out/test_template.o43144464 new file mode 100644 index 0000000..9f97006 --- /dev/null +++ b/examples/test_template/pbs.out/test_template.o43144464 @@ -0,0 +1 @@ +HELLO beginning the script body diff --git a/examples/test_template/submit.org b/examples/test_template/submit.org new file mode 100644 index 0000000..1bff85d --- /dev/null +++ b/examples/test_template/submit.org @@ -0,0 +1,454 @@ +* Fill in these fields here: +** task name + +When running PBS scripts you might want to run something multiple +times or have to deal with debugging which can become messy. +We organize by "task", which is a unit of something we want to get +done, and then organize all the outputs it produces in a sane way. + +The task name must (and should) also be the name of the directory the +task is set up in. + +#+name: task-name +#+BEGIN_SRC bash + test_template +#+END_SRC + +** task dir + +Task directories should be organized in some way. This variable is the +path preceeding your task dir. + +For instance if you working in `$HOME/this_project/task1` then your +task dir would be `$HOME/this_project`. + +#+name: task-dir-path +#+BEGIN_SRC bash + $MY_DICKSON_LAB/pbs_templates/examples +#+END_SRC + + +** wall time + +#+name: walltime +#+BEGIN_SRC bash + 5:00 +#+END_SRC + +** memory +#+name: memory +#+BEGIN_SRC bash + 4gb +#+END_SRC + +** number of nodes +#+name: num-nodes +#+BEGIN_SRC bash + 1 +#+END_SRC + +** number of processors +#+name: num-processors +#+BEGIN_SRC bash + 1 +#+END_SRC + +** number of GPUs + +#+name: num-gpus +#+BEGIN_SRC bash +#+END_SRC + +*** examples + +#+BEGIN_SRC bash + #PBS -l gpus=0 +#+END_SRC + + + +** feature + +#+name: feature +#+BEGIN_SRC bash +#+END_SRC + +*** examples +#+BEGIN_SRC bash + #PBS -l feature='gpgpu:intel16' +#+END_SRC + + +** email +#+name: email +#+BEGIN_SRC bash + samuel.lotz@gmail.com +#+END_SRC + +** GNU module +#+name: gnu-module +#+BEGIN_SRC bash + GNU/4.8.3 +#+END_SRC + +** CUDA module +#+name: cuda-module +#+BEGIN_SRC bash + CUDA/7.0 +#+END_SRC + +** script parameters +#+name: script-parameters +#+BEGIN_SRC bash + TEST_PARAM="HELLO" + TEST_VAL=8 + TEST_FILE="test_input.txt" +#+END_SRC + +** script +#+name: script +#+BEGIN_SRC bash + echo "HELLO beginning the script body" + cat $TEST_FILE 1>> $LOG 2>> $LOG + for ((x=0; x<=${TEST_VAL}; x++)); do + echo ${TEST_PARAM} > "file_${x}.txt" 2>> $LOG; + done +#+END_SRC + + +* Additional Tweaks + +These are things that change less often but are particular to a +particular place and user. + +** Environmental Variables + +#+name: env-vars +#+BEGIN_SRC bash + # scratch directory + export SCRATCHDIR=/mnt/ls15/scratch + # your home scratch + export SCRATCH_HOME=$SCRATCH + + # the Dickson lab directory + export DICKSON_LAB=/mnt/research/DicksonLab + # your Dickson lab dir + export MY_DICKSON_LAB=$DICKSON_LAB/$USER + # locally installed programs + export DICKSON_LAB_PROGS=${DICKSON_LAB}/programs + # the installed binaries for Dickson lab + export DICKSON_LAB_BIN=${DICKSON_LAB_PROGS}/bin + # directory with installed libraries + export DICKSON_LAB_LIB=${DICKSON_LAB_PROGS}/lib + # Dickson lab scratch directory + export DICKSON_SCRATCH=$SCRATCHDIR/groups/DicksonLab + # your Dickson lab scratch dir + export MY_DICKSON_SCRATCH=$DICKSON_SCRATCH/$USER + + # set environmental variables to programs needed + #---------------------------------------- + # charmm executables + export CHARMM_DIR=${DICKSON_LAB_PROGS}/charmm + export CHARMM=${CHARMM_DIR}/exec/gnu/charmm + export MPICHARMM=${CHARMM_DIR}/exec/gnu_M/charmm + export OPENMMCHARMM=${CHARMM_DIR}/exec/gnu/charmm_OpenMM6.3 + + # charmm data + export CHARMMDATA=${DICKSON_LAB}/toppar + + # set the path for mmtsb + export MMTSBDIR=$DICKSON_LAB_PROGS/mmtsb + + # OpenMM + export OPENMM_ROOT_DIR=/mnt/research/DicksonLab/programs/OpenMM6.3 + export OPENMM_PLUGIN_DIR=$OPENMM_ROOT_DIR/lib/plugins + export OPENMM_PLATFORM=CUDA + unset OPENMM_DEVICE + + + # Anaconda Python paths + export ANACONDA_DIR=$DICKSON_LAB_PROGS/anaconda2 + export ANACONDA_BIN=$ANACONDA_DIR/bin + export PYTHONHOME=$ANACONDA_DIR + + # Library paths + export LD_LIBRARY_PATH=${DICKSON_LAB_LIB}:${CUDADIR}/lib:${OPENMM_ROOT_DIR}/lib:${OPENMM_PLUGIN_DIR}:${LD_LIBRARY_PATH} + + # PATH + export PATH=$ANACONDA_BIN:${DICKSON_LAB_BIN}:${OPENMM_ROOT_DIR}/bin:${MMTSBDIR}/perl:${DICKSON_LAB_PROGS}/charmm/exec/gnu:${PATH} + +#+END_SRC + + + +* Noweb Template + +When you export/tangle this script the named code blocks will be +inserted into their targets marked by `<>` for a target +called `my-target`. + +#+BEGIN_SRC bash :tangle submit.pbs :noweb yes :shebang #!/bin/sh -login + #PBS -N <> + #PBS -l walltime=<> + #PBS -l mem=<> + #PBS -l nodes=<>:ppn=<> + <> + <> + #PBS -e pbs.err + #PBS -o pbs.out + #PBS -M <> + #PBS -m abe + + # ------------------------------ + # Script for running the task <> with PBS on hpcc + + # task name, this is the name of the directory for the task + TASK="<>" + # directory path for this task + TASKDIR="<>/$TASK" + + # the directory with all of the input + INPUTDIR=$TASKDIR/input + # directory to put job results in + JOBSDIR=$TASKDIR/jobs + # we should redirect our own output streams but we will catch the PBS + # files as well + PBS_STDOUT=$PBS_O_WORKDIR/pbs.out + PBS_STDERR=$PBS_O_WORKDIR/pbs.err + + # the job name + JOBNAME=${TASK}-${PBS_JOBID} + # the directory for the output of this job + JOBDIR=$JOBSDIR/$JOBNAME + # the directory output will be put into after completion + OUTPUTDIR=$JOBDIR/output + # log files + LOG=$JOBDIR/log + # directories to copy the stdout and stderr from scripts + STDOUT=$JOBDIR/pbs.out + STDERR=$JOBDIR/pbs.err + + # directory to execute code in + # EXECDIR=$TMPDIR + # EXECDIR=$MY_DICKSON_SCRATCH/$JOBNAME + # Problems with Scratch so I will just run in the jobs dir + # TODO make an if for different options + EXECDIR=$JOBDIR/exec + + # make these directories if they do not exist + mkdir -p $JOBDIR + mkdir -p $EXECDIR + mkdir -p $PBS_STDOUT + mkdir -p $PBS_STDERR + + # Starting the LOG file + echo "Starting Log file" 1> $LOG 2> $LOG + echo "----------------------------------------" 1>> $LOG 2>> $LOG + echo "" 1>> $LOG 2>> $LOG + + # write out explicitly the run parameters + echo "----------------------------------------" 1>> $LOG 2>> $LOG + echo "PBS Resource requests" 1>> $LOG 2>> $LOG + echo "----------------------------------------" 1>> $LOG 2>> $LOG + + echo "" 1>> $LOG 2>> $LOG + echo "walltime <>" 1>> $LOG 2>> $LOG + echo "memory <>" 1>> $LOG 2>> $LOG + echo "num-nodes <>" 1>> $LOG 2>> $LOG + echo "num-processors <>" 1>> $LOG 2>> $LOG + echo "num-gpus <>" 1>> $LOG 2>> $LOG + echo "feature <>" 1>> $LOG 2>> $LOG + echo "" 1>> $LOG 2>> $LOG + + # print the paths for all the variables constructed in this script + echo "----------------------------------------" 1>> $LOG 2>> $LOG + echo "Check Paths" 1>> $LOG 2>> $LOG + echo "----------------------------------------" 1>> $LOG 2>> $LOG + + echo "" 1>> $LOG 2>> $LOG + echo "LOG $LOG" 1>> $LOG 2>> $LOG + echo "PBS_JOBID $PBS_JOBID" 1>> $LOG 2>> $LOG + echo "TASK $TASK" 1>> $LOG 2>> $LOG + echo "TASKDIR $TASKDIR" 1>> $LOG 2>> $LOG + echo "INPUTDIR $INPUTDIR" 1>> $LOG 2>> $LOG + echo "JOBSDIR $JOBSDIR" 1>> $LOG 2>> $LOG + echo "JOBNAME $JOBNAME" 1>> $LOG 2>> $LOG + echo "JOBDIR $JOBDIR" 1>> $LOG 2>> $LOG + echo "OUTPUTDIR $OUTPUTDIR" 1>> $LOG 2>> $LOG + echo "EXECDIR $EXECDIR" 1>> $LOG 2>> $LOG + echo "STDOUT $STDOUT" 1>> $LOG 2>> $LOG + echo "STDERR $STDERR" 1>> $LOG 2>> $LOG + echo "PBS_STDOUT $PBS_STDOUT" 1>> $LOG 2>> $LOG + echo "PBS_STDERR $PBS_STDERR" 1>> $LOG 2>> $LOG + echo "PBS_NODEFILE" 1>> $LOG 2>> $LOG + cat $PBS_NODEFILE 1>> $LOG 2>> $LOG + echo "" 1>> $LOG 2>> $LOG + + # saving PBS environmental variables + echo "----------------------------------------" 1>> $LOG 2>> $LOG + echo "PBS Environmental Variables" 1>> $LOG 2>> $LOG + echo "----------------------------------------" 1>> $LOG 2>> $LOG + + echo "" 1>> $LOG 2>> $LOG + # PBS env vars + echo "PBS ENV:" 1>> $LOG 2>> $LOG + env | grep PBS 1>> $LOG 2>> $LOG + + # PBS JOBID number + echo "PBS_JOBID number" 1>> $LOG 2>> $LOG + PBS_JOBID_NUM=(${PBS_JOBID//./ }) + echo "PBS_JOBID_NUM: $PBS_JOBID_NUM" 1>> $LOG 2>> $LOG + + # PBS stdout and stderr file paths + PBS_JOB_STDOUT="$PBS_STDOUT/$TASK.o$PBS_JOBID_NUM" + PBS_JOB_STDERR="$PBS_STDERR/$TASK.e$PBS_JOBID_NUM" + + # NODEFILE + echo "PBS_NODEFILE:" 1>> $LOG 2>> $LOG + cat $PBS_NODEFILE 1>> $LOG 2>> $LOG + # GPUFILE + echo "PBS_GPUFILE:" 1>> $LOG 2>> $LOG + cat $PBS_GPUFILE 1>> $LOG 2>> $LOG + # MICFILE + echo "PBS_MICFILE:" 1>> $LOG 2>> $LOG + cat $PBS_MICFILE 1>> $LOG 2>> $LOG + echo "" 1>> $LOG 2>> $LOG + + # initial + echo "" 1>> $LOG 2>> $LOG + echo "----------------------------------------" 1>> $LOG 2>> $LOG + echo "Initialization" 1>> $LOG 2>> $LOG + echo "----------------------------------------" 1>> $LOG 2>> $LOG + + # load profile + echo "------------" 1>> $LOG 2>> $LOG + echo "RUNNING: source /etc/profile" 1>> $LOG 2>> $LOG + echo "------------" 1>> $LOG 2>> $LOG + source /etc/profile 1>> $LOG 2>> $LOG + echo "" 1>> $LOG 2>> $LOG + + # load hpcc modules + echo "------------" 1>> $LOG 2>> $LOG + echo "RUNNING: source /opt/software/modulefiles/setup_modules.sh" 1>> $LOG 2>> $LOG + echo "------------" 1>> $LOG 2>> $LOG + source /opt/software/modulefiles/setup_modules.sh 1>> $LOG 2>> $LOG + echo "" 1>> $LOG 2>> $LOG + + # load specific library modules + # GNU Compilers + echo "------------" 1>> $LOG 2>> $LOG + echo "RUNNING: module load <>" 1>> $LOG 2>> $LOG + echo "------------" 1>> $LOG 2>> $LOG + module load <> 1>> $LOG 2>> $LOG + echo "" 1>> $LOG 2>> $LOG + + # CUDA compilers + echo "------------" 1>> $LOG 2>> $LOG + echo "RUNNING: module load <>" 1>> $LOG 2>> $LOG + echo "------------" 1>> $LOG 2>> $LOG + module load <> 1>> $LOG 2>> $LOG + echo "" 1>> $LOG 2>> $LOG + + + # set environmental variables and other local variables that are used for + # many types of scripts + # =============================================================================== + echo "------------" 1>> $LOG 2>> $LOG + echo "Setting environmental variables" 1>> $LOG 2>> $LOG + echo "------------" 1>> $LOG 2>> $LOG + + <> + # =============================================================================== + + # change to the exec dir + echo "------------" 1>> $LOG 2>> $LOG + echo "moving to EXECDIR: $EXECDIR" 1>> $LOG 2>> $LOG + echo "------------" 1>> $LOG 2>> $LOG + cd $EXECDIR 1>> $LOG 2>> $LOG + echo "" 1>> $LOG 2>> $LOG + + # remove current contents of the execdir, useful for if running + # interactive job which writes to same dir, harmless if not + echo "------------" 1>> $LOG 2>> $LOG + echo "Removing existing files if they exist in EXECDIR: $EXECDIR" 1>> $LOG 2>> $LOG + echo "------------" 1>> $LOG 2>> $LOG + rm -rf $EXECDIR/* 1>> $LOG 2>> $LOG + echo "" 1>> $LOG 2>> $LOG + + # copy the input files to the execution directory + echo "------------" 1>> $LOG 2>> $LOG + echo "Copying input files from INPUTDIR: $INPUTDIR to EXECDIR: $EXECDIR" 1>> $LOG 2>> $LOG + echo "------------" 1>> $LOG 2>> $LOG + cp -rf $INPUTDIR/* $EXECDIR/ 1>> $LOG 2>> $LOG + echo "" 1>> $LOG 2>> $LOG + + # copy the actual submission script used + echo "------------" 1>> $LOG 2>> $LOG + echo "Copying submission script ${TASKDIR}/${PBS_JOBNAME} to EXECDIR: $EXECDIR" 1>> $LOG 2>> $LOG + echo "------------" 1>> $LOG 2>> $LOG + cp "${0}" $EXECDIR/ 1>> $LOG 2>> $LOG + echo "" 1>> $LOG 2>> $LOG + + # write file names in $EXECDIR to log + echo "------------" 1>> $LOG 2>> $LOG + echo "listing of EXECDIR: $EXECDIR" 1>> $LOG 2>> $LOG + echo "------------" 1>> $LOG 2>> $LOG + ls $EXECDIR 1>> $LOG 2>> $LOG + echo "" 1>> $LOG 2>> $LOG + + # print out the environmental variables after modifications + echo "------------" 1>> $LOG 2>> $LOG + echo "Environmental variables before execution:" 1>> $LOG 2>> $LOG + echo "------------" 1>> $LOG 2>> $LOG + env 1>> $LOG 2>> $LOG + echo "" 1>> $LOG 2>> $LOG + + # ------------------------------ + # set the parameters that will be used in this script + # =============================================================================== + echo "------------" 1>> $LOG 2>> $LOG + echo "Setting Script parameters" 1>> $LOG 2>> $LOG + echo "------------" 1>> $LOG 2>> $LOG + echo "" 1>> $LOG 2>> $LOG + + <> + + echo "" 1>> $LOG 2>> $LOG + # =============================================================================== + + + # ------------------------------ + # The code for this script + # =============================================================================== + echo "------------" 1>> $LOG 2>> $LOG + echo "Running script" 1>> $LOG 2>> $LOG + echo "===============================================================================" 1>> $LOG 2>> $LOG + + <