subgms

#!/bin/csh

# GAMESS 20180214 R1 Interactive Job Submission on TAIWANIA Cluster, NCHC, Taiwan
#
# Updated 20180706  Rangsiman Ketkaew  rangsiman1993@gmail.com
# https://github.com/rangsimanketkaew/QM-on-TAIWANIA/

########################### BOX A ############################
set GAMESS_TOP = "$HOME/gamess-OpenMP"
set GAMESS_VER = "00"
##############################################################

######### DO NOT MODIFY ANY COMMANDS BELOW THIS LINE #########
set GAMESS_EXE = "gamess.${GAMESS_VER}.x"
set GAMESS_RUN = "rungms"
set RUNGMS_MOD = "rungms.mod"
##############################################################
set PROJ_ID_FILE = "$HOME/.proj_id"
##############################################################

# Check if help is requested.
if ( "$1" == "-help" || "$1" == "--help" || "$1" == "-h" || "$1" == "help" ) goto help

echo ""

# Check GAMESS top dir.
if ( ! -e $GAMESS_TOP ) then
  echo "Error: unable to locate GAMESS top directory. Please check" '$GAMESS_TOP' "again."
  echo ""
  exit 1
endif

# Check GAMESS_* env vir.
if ( ! -f $GAMESS_TOP/$GAMESS_EXE ) then
  echo "Error: $GAMESS_EXE executable not found in $GAMESS_TOP, please check again."
  echo ""
  exit 1
endif

if ( -f $HOME/$RUNGMS_MOD ) then
  set GAMESS_RUNGMS_MOD = "$HOME/$RUNGMS_MOD"
  goto check_rungmsmod
endif

if ( -f $GAMESS_TOP/$RUNGMS_MOD ) then
  set GAMESS_RUNGMS_MOD = "$GAMESS_TOP/$RUNGMS_MOD"
  goto check_rungmsmod
endif

echo "Error: rungms.mod file not found. rungms.mod should be in either" '$GAMESS_TOP' "or in" '$HOME' "directories."
echo ""
echo "       where " '$GAMESS_TOP' " is $GAMESS_TOP"
echo "             " '$HOME      ' " is $HOME"
echo ""
echo "subgms script executes GAMESS calculation via rungms.mod, not rungms."
echo "Type 'subgms -help' for instruction of preparation of rungms.mod file."
echo ""
exit 1

check_rungmsmod:

set CHK_ERR = 0

if ( `grep -wc 'TEMPLATE_1' $GAMESS_RUNGMS_MOD` == 0 ) then
  echo "Error: set SCR=TEMPLATE_1 command line not found in rungms.mod file."
  echo ""
  set CHK_ERR = 1
endif

if ( `grep -wc 'TEMPLATE_2' $GAMESS_RUNGMS_MOD` == 0 ) then
  echo "Error: set USERSCR=TEMPLATE_2 command line not found in rungms.mod file."
  echo ""
  set CHK_ERR = 1
endif

if ( `grep -wc 'TEMPLATE_3' $GAMESS_RUNGMS_MOD` == 0 ) then
  echo "Error: set GMSPATH=TEMPLATE_3 command line not found in rungms.mod file."
  echo ""
  set CHK_ERR = 1
endif

if ( $CHK_ERR == 1 ) exit 1

onintr inter

if ($#argv == 0) then
  echo "   GAMESS 20180214 R1 Interactive Job Submission on TAIWANIA, NCHC, Taiwan"
  echo "   -----------------------------------------------------------------------"
  echo ""
  echo "   Usage: subgms input[.inp] [output[.out]] [-help]"
  echo ""
  echo "   Example: subgms water_hf.inp"
  echo "            subgms water_hf.inp water_hf_16cores.out"
  echo ""
  exit 0
endif

ask_input:
set INPUTFILE   = "$1"
set USERINP = `basename $INPUTFILE .inp`
set INPUTNAME = "$USERINP".inp
set REALPATH = `realpath $INPUTNAME`
set DIR_PATH = `dirname $REALPATH`
set INPUTFILE = "$DIR_PATH/$USERINP".inp
if (! -f $INPUTFILE) then
  echo "Error: Unable to locate $INPUTFILE"
  echo ""
  exit 1
endif

if ("null$2" == "null") then
  set OUTPUTNAME = "$USERINP".out
  set OUTPUTFILE = "$DIR_PATH/$USERINP".out
else
  set OUTNAMEGET = `basename $2 .out`
  set OUTPUTNAME = "$OUTNAMEGET".out
  set OUTPUTFILE = "$DIR_PATH/$OUTNAMEGET".out
endif

ask_cpus:
set SERIAL = 0
set PARALLEL = 0

echo -n "Enter number of OMP threads [1]: "
set NUMPAL = "$<"

if ( $NUMPAL == "" || $NUMPAL == "1" ) then
  set CPUS = 1
  set JOBQUEUE = serial
  set SERIAL = 1
  goto set_resource
endif

if ( `echo $NUMPAL | grep -c '[a-z][A-Z]'` != 1 ) then
  set CPUS = "$NUMPAL"
  if ( $CPUS >= 2 && $CPUS <= 40 ) then
    set JOBQUEUE = cf40
    set PARALLEL = 1
    goto ask_queue
  else if ( $CPUS > 40 ) then
    echo "Warning: Number of MPI processes is over 40. Hence, it will be set to 40."
    set CPUS = 40
    set JOBQUEUE = cf40
    set PARALLEL = 1
    goto ask_queue
  endif
else
  echo "Error: Number of OMP threads must be 1 - 40."
  goto ask_cpus
endif

ask_queue:
if ( $PARALLEL == 1 ) then
  echo -n "Enter optimal queue [cf40]: "
  set QUEUE = "$<"
endif

if ( $QUEUE == "" || $QUEUE == "cf40" ) then
  set JOBQUEUE = cf40
else if ( $QUEUE == "ctest" ) then
  set JOBQUEUE = ctest
  goto set_resource
else
  echo "Error: Enter queue again: cf40 or ctest."
  goto ask_queue
endif

set_resource:
set NODES = 1
set THREADS = "$CPUS"
set MPIPROCS = 1
set JOBNAME = `basename $OUTPUTNAME .out`

set LISTID = ( `get_su_balance | awk -F, '{print $2}' | xargs` )

if ( "$LISTID" == "" ) then
  echo "Error: No available Project ID."
  exit 1
else if ( $#LISTID == 1 ) then
  set PROJ_ID = "$LISTID[1]"
  goto jobinfo
endif

if ( -f $PROJ_ID_FILE ) then
  set PROJID_1 = `head -1 $PROJ_ID_FILE`
  if ( `get_su_balance | grep -wc "$PROJID_1"` == 0 ) then
    echo "Error: Project ID specified in first line of $PROJ_ID_FILE is not correct."
    exit 1
  endif
  set PROJ_ID = "$PROJID_1"
  goto jobinfo
endif

set BALANCE = ( `get_su_balance | awk -F, '{print $1}' | xargs` )
echo ""
echo " Your available Project ID & SU Balance:"
@ i = 1
while ( $i <= $#LISTID )
echo " [$i]  $LISTID[$i]  $BALANCE[$i]"
set PROJ_ID = "$LISTID[1]"
@ i++
end
echo ""

ask_id_choice:
echo -n "Enter Project ID [1]: "
set PROJINP = "$<"

if ( "null$PROJINP" == "null" ) then
  set PROJ_ID = "$LISTID[1]"
  goto jobinfo
endif

if ( `echo $PROJINP | grep -c '[a-z][A-Z]*'` == 1 ) then
  echo "Error: Please assign choice as positive integer."
  goto ask_id_choice
endif

if ( $PROJINP > $#LISTID || $PROJINP <= 0 ) then
  echo "Error: Choice you selected is out of range."
  goto ask_id_choice
else
  set PROJ_ID = "$LISTID[$PROJINP]"
endif

jobinfo:

#####################################################
# Show all info before submitting job
#####################################################

echo ""
echo "      ----- Job Info -----"
echo ""
echo "    Input file  :  $INPUTFILE"
echo "   Output file  :  $OUTPUTFILE"
echo "  Compute node  :  $NODES"
echo "     CPU cores  :  $CPUS"
echo " MPI processes  :  $MPIPROCS"
echo "   OMP Threads  :  $THREADS"
echo "      Job Name  :  $JOBNAME"
echo "     Job Queue  :  $JOBQUEUE"   
echo "     Project ID :  $PROJ_ID"
echo ""

echo -n "Submit your job now ? [yes]: "
set SUBMIT = "$<"
if ("null$SUBMIT" == "null" || "null$SUBMIT" == "nully" || "null$SUBMIT" == "nullyes") then
  goto submit
else
  echo "...Quit..."
  exit 0
endif

submit:
set PBS_SCRIPT = "$DIR_PATH/submit.GAMESS.`basename $OUTPUTNAME .out`.sh"

cat <<EOF > $PBS_SCRIPT
#!/bin/bash 

#PBS -l select=${NODES}:ncpus=${CPUS}:ompthreads=${THREADS}
#PBS -q $JOBQUEUE
#PBS -N $JOBNAME
#PBS -P $PROJ_ID

#############################################################
#### This PBS Pro script was generated by subgms program ####
#############################################################

module purge
module load intel/2018_u1 gcc/6.3.0

cd \$PBS_O_WORKDIR

export GAMESS_TOP="$GAMESS_TOP"
export GAMESS_VER="$GAMESS_VER"
export GAMESS_EXE="$GAMESS_EXE"
export GAMESS_RUN="$GAMESS_RUN"
export RUNGMS_MOD="$RUNGMS_MOD"
export GAMESS_SCR_DIR="/work1/$USER/SCRATCH/gamess/pbs.\${PBS_JOBID/\.srvc1/}"
if [ ! -d \$GAMESS_SCR_DIR ]; then mkdir -p "\$GAMESS_SCRATCH_DIR" ; fi
mkdir -p "\$GAMESS_SCR_DIR/USER"

cp -r \$GAMESS_TOP/\$RUNGMS_MOD \$GAMESS_SCR_DIR/USER/\$GAMESS_RUN
sed -i 's/TEMPLATE_1/\\\$GAMESS_SCR_DIR/' \$GAMESS_SCR_DIR/USER/\$GAMESS_RUN
sed -i 's/TEMPLATE_2/\\\$GAMESS_SCR_DIR\/USER/' \$GAMESS_SCR_DIR/USER/\$GAMESS_RUN
sed -i 's/TEMPLATE_3/\\\$GAMESS_TOP/' \$GAMESS_SCR_DIR/USER/\$GAMESS_RUN

EOF

if ( $SERIAL == 1 ) then
cat <<EOF >> $PBS_SCRIPT
\$GAMESS_SCR_DIR/USER/\$GAMESS_RUN $INPUTNAME >& $OUTPUTFILE

EOF
endif

if ( $PARALLEL == 1 ) then
cat <<EOF >> $PBS_SCRIPT
\$GAMESS_SCR_DIR/USER/\$GAMESS_RUN $INPUTNAME $GAMESS_VER $CPUS >& $OUTPUTFILE

EOF
endif

qsub $PBS_SCRIPT
#echo "Your job has been submitted."
exit 0

help:
clear
cat << EOF | less

                                        ---------------------------------
                                        GAMESS Interactive Job Submission
                                        ---------------------------------

======
subgms
======

subgms is used to submit share-memory GAMESS calculation on a single node.

This version of subgms supports only shared memory parallel (SMP) system using OpenMP protocol.

TAIWANIA Policy: Max OpenMP threads per node (OMP_NUM_THREADS) is 40.


==========
rungms.mod
==========

GAMESS program normally uses 'rungms' script to execute the calculation in both serial and parallel methods.
Original rungms provided by developer specify the locatation of SCRATCH and USER's SCRATCH as a static path.
This setting is against to job scheduler when submitting the job and writing scratch files in different location.
Therefore, rungms must be modified in order to specify the \$SCR and \$USERSCR as dynamics path. 

The following is step-by-step preparation of 'rungms.mod' file, which will be used instead of original rungms.

Step 1: Create rungms.mod
-------------------------

   rungms.mod can be copied from an original rungms. rungms.mod must be saved in either GAMESS or user's home directory. 
   For example, following command is copy of an original rungms file to rungms.mod in user's home directory.
  
   \$ cp \$GAMESS_TOP/rungms \$HOME/rungms.mod
  
   where \$GAMESS_TOP is set to $GAMESS_TOP
         \$HOME       is set to $HOME
  
   Note that \$GAMESS_TOP setting can be found in 'BOX A' around the beginning lines of subgms source code.


Step 2: Modify rungms.mod
-------------------------

   Modify rungms.mod source code around the beginning line of file as following
  
   Original rungms             -->         Modified rungms.mod
   ---------------                         --------------------
   set SCR=/scr/\$USER                      set SCR=TEMPLATE_1
   set USERSCR=/u1/\$USER/scr               set USERSCR=TEMPLATE_2
   set GMSPATH=/u1/mike/gamess             set GMSPATH=TEMPLATE_3
  
   where \$SCR     is Scratch directory.
         \$USERSCR is Specific local user's scratch directory.
         \$GMSPATH is Top directory of GAMESS program, which also contains program executable.

Step 3: Check if rungms.mod exists
----------------------------------

   Execute subgms program to check if subgms can locate rungms.mod in working directory.


======
AUTHOR
======

Rangsiman Ketkaew (MSc student)
E-mail: rangsiman1993@gmail.com
Computational Chemistry Research Unit
Department of Chemistry
Faculty of Science and Technology
Thammasat University, Thailand

EOF

exit 0

inter: 
  echo ""
  echo "Error: you pressed Ctrl+C ...Quit..."
  echo ""
  exit 1