preprocessworker.sh

#!/bin/bash

script_name=`basename $0`
script_dir=`dirname $0`

source "${script_dir}/commonfunctions.sh"

version="???"
maxpackages="5"
waitinterval="60"

if [ -f "$script_dir/VERSION" ] ; then
    version=`cat $script_dir/VERSION`
fi

function usage()
{
    echo "usage: $script_name [-h]
                      predictdir

              Version: $version

              Runs PreprocessPackage.m in a serial fashion 
              waiting if too many packages have yet to be 
              processed. This script uses predict.config 
              file to obtain location of trained model 
              and image data

positional arguments:
  predictdir           Predict directory generated by
                       runprediction.sh

optional arguments:
  -h, --help           show this help message and exit
  --maxpackages        Number of completed packages allowed before
                       script should wait to let prediction catch up.
                       (default $maxpackages)

  --waitinterval       Number of seconds to wait between checking
                       for number of completed packages 
                       (default $waitinterval)

    " 1>&2;
    exit 1;
}

TEMP=`getopt -o h --long "help,maxpackages:,waitinterval:" -n '$0' -- "$@"`
eval set -- "$TEMP"

while true ; do
    case "$1" in
        -h ) usage ;;
        --help ) usage ;;
        --waitinterval ) waitinterval=$2 ; shift 2 ;;
        --maxpackages ) maxpackages=$2 ; shift 2 ;;
        --) shift ; break ;;
    esac
done

if [ $# -ne 1 ] ; then
    usage
fi

out_dir=$1

echo ""

predict_config="$out_dir/predict.config"

parse_predict_config "$predict_config"

if [ $? != 0 ] ; then
    fatal_error "$out_dir" "ERROR parsing $predict_config" 2
fi

echo "Running PreprocessPackage"
echo ""

echo "Trained Model Dir: $trained_model_dir"
echo "Image Dir: $img_dir"
echo "Models: $model_list"
echo "Speed: $aug_speed"
echo ""

package_proc_info="$out_dir/augimages/package_processing_info.txt"

if [ ! -s "$package_proc_info" ] ; then
    fatal_error "$out_dir" "ERROR $package_proc_info not found" 7
fi

parse_package_processing_info "$package_proc_info"

space_sep_models=$(get_models_as_space_separated_list "$model_list")

for model_name in `echo $space_sep_models` ; do
  
    let cntr=1
    for CUR_PKG in `seq -w 001 $num_pkgs` ; do
        for CUR_Z in `seq -w 01 $num_zstacks` ; do
            package_name=$(get_package_name "$CUR_PKG" "$CUR_Z")
            Z="$out_dir/augimages/$model_name/$package_name"
            out_pkg="$out_dir/$model_name/$package_name"
            if [ -f "$out_pkg/DONE" ] ; then
                echo "  Found $out_pkg/DONE. Prediction completed. Skipping..."
                continue
            fi
            echo "Preprocessing $package_name in model $model_name"
            augoutfile="$out_dir/augimages/preproc.${model_name}.${package_name}.log"
      
            /usr/bin/time -p PreprocessPackage.m "$img_dir" "$out_dir/augimages" $CUR_PKG $CUR_Z $model_name $aug_speed > "$augoutfile" 2>&1 
            ecode=$?
            if [ $ecode != 0 ] ; then
                fatal_error "$out_dir" "ERROR, a non-zero exit code ($ecode) received from PreprocessPackage.m $CUR_PKG $CUR_Z $model_name $aug_speed" 8
            fi
            echo "Waiting for prediction to catch up"
            res=$(wait_for_prediction_to_catchup "$out_dir/augimages" $maxpackages $waitinterval)
            if [ "$res" == "killed" ] ; then
                echo "KILL.REQUEST file found. Exiting"
                exit 1
            fi
            let cntr+=1
        done
    done
done

echo ""
echo "PreprocessPackaging has completed."
echo ""