Merge pull request #117 from arulalant/master

Support for EPS Average VSDB Input

bsubScripts/ncum_global_solar_energy_input/solar_energy_create_tarball_g2files_put_into_ftp.py

@@ -42,7 +42,7 @@ def createTarBalls(path, today, utc, stephr=3):
     # Lets create tar ball for every day individually.
     for i, hr in enumerate(range(0, 241, 24)):
         # create forecast files tar file in parallel # -m500 need to be include for pbzip2
-        cmd = "tar -c ./fcst_{%s..%d}h%s*.grb2 | %s  -v  -c -f -p32 > %s/fcst_day%s_solar_ind_0.25_%s.tar.gz" % (str(hr+1).zfill(2), hr+24, today, pigz, '../TarFiles', str(hr).zfill(2), today)
+        cmd = "tar -c ./fcst_{%s..%d}h%s*.grb2 | %s  -v  -c -f -p32 > %s/fcst_day%s_solar_ind_0.25_%s.tar.gz" % (str(hr+1).zfill(2), hr+24, today, pigz, '../TarFiles', str(i+1).zfill(2), today)
         print cmd
         subprocess.call(cmd, shell=True)
     # end of for i, hr in enumerate(range(0, 241, 24)):

bsubScripts/ncum_regional_wind_energy_input/OthersScripts/make_winddata_csv_zip_put_into_ftp.sh

@@ -3,21 +3,16 @@ set -x
 
 date=$1
 cyc=$2
-
 eleventhDay=$3
 
-
-OUTDIR=/gpfs3/home/umfcst/ShortJobs/NCUMReg_WIND_ENERGY/0.04/CSVFiles/$date
+OUTDIR=/gpfs3/home/umfcst/ShortJobs/NCUMReg_WIND_ENERGY/0.04/CSVFiles
 mkdir -p $OUTDIR
 
-cp /gpfs3/home/umfcst/ShortJobs/NCUMReg_WIND_ENERGY/0.04/TarFiles/WindEnergy.India.0.04.$date$cyc.tar.gz  $OUTDIR/
 cp -r timeseries_reg.gs latlon.csv ${OUTDIR}/
 
 cd $OUTDIR
 
-rm -f *.nc
-rm -f *tar.gz
-rm -f wind*csv
+cp /gpfs3/home/umfcst/ShortJobs/NCUMReg_WIND_ENERGY/0.04/TarFiles/WindEnergy.India.0.04.$date$cyc.tar.gz  .
 
 tar -zxf WindEnergy.India.0.04.$date$cyc.tar.gz
 for tt in {01..72..01} ; do
@@ -31,6 +26,8 @@ OUTFILE=wind50reg${date}${cyc}.zip
 
 zip $OUTFILE wind50*csv
 rm -f wind50*csv
+rm -f *tar.gz
+rm -f *.nc
 rm timeseries_reg.gs
 
 ssh ncmlogin3 "scp -p $OUTDIR/$OUTFILE  prod@ftp:/data/energon/4km/"

bsubScripts/ncumeps_global_post/ncumeps_create_memavg_vsdb_input.py

@@ -0,0 +1,198 @@
+#!/usr/bin/env python
+
+## This is call back script which will be executed after created the grib2 files.
+## 
+## After created NCUM_EPS grib2 files (which contain all 45 members in it), lets
+## do calculate mean over all 45 members and then convert to 2.5X2.5 degree 
+## resolution, finally convert to grib1 format for vsdb input purpose.
+##
+## Arulalan.T
+## 25-July-2016.
+
+import os, subprocess, datetime, getopt, sys, iris, numpy, time 
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '../../g2utils')))
+from cubeutils import cubeRealizationAverager
+from um2grb2 import tweaked_messages, getCubeData, createDirWhileParallelRacing
+
+
+neededVars = [
+    ## Pressure Level Variable names & STASH codes
+    ('geopotential_height', 'm01s16i202'),      
+    ('x_wind', 'm01s15i243'),
+    ('y_wind', 'm01s15i244'), 
+    ('air_temperature', 'm01s16i203'),
+    ('relative_humidity', 'm01s16i256'),
+    ## Non Pressure Level Variable names & STASH codes
+    ('air_pressure_at_sea_level', 'm01s16i222'),
+    ]
+
+#Define _precipVars_
+# The following vars should contains only precipitation, rainfall, snow 
+# variables, those whose regrid extrapolate should be only in 'linear' mode
+# and not in 'mask' mode, and should not have -ve values.
+_precipVars_ = [('precipitation_amount', 'm01s05i226'),]
+
+g2ctl = "/gpfs2/home/umtid/Softwares/grib2ctl/g2ctl.pl"
+grib2ctl = "/gpfs2/home/umtid/Softwares/grib2ctl/grib2ctl.pl"
+gribmap = "/gpfs1/home/Libs/GNU/GRADS/grads-2.0.2.oga.1/Contents/gribmap"
+cnvgrib = "/gpfs1/home/Libs/INTEL/CNVGRIB/CNVGRIB-1.4.1/cnvgrib-1.4.1/cnvgrib"
+wgrib2 = "/gpfs1/home/Libs/GNU/WGRIB2/v2.0.1/wgrib2"
+
+_targetGridFile_ = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../data/sample_global_2p5X2p5_73X144.grib2'))
+_targetGrid_ = iris.load(_targetGridFile_)[0]
+
+__grib1FilesNameSuffix__ = None
+__wgrib2Arguments__ = ' -set_bin_prec 12 -set_grib_type complex2 -grib_out '
+__OVERWRITE__ = True
+
+def createENSavg_VSDB_Grib1Files(inpath, outpath, today, utc, start_long_fcst_hour, stephr=24):    
+
+
+    day = int(start_long_fcst_hour) / int(stephr)
+    pfileday = str(day).zfill(2)
+    pfilename = 'umeps_prg_1cntl_44ens_24hourly_day' + pfileday
+    needed_fname = pfilename + '_' + today + '_' + utc + 'Z.grib2'
+    files = [f for f in os.listdir(inpath) if f == needed_fname]
+    if not files: return 
+
+    tDay = datetime.datetime.strptime(today, "%Y%m%d")        
+    tvDay = tDay.strftime('%d%m%y')
+
+    opath = os.path.join(outpath, today)
+    createDirWhileParallelRacing(opath)        
+
+    g2filepath = os.path.join(opath, 'prg_' + today + pfileday + '.grib2')
+    if __OVERWRITE__ and os.path.isfile(g2filepath): os.remove(g2filepath)
+
+    wg2filename = 'prg%d%sz%s.grib2' % (day, utc.zfill(2), tvDay)        
+    wg2filepath = os.path.join(opath, wg2filename)
+    if __OVERWRITE__ and os.path.isfile(wg2filepath): os.remove(wg2filepath)
+
+    ensfpath = os.path.join(inpath, files[0])
+    inf = iris.load(ensfpath)
+    for varName, varSTASH  in neededVars:
+        # define variable name constraint
+        varConstraint = iris.Constraint(name=varName)
+        print varName, varSTASH
+        ensCube = inf.extract(varConstraint)
+        if not ensCube: continue        
+        print "doing members average for", varName
+        # do average over ensembles (1 control + 44 ensemble members)
+        ensAvgCube = cubeRealizationAverager(ensCube[0])
+        # make memory free
+        del ensCube
+
+        if (varName, varSTASH) in _precipVars_:
+            # DO NOT APPLY iris.analysis.Linear(extrapolation_mode='mask'), 
+            # which writes nan every where for the snowfall_flux,  
+            # rainfall_flux, precipitation_flux. So donot apply that.         
+            exmode = 'linear'
+        else:
+            # In general all the other variables should not be 
+            # extrapolated over masked grid points.
+            exmode = 'mask'
+        # end of if (...):
+
+        scheme = iris.analysis.Linear(extrapolation_mode=exmode)
+        try:
+            # This lienar interpolate will do extra polate over ocean even 
+            # though original data doesnt have values over ocean and wise versa.
+            # So lets be aware of this.            
+            regdCube = ensAvgCube.regrid(_targetGrid_, scheme)
+        except Exception as e:
+            print "ALERT !!! Error while regridding!! %s" % str(e)
+            print " So skipping this without saving data"
+            continue
+        # end of try:      
+
+        # make memory free
+        del ensAvgCube
+
+        if (varName, varSTASH) in _precipVars_:
+            # Since we are not using 'mask' option for extrapolate while 
+            # doing linear regrid, which bring -ve values after regrid in 
+            # extrapolated grids. So lets make it as 0 as minimum value.
+            regdCube.data[regdCube.data < 0.0] = 0.0
+        # end of if (varName, varSTASH) in _precipVars_:
+
+        if exmode == 'mask':            
+            regdCube.data = numpy.ma.masked_array(regdCube.data, 
+                                dtype=numpy.float64, fill_value=9.999e+20) 
+
+        print regdCube
+        # save into grib2 file 
+        iris.fileformats.grib.save_messages(tweaked_messages([regdCube]), 
+                                                     g2filepath, append=True) # save grib2 file        
+
+        print "Appending %s to grib2 file" % varName
+        # make memory free
+        del regdCube       
+    # end of for varName, varSTASH  in neededVars:                      
+
+    # execute post wgrib2 command compression algorithm
+    cmd = "%s %s %s %s" % (wgrib2, g2filepath, __wgrib2Arguments__, wg2filepath)
+    print cmd
+    subprocess.call(cmd, shell=True)            
+    time.sleep(10)
+    # remove the grib2 file generated by IRIS
+    os.remove(g2filepath)
+    # rename g2filepath as wg2filepath
+    g2filepath = wg2filepath
+    print "Created grib2 file using wgrib2 command with compress arguments " 
+
+    # Conver grib2 to grib1 
+    g1filepath = '.'.join(g2filepath.split('.')[:-1])
+    g1filepath = g1filepath if g1filepath else g2filepath[:-1]
+    if __grib1FilesNameSuffix__: g1filepath += str(__grib1FilesNameSuffix__)
+
+    if os.path.isfile(g1filepath): os.remove(g1filepath)
+
+    cmd = [cnvgrib, '-g21', g2filepath, g1filepath]
+    subprocess.call(cmd, shell=False)
+    cmd = ['chmod', '644', g1filepath]
+    subprocess.call(cmd, shell=False)
+    print "Converted grib2 to grib1 file : -", g1filepath
+    os.remove(g2filepath)                                
+# end of def createTarBalls(path, today, ...):
+
+if __name__ == '__main__':
+
+
+    date = None
+    inpath = None
+    oftype = None
+    utc = None
+    outpath = '/gpfs3/home/umeps/EPS/ShortJobs/NCUM_EPS_VSDB_Input'
+
+    helpmsg = './ncumeps_create_memavg_vsdb_input.py --date=20160302 --outpath=path --oftype=forecast --utc=00 --start_long_fcst_hour=24 --end_long_fcst_hour=24 --fcst_step_hour=24'
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], "d:o:t:z:s:e:i", ["date=",
+            "outpath=", "oftype=", "utc=", "start_long_fcst_hour=", "end_long_fcst_hour=", "fcst_step_hour="])
+    except getopt.GetoptError:
+        print helpmsg
+        sys.exit(2)
+    for opt, arg in opts:
+        if opt == '-h':
+            print helpmsg
+            sys.exit()
+        elif opt in ("-d", "--date"):
+            date = arg
+        elif opt in ("-o", "--outpath"):
+            inpath = arg 
+        elif opt in ("-t", "--oftype"):
+            oftype = arg
+        elif opt in ("-z", "--utc"):
+            utc = arg
+        elif opt in ("-s", "--start_long_fcst_hour"):
+            start_long_fcst_hour = arg
+        elif opt in ("-e", "--end_long_fcst_hour"):
+            end_long_fcst_hour = arg
+        elif opt in ("-i", "--fcst_step_hour"):
+            fcst_step_hour = arg
+    # end of for opt, arg in opts:
+
+    # create tar balls only if forecast & utc is 00, otherwise skip it!    
+    if oftype == 'forecast' and utc == '00' and fcst_step_hour == '24': 
+        # pass the arg to function  
+        createENSavg_VSDB_Grib1Files(inpath, outpath, date, utc, start_long_fcst_hour, stephr=fcst_step_hour)    
+    # end of if oftype == 'forecast' and utc == '00': 

bsubScripts/ncumeps_global_post/ncumeps_global_post_anl_00Z.bash

@@ -0,0 +1,47 @@
+#!/bin/bash
+#
+#BSUB -a poe                  # set parallel operating environment
+#BSUB -J umeps2g2anl          # job name
+#BSUB -W 06:00                # wall-clock time (hrs:mins)
+#BSUB -n 16                  # number of tasks in job
+#BSUB -x
+#BSUB -R span[ptile=16]
+#BSUB -R rusage[mem=61440]
+#BSUB -q small             	  # queue
+#BSUB -e /gpfs3/home/umeps/UMRiderLogs/post/bsub/um2grb2.anl.00hr.err.%J.%I.hybrid     # error file name in which %J is replaced by the job ID
+#BSUB -o /gpfs3/home/umeps/UMRiderLogs/post/bsub/um2grb2.anl.00hr.out.%J.%I.hybrid     # output file name in which %J is replaced by the job ID
+
+
+# find out the directory of this bash script after submitted to bsub
+DIR="$( cd "$( dirname "${BASH_SOURCE[1]}" )" && pwd )"
+
+# get the absolute path of the local table 
+localTable_relative_dir="$DIR/../../tables/local/ncmr/v1/"
+localTable_absolute_dir="$( cd "$localTable_relative_dir" && pwd )"
+localTable=$localTable_absolute_dir/ncmr_grib2_local_table
+
+# get the absolute path of the script for forecast 00utc
+g2scripts_relative_dir="$DIR/../../g2scripts/"
+g2scripts_absolute_dir="$( cd "$g2scripts_relative_dir" && pwd )"
+g2script=$g2scripts_absolute_dir/umeps2grb2_fcst_00Z.py
+
+# export the configure paths to needed variables
+export UMRIDER_SETUP=$DIR/ncumeps_global_post_um2grb2_24hourly_setup.cfg  ## yes, this is correct program.
+export UMRIDER_VARS=$DIR/ncumeps_global_post_um2grb2_anl_vars.cfg
+export GRIB2TABLE=$localTable
+
+echo "export UMRIDER_SETUP="$UMRIDER_SETUP
+echo "export UMRIDER_VARS="$UMRIDER_VARS
+echo "export GRIB2TABLE="$GRIB2TABLE
+
+# sourcing umtid_bashrc to load module python-uvcdat-iris!
+#source "$DIR/../umtid_bashrc"
+# execute the script
+#srun -n 1 -c 64 python $g2script
+
+export SHELL=/bin/bash
+
+hour=0  # 0 will produce 0th hour prognostic data (equivalent to analysis)
+echo "hour="${hour}
+
+python $g2script --start_long_fcst_hour=${hour} --end_long_fcst_hour=${hour}

bsubScripts/ncumeps_global_post/ncumeps_global_post_fcst_00Z_6hourly.bash

@@ -8,8 +8,8 @@
 #BSUB -R span[ptile=16]
 #BSUB -R rusage[mem=61440]
 #BSUB -q ensemble             	  # queue
-#BSUB -e /gpfs3/home/umeps/UMRiderLogs/post/bsub/um2grb2.fcst.00hr.err.%J.hybrid     # error file name in which %J is replaced by the job ID
-#BSUB -o /gpfs3/home/umeps/UMRiderLogs/post/bsub/um2grb2.fcst.00hr.out.%J.hybrid     # output file name in which %J is replaced by the job ID
+#BSUB -e /gpfs3/home/umeps/UMRiderLogs/post/bsub/um2grb2.fcst.06hr.err.%J.hybrid     # error file name in which %J is replaced by the job ID
+#BSUB -o /gpfs3/home/umeps/UMRiderLogs/post/bsub/um2grb2.fcst.06hr.out.%J.hybrid     # output file name in which %J is replaced by the job ID
 
 # find out the directory of this bash script after submitted to bsub
 DIR="$( cd "$( dirname "${BASH_SOURCE[1]}" )" && pwd )"

bsubScripts/ncumeps_global_post/ncumeps_global_post_um2grb2_24hourly_setup.cfg

@@ -242,6 +242,6 @@ wgrib2Arguments = -ncpu 4 -set_bin_prec 12 -set_grib_type complex2 -append -grib
 ##       '--outpath' -> out files path,
 ##       '--oftype'  -> 'analysis' or 'forecast'
 ##       '--utc'  -> UTC cycle value in string ('00' or '06' or '12' or '18')  
-callBackScript = None
+callBackScript = ./ncumeps_create_memavg_vsdb_input.py
 
 ############ END OF UMRIDER SETUP CONFIGURE FOR um2grb2 SCRIPTS ##############

bsubScripts/ncumeps_global_post/ncumeps_global_post_um2grb2_anl_vars.cfg

@@ -0,0 +1,21 @@
+##############################################################################
+## vars configure file: Used for the purpose of um2grb2 conversion of only  ##
+## needed NCUM model out variables. um2grb2 python parallel scripts will    ##
+## create analysis and forecast files, by conveting to gri2 file only for   ##
+## for the following cf_standard_name and varSTASH coded vars.              ##
+##                                                                          ##
+## Author : Arulalan <arulalan@ncmrwf.gov.in>                               ##
+## Updated : 18-Feb-2016                                                    ##
+##############################################################################
+
+##### BEGIN OF UMRIDER VARS CONFIGURE FOR um2grb2 SCRIPTS ######
+## Pressure Level Variable names & STASH codes
+
+('geopotential_height', 'm01s16i202')
+('relative_humidity', 'm01s16i256')
+('x_wind', 'm01s15i243')
+('y_wind', 'm01s15i244')
+('upward_air_velocity', 'm01s15i242')
+('air_temperature', 'm01s16i203')
+('air_pressure_at_sea_level', 'm01s16i222')
+('surface_air_pressure', 'm01s00i409')

bsubScripts/umtid_bashrc

@@ -16,4 +16,5 @@ module load GNU-qt/4.8.4
 module load GNU-gribapi/1.13.1
 module load GNU-wgrib2/2.0.4
 module load cnvgrib/1.4.1
+module load GNU-grads/grads-2.1.a2
 # end of moulde load