version 01.00.32

Release/MANIFEST.in

@@ -2,6 +2,8 @@
 
 recursive-include docs *
 include LICENSE
+include dreqPy/utilP2/__init__.py
+include dreqPy/utilP2/util.py
 include dreqPy/docs/BlockSchema.csv
 include dreqPy/docs/CMIP6_MIP_tables.xlsx
 include dreqPy/docs/dc1.xsd
@@ -26,5 +28,3 @@ include dreqPy/docs/dreqSupp.xml
 include dreqPy/docs/dreqManifest.txt
 include dreqPy/docs/md5Manifest.txt
 include dreqPy/docs/sfheadings.csv
-include dreqPy/utilP2/__init__.py
-include dreqPy/utilP2/util.py

Release/dreqPy/__init__.py

@@ -5,13 +5,3 @@
 except:
   #from dreqPy import packageConfig
   from .packageConfig import *
-
-try:
-  import utilP2
-except:
-  from dreqPy import utilP2
-
-##try:
-  ##import utilP2.util
-##except:
-  ##from dreqPy.utilP2 import util

Release/dreqPy/docs/dreq2Defn.xml

@@ -127,7 +127,7 @@ which  case  a  primary  realm  is  assigned as the first listed and other relev
   <rowAttribute label="label" type="xs:string" title="Record Label" useClass="" techNote="" description="" uid="ATTRIBUTE::tableSection.label" superclass="" required="true"/>
   <rowAttribute label="title" type="xs:string" title="Record Title" useClass="" techNote="" description="" uid="ATTRIBUTE::tableSection.title" superclass="" required="true"/>
   <rowAttribute label="uid" type="aa:st__uid" title="Record Identifier" useClass="recordId" techNote="" description="" uid="ATTRIBUTE::tableSection.uid" superclass="" required="true"/>
-  <rowAttribute label="gpid" type="xs:string" title="Identifier for CMOR Tables" useClass="internalLink" techNote="requestVarGroup" description="Link to a record defining a CMOR table." uid="ATTRIBUTE::tableSection.gpid" superclass="" required="true"/>
+  <rowAttribute label="gpid" type="xs:string" title="Identifier for CMOR Tables" useClass="internalLink" techNote="mipTable" description="Link to a record defining a CMOR table." uid="ATTRIBUTE::tableSection.gpid" superclass="" required="true"/>
   <rowAttribute label="mip" type="xs:string" title="Project" useClass="" techNote="" description="Redundant. Specification of a MIP." uid="ATTRIBUTE::tableSection.mip" superclass="" required="true"/>
   <rowAttribute label="ref" type="xs:string" title="Reference" useClass="" techNote="" description="Comment on provenance." uid="ATTRIBUTE::tableSection.ref" superclass="" required="false"/>
   <rowAttribute label="refNote" type="xs:string" title="Note on reference" useClass="" techNote="" description="Further information on provenance." uid="ATTRIBUTE::tableSection.refNote" superclass="" required="false"/>

Release/dreqPy/dreq.py

@@ -384,15 +384,23 @@ def dictInit( self, idict ):
 
        def mdInit( self, el, etree=False ):
          __doc__ = """Initialisation from a mindom XML element. The list of attributes must be set by the class factory before the class is initialised"""
-         deferredHandling=False
-         nw1 = 0
          tvtl = []
          if etree:
            ks = set( el.keys() )
            for a in self._a.keys():
              if a in ks:
                aa = '%s%s' % (self.ns,a)
-               tvtl.append( (a,True, str( el.get( a ) ) ) )
+#
+# exception trapping to provide more diagnostics when a character code issue arises
+# added in 01.00.32beta
+#
+               this = el.get( a )
+               try:
+                 tvtl.append( (a,True, str( this ) ) )
+               except:
+                 print( 'ERROR: FAILED to str %s: %s' % (a,this) )
+                 tvtl.append( (a,True, this.encode( 'utf-8' ) ) )
+
              elif self._a[a].__dict__.get( 'required', True ) in [False,'false',u'false']:
                tvtl.append( (a,True,None) )
              else:
@@ -408,18 +416,82 @@ def mdInit( self, el, etree=False ):
 ##
              elif self._a[a].__dict__.get( 'required', True ) not in [False,'false',u'false']:
                tvtl.append( (a,False,None) )
-
+
+         self._tvtlInit( tvtl )
+         self._contentInitialised = True
+
+       def _tvtlInit( self, tvtl ):
+         """Coercing string values into a python objects appropriate to declared types: looping over tuples containing tags and values.
+            the code sets the default if appropriate, otherwise call _avInit to coerce the string value to an appropriate python object."""
+
+         nw1 = 0
          for a,tv,v in tvtl:
            if a == 'uid':
              uid = v
 
          for a,tv,v in tvtl:
            if tv:
-             erase = False
              if v == None:
-               pass
-               erase = True
-             elif self._a[a].type == u'xs:float':
+               self.__dict__[a] = '__tmp__'
+               delattr( self, a )
+               ### need to overwrite attribute (which is inherited from parent class) before deleting it.
+               ### this may not be needed in python3
+             else:
+               self._avInit( a, v )
+           else:
+             if a in ['uid',]:
+               thissect = '%s' % (self._h.title)
+               print ( 'ERROR.020.0001: missing uid: %s: a,tv,v: %s, %s, %s\n %s' % (thissect,a,tv,v,str( self.__dict__ )) )
+               ##if etree:
+                 ##p = self.parent_map( el )
+                 ##print ( ks )
+               raise
+                 ##import sys
+                 ##sys.exit(0)
+             self.__dict__[a] = self._d.defaults.get( a, self._d.glob )
+
+
+       def _avInit( self, a, v ):
+             """Coercing a string value into a python object appropriate to the declared type: non-numeric data types:
+                 a: the tag identifying the data element;
+                 v: the string serialisation of the data value:
+
+                 Approach
+                 --------
+                 (1) Check whether type is numric -- if true call _avInitNumeric;
+                 (2) Otherwise, deal with boolean or string list types, or default which is to take string value as is."""
+
+             numericTypes = [u'xs:float',u'aa:st__floatList', u'aa:st__floatListMonInc',u'aa:st__integerList', u'aa:st__integerListMonInc',u'xs:integer']
+
+             if self._a[a].type in numericTypes:
+               return self._avInitNumeric( a,v )
+             elif self._a[a].type == u'xs:boolean':
+               v = v in ['true','1']
+             elif self._a[a].type == u'aa:st__stringList':
+               if v.find(' ') != -1:
+                 v = tuple( v.split() )
+               else:
+                 v = (v,)
+
+             elif self._a[a].type not in [u'xs:string', u'aa:st__uid', 'aa:st__fortranType', 'aa:st__configurationType','aa:st__sliceType']:
+               print ('ERROR: Type %s not recognised [%s:%s]' % (self._a[a].type,self._h.label,a) )
+
+             self.__dict__[a] = v
+
+       def _avInitNumeric( self, a, v ):
+             """Coercing a value into a python object appropriate to the declared type: numeric data types.
+                 a: the tag identifying the data element;
+                 v: the string serialisation of the data value:
+                 
+                 list types are split into elements;
+                 integers coerced with the python int function;
+                 floats coerced with the python float function;
+                 -- if integer coercion fails, indirect coercion is attempted via int (float (v) )
+                 -- if specified in the type, monotonicity is checked.
+                 -- if type is integer, a blank is taken as zero.
+             """
+             msg = ''
+             if self._a[a].type == u'xs:float':
                if v == '':
                  v = None
                else:
@@ -457,41 +529,10 @@ def mdInit( self, el, etree=False ):
                      print ( 'WARN: input integer non-compliant: %s: %s: %s [%s] %s' % (thissect,a,v0,v,uid) )
                    except:
                      msg = 'ERROR: failed to convert integer: %s: %s: %s, %s' % (thissect,a,v,type(v))
-                     deferredHandling=True
-             elif self._a[a].type == u'xs:boolean':
-               v = v in ['true','1']
-             elif self._a[a].type == u'aa:st__stringList':
-               if v.find(' ') != -1:
-                 v = tuple( v.split() )
-               else:
-                 v = (v,)
-             elif self._a[a].type not in [u'xs:string', u'aa:st__uid', 'aa:st__fortranType', 'aa:st__configurationType','aa:st__sliceType']:
-               print ('ERROR: Type %s not recognised [%s:%s]' % (self._a[a].type,self._h.label,a) )
-
-             if erase:
-               ### need to overwrite attribute (which is inherited from parent class) before deleting it.
-               ### this may not be needed in python3
-               self.__dict__[a] = '__tmp__'
-               delattr( self, a )
+                     print ( msg )
              else:
-               self.__dict__[a] = v
-           else:
-             if a in ['uid',]:
-               thissect = '%s' % (self._h.title)
-               print ( 'ERROR.020.0001: missing uid: %s: a,tv,v: %s, %s, %s\n %s' % (thissect,a,tv,v,str( self.__dict__ )) )
-               if etree:
-                 ##p = self.parent_map( el )
-                 print ( ks )
-                 raise
-                 import sys
-                 sys.exit(0)
-             self.__dict__[a] = self._d.defaults.get( a, self._d.glob )
-
-           if deferredHandling:
-             print ( msg )
-
-         self._contentInitialised = True
-
+               raise
+             self.__dict__[a] = v
 
 class config(object):
   """Read in a vocabulary collection configuration document and a vocabulary document"""
@@ -565,6 +606,7 @@ def __read__(self, thisdoc, configdoc):
 ##
     self.etree = False
     self.etree = True
+    root=None
     self.ns = None
     if not self.configOnly:
       if self.etree:
@@ -660,11 +702,19 @@ def __read__(self, thisdoc, configdoc):
 ##
     self.coll['__core__'] = self.ntf( self._t0.header, self._t0.attributes, [self.tt0[k] for k in self.tt0] )
 
+    self._read_vl(vl,tables)
+
+    for k in tables:
+      self.recordAttributeDefn[k] = tables[k]
+
+    if not self.configOnly:
+      self.__read_tables__(root,tables)
+
+  def _read_vl(self,vl,tables):
     ec = {}
     for i in self.coll['__core__'].items:
       ec[i.label] = i
 
-
     for v in vl:
       t = self.parsevcfg(v)
       tables[t[0].label] = t
@@ -688,11 +738,7 @@ def __read__(self, thisdoc, configdoc):
         assert k in ec, 'Key %s [%s] not found' % (k,sct)
         self.coll[sct].attDefn[k] = ec[k]
 
-    for k in tables:
-      self.recordAttributeDefn[k] = tables[k]
-
-    if self.configOnly:
-      return
+  def __read_tables__(self,root,tables):
     for k in tables.keys():
       if self.etree:
         vl = root.findall( './/%s%s' % (self.ns,k) )

Release/dreqPy/dreqCmdl.py

@@ -34,12 +34,20 @@ def main_entry():
         import simpleCheck
       else:
         from . import simpleCheck
+      if '--strict' in sys.argv:
+        assert simpleCheck.all, 'Errors detected in simpleCheck'
       print( "Starting test suite 2" )
       if scr:
         import examples.ex203 as ex203
       else:
         from .examples import ex203
       ex203.main( scope )
+      print( "Starting test suite 3" )
+      if scr:
+        import utilP2.info as info
+      else:
+        from .utilP2 import info
+      print( info.__doc__ )
       print( "Tests completed" )
   elif sys.argv[1] == '--makeTables':
       print( "Making web page tables" )

Release/dreqPy/extensions/collect.py

@@ -32,6 +32,42 @@ def _isLinked(self):
       return True
   return False
 
+###
+### can generalise here
+###
+### _inx.iref_by_sect[ <this uid> ].a[ <path attribute:target:this sect> ]
+###             
+### 
+### resolveRequestPath( target=<var|mip|experiment|.....>, )
+###
+### "path" is a given by "spine" plus - objective - link.   3 elements. once origin and target are known, it reduces to a line: this logic sits in
+### instantiation.
+### Alternatively, if we instantiate only on the start, the target becomes a call argument: the path should then be derived from the links. 
+####
+###  E.g. assign each branch a name (mip,var,expt) and number sections (varPack = var.1),  then X.n --> Y.m goes X.n-1, .... C, y.1, ..Y.m
+### Gives a simple alogoithm to establish path from current location.
+### Not much benefit gained from instantiation on start ... but it gives some transparency.
+
+### to modify doc strings, need to use a class factory rather than instantiation: cannot write to __doc__ attribute of a method
+### You can modify the class .... but that doesn't help ..
+### unless all the generic stuff is in a base class ..
+###
+
+def manufacture_resolve_requestPath(start):
+  _startId = get_sectionId(start)
+  def _resolve_requestPath(self,target,filter=None,fmt=None):
+    startId = _startId
+    targetId = get_sectionId(target)
+
+
+
+class ResolveRequestPathMaster(object):
+   def __init__( start='requestLink'):
+     self.start = start
+
+   def __call__(self,target,filter=None,fmt=None):
+     """Return set of target objects linked on request path ..."""
+###
 def _var__mips(self):
     """Return set of mip item identifiers for MIPs requesting this var"""
     mips = set()

Release/dreqPy/makeTables.py

@@ -379,7 +379,7 @@ def run():
   assert os.path.isdir( 'html/index' ), 'Before running this script you need to create "html", "html/index" and "html/u" sub-directories, or edit the call to dq.makeHtml, and refernces to "u" in style lines below'
   assert os.path.isdir( 'tables' ), 'Before running this script you need to create a "tables" sub-directory, or edit the table_utils.makeTab class'
 
-  dq = dreq.loadDreq( htmlStyles=htmlStyle, manifest='docs/dreqManifest.txt' )
+  dq = dreq.loadDreq( htmlStyles=htmlStyle, manifest='out/dreqManifest.txt' )
 ##
 ## add special styles to dq object "itemStyle" dictionary.
 ##

Release/dreqPy/overviewTabs.py

@@ -53,8 +53,8 @@ def __init__(self,sc,mt_tables,tiermax=1,pmax=1,only=False,vols=None,fnm='new',m
     assert vols == None or type(vols) == type( () ), 'vols argument must be none or tuple of length 2: %s' % type(vols)
     self.dq = sc.dq
     self.mips = ['CMIP'] + scope_utils.mips
-    self.mipsp = [x for x in self.mips if x not in scope_utils.mipsdiag]
-
+    self.mipsp = self.mips[:-3]
+    self.mipsp.remove( 'VIACSAB' ) 
     self.sc = sc
     self.pmax=pmax
     self.efnsfx = ''

Release/dreqPy/packageConfig.py

@@ -16,7 +16,7 @@
 VERSION_DEFAULT_DIR = os.path.join(HOME, '.dreqPy')
 VERSION_DIR = os.environ.get('DRQ_VERSION_DIR', VERSION_DEFAULT_DIR)
 
-__version__ = "01.00.31.post6"
+__version__ = "01.00.32"
 __versionComment__ = "Version %s" % __version__
 __title__ = "dreqPy"
 __description__ = "CMIP6 Data Request Python API"

Release/dreqPy/scope.py

@@ -254,10 +254,8 @@ def __init__(self,dq=None,tierMax=1):
 
     ##self.mips = set( [x.label for x in self.dq.coll['mip'].items ] )
     ##self.mips = ['CMIP','AerChemMIP', 'C4MIP', 'CFMIP', 'DAMIP', 'DCPP', 'FAFMIP', 'GeoMIP', 'GMMIP', 'HighResMIP', 'ISMIP6', 'LS3MIP', 'LUMIP', 'OMIP', 'PAMIP', 'PMIP', 'RFMIP', 'ScenarioMIP', 'VolMIP', 'CORDEX', 'DynVar', 'DynVarMIP', 'SIMIP', 'VIACSAB']
-
     self.mips = ['CMIP'] + scope_utils.mips
-    self.mipsp = [x for x in self.mips if x not in scope_utils.mipsdiag]
-
+    self.mipsp = ['CMIP'] + scope_utils.mipsp
     self.cmvGridId, i4 = fgrid.fgrid( self.dq )
     assert len(i4) == 0
 
@@ -1622,7 +1620,7 @@ class dreqUI(object):
               <section:attribute>: definition of an attribute.
       -h :       help: print help text;
       -e <expt>: experiment;
-      -t <tier> maxmum tier;
+      -t <tier> maximum tier;
       -p <priority>  maximum priority;
       --xls : Create Excel file with requested variables;
       --sf : Print summary of variable count by structure and frequency [default];
@@ -1836,6 +1834,7 @@ def run(self, dq=None):
     if 'mcfg' in self.adict:
       self.sc.setMcfg( lli )
 
+    self.tierByDefault = 't' not in self.adict
     tierMax = self.adict.get( 't', 1 )
     self.sc.setTierMax(  tierMax )
     pmax = self.adict.get( 'p', 1 )
@@ -1901,9 +1900,11 @@ def run(self, dq=None):
       if ex in self.sc.mipsp:
         eid = set( self.dq.inx.iref_by_sect[ex].a['experiment'] )
         self.sc.exptFilter = eid
+        self.exptMode = 'mip'
       elif self.adict['e'] in self.sc.exptByLabel:
         eid = self.sc.exptByLabel[ self.adict['e'] ]
         self.sc.exptFilter = set( [eid,] )
+        self.exptMode = 'expt'
       else:
         try:
           ns = 0
@@ -1936,6 +1937,11 @@ def run(self, dq=None):
           print ( """WARNING: filter settings give no experiments: try using --esm flag: by default esm experiments are filtered out""" )
           return
 
+    if self.exptMode == 'expt' and self.tierByDefault:
+      tier = min(self.dq.inx.uid[eid].tier)
+      if tier > tierMax:
+         print ( """WARNING: default is to filter experiment tier > %s, default tier for %s is %s""" % (tierMax, self.dq.inx.uid[eid].label, tier) )
+         print ( """Resubmit with '-t %s' to avoid this""" % tier )
 
     makeTxt = self.adict.get( 'txt', False )
     makeXls = self.adict.get( 'xls', False )
@@ -1984,7 +1990,7 @@ def run(self, dq=None):
           vsmode='full'
         else:
           vsmode='short'
-        vs.anal(olab=mlab,doUnique=True, mode=vsmode, makeTabs=makeXls)
+        vs.anal(olab=mlab,doUnique=False, mode=vsmode, makeTabs=makeXls)
         for f in sorted( vs.res['vf'].keys() ):
            mlg.prnt ( 'Frequency: %s: %s' % (f, vs.res['vf'][f]*2.*1.e-12 ) )
         ttl = sum( [x for k,x in vs.res['vu'].items()] )*2.*1.e-12