Merge pull request #102 from ODM2/development

Development 0.8.0

setup/Mac/sdl_setup.packproj

@@ -548,7 +548,7 @@
 		<key>IFPkgFlagPackageSelection</key>
 		<integer>0</integer>
 		<key>Name</key>
-		<string>SDL_v0.7.0_Beta_Mac_Installer</string>
+		<string>SDL_v0.8.0_Beta_Mac_Installer</string>
 		<key>Status</key>
 		<integer>1</integer>
 		<key>Type</key>

setup/Windows/sdl_setup.iss

@@ -4,7 +4,7 @@
 #define MyAppName "StreamingDataLoader"
 #define MyAppInstaller "SDL"
 #define MyWizExeName "SDLLoader"
-#define MyAppVersion "v0.7.0-beta"
+#define MyAppVersion "v0.8.0-beta"
 #define MyAppPublisher "ODM2"
 #define MyAppURL "https://github.com/ODM2/ODM2StreamingDataLoader"
 

setup/Windows/setup instructions.txt

@@ -1,3 +1,5 @@
 run the following commands 
+
 pyinstaller --onefile src\StreamingDataLoader.py
+
 pyinstaller --onefile src\StreamingDataLoaderWizard.py

src/StreamingDataLoader.py

@@ -153,13 +153,28 @@ def main(arguments):
     parser = ArgumentParser(description="StreamingDataLoader")
     parser.add_argument('-r', '--restart', action='store_true', help="Read the entire CSV file to ensure that data is correct and accounted for. This option affects performance. While the integrity of the data is checked each time the program executes, it is still recommended to use this option if you have manually modified your data file between executions.")
     parser.add_argument('-v', '--verbose', action='store_true', help="Enable more verbose logging in the logfile.")
-    parser.add_argument('-c', '--config', nargs='+', dest="yamlFile", help="Specify a YAML configuration file in the form of one of these formats:\n1. A single YAML (.yaml) file.\n2. A list of YAML files (.yaml), deliminated by white space.\n3. A directory containing multiple YAML (.yaml) files.", required=True, action="store")
+    parser.add_argument('-c', '--config', nargs='+', dest="yamlFile", help="Specify a YAML configuration file in the form of one of these formats:\n1. A single YAML (.yaml) file.\n2. A list of YAML files (.yaml), deliminated by white space.\n3. A directory containing multiple YAML (.yaml) files.", required=False, action="store")
     parser.add_argument('-f', '--file', dest="csvFile", help="Specify a single CSV data file to target instead of the one listed in the configuration file.", required=False, action="store")
     args = parser.parse_args()
 
+
     correct_path = ""
-    for path in args.yamlFile:
-        correct_path += path + " "
+
+
+
+    if args.yamlFile is not None:
+        # if args in args.yamlFile:
+        correct_path = ""
+        for path in args.yamlFile:
+            correct_path += path + " "
+    else:
+        # os.chdir(os.getcwd())
+        for filename in os.listdir('.'):
+            if filename.endswith('.yaml'):
+                correct_path = os.getcwd()
+                # correct_path += filename
+                correct_path = os.path.join(correct_path, filename)
+                break
 
     args.yamlFile = [correct_path.strip()]
     start_time = time.time()

src/StreamingDataLoaderWizard.py

@@ -1,19 +1,4 @@
-# import os
-# import sys
 
-# import ObjectListView
-# import pyodbc
-# import pymysql
-# import sqlite3
-# sys.path.insert(0,os.path.dirname(os.path.dirname(__file__)))
-# from src.wizard.controller.frmMain import MainController
-# import wx
-# if __name__ == '__main__':
-    # app = wx.App()
-    # frame = MainController(None)
-    # frame.CenterOnScreen()
-    # frame.Show()
-    # app.MainLoop()
 
 import os
 import sys

src/controllers/Database.py

@@ -34,15 +34,18 @@ def createConnection(self, Credentials):
         print Credentials.host
         print Credentials.db_name
         print Credentials.uid
-        print Credentials.pwd
+        # print Credentials.pwd
 
         self.session_factory = \
             dbconnection.createConnection(Credentials.engine, Credentials.host,
                                             Credentials.db_name,
                                             Credentials.uid,
                                             Credentials.pwd, 2.0)
         if not self.session_factory:
-            logger.error("Unable to connect to database with host='%s', database='%s', user='%s', pwd='%s'." % (Credentials.host, Credentials.db_name, Credentials.uid, Credentials.pwd))
+            message = "Unable to connect to database with host='%s', database='%s', user='%s', pwd='%s'." % (Credentials.host, Credentials.db_name, Credentials.uid, Credentials.pwd)
+
+            logger.error(message)
+            print message
             return False
 
         return True

src/controllers/Mapper.py

@@ -197,9 +197,8 @@ def _buildTables(self):
             # SDL Test Data is just for our testing purposes.
             df['QualityCodeCV'] = 'None'
             # df['QualityCodeCV'] = 'SDL Test Data'
-            # TODO add unknown to database.
-            #df['CensorCodeCV'] = 'Unknown'
-            df['CensorCodeCV'] = 'Non-detect'
+
+            df['CensorCodeCV'] = 'Not censored'
             df['ResultID'] = series['ResultID']
             df['ValueDateTimeUTCOffset'] = self.mapping['Settings']['UTCOffset']
 

src/meta/data.py

@@ -1,5 +1,5 @@
 app_name = "Streaming Data Loader"
-version = "0.7.0_Beta"
+version = "0.8.0_Beta"
 copyright = "Copyright (c) 2013 - 2015, Utah State University. All rights reserved."
 description = "Streaming Data Loader a program for streaming continuous sensor data into an instance of the Observations Data Model (ODM)"
 

src/test.yaml

@@ -0,0 +1,30 @@
+## Configuration file
+## Last modified: 2017-07-11 13:39
+---
+test:
+  Database:
+    Address: localhost
+    DatabaseName: odm2
+    Engine: mysql
+    Password: bobrules
+    UserName: Bob
+  Mappings:
+    RelativeHumidity-%:
+      CalculateAggInterval: 'false'
+      IntendedTimeSpacing: 0
+      IntendedTimeSpacingUnitID: 136
+      LastByteRead: '127003'
+      ResultID: 22293
+  Schedule:
+    Beginning: 01/01/2014 12:00:00 AM
+    Frequency: Hour
+    LastUpdate: '2017-07-11 13:39:41'
+    Time: 1
+  Settings:
+    DataRowPosition: 21
+    DateTimeColumnName: DateTime
+    Delimiter: ','
+    FileLocation: C:\Sites\ODM2StreamingDataLoader\ODM2StreamingDataLoader\tests\test_handlers\test_csvHandler\csvFiles\Treeline_HrlySummary_2014.csv
+    FillGaps: 'false'
+    HeaderRowPosition: 20
+    UTCOffset: 0

src/wizard/controller/WizardDialog.py

@@ -62,8 +62,8 @@ def addButtons(self):
         self.btnNext.Bind(wx.EVT_BUTTON, self.onFinish)
         self.btnPrev.Bind(wx.EVT_BUTTON, self.onPrev)
 
-    def addPage(self, pnl):
-        newPnl = pnl(self, self.existingResult)
+    def addPage(self, pnl, **kwargs):
+        newPnl = pnl(self, existing_result= self.existingResult, **kwargs)
         newPnl.Hide()
         self.pnlList.append(newPnl)
         self.pnlSizer.Add(newPnl, 1, wx.ALL|wx.EXPAND, 5)

src/wizard/controller/frmAddNewActionsPanel.py

@@ -51,11 +51,15 @@ def onNewMethod(self, event):
         newMethodPanel.setTypeFilter(str(self.action_type_combo.GetStringSelection()))
         if dlg.ShowModal() == wx.ID_OK:
             newMethod = newMethodPanel.method
-            self.methods = [{i.MethodName:i.MethodID}\
-                for i in [newMethod]]
+            self.methods = [{i.MethodName: i.MethodID} for i in [newMethod]]
             self.method_combo.AppendItems([y for x in [i.keys() for i in self.methods] for y in x])
-            #self.m_comboBox134.SetValue(newMethod.MethodName)
+            # self.method_combo.SetValue(newMethod.MethodName)
+            i = self.method_combo.FindString(newMethod.MethodName)
+            self.method_combo.Select(i)
         dlg.Destroy()
+        if self.method_combo.IsEmpty():
+            self.method_combo.Select(0)
+
         event.Skip()
 
     def onNewAffiliation(self, event):
@@ -91,11 +95,15 @@ def onOK(self, event):
                     MethodID=self.methodID,
                     BeginDateTime=self.beginDT,
                     BeginDateTimeUTCOffset=self.beginDTUTC,
-                    EndDateTime=self.endDT,
-                    EndDateTimeUTCOffset=self.endDTUTC,
-                    ActionDescription=self.actionDesc,
-                    ActionFileLink=self.actionLink
                 )
+                try:
+                    action.EndDateTime = self.endDT,
+                    action.EndDateTimeUTCOffset = self.endDTUTC,
+                    action.ActionDescription = self.actionDesc,
+                    action.ActionFileLink = self.actionLink
+                except Exception as error:
+                    print error
+
                 action = write.createAction(action)
 
                 self.actionID = action.ActionID
@@ -121,19 +129,30 @@ def onOK(self, event):
 
     def getFieldValues(self):
 
-        keys = [y for x in [i.keys() for i in self.methods] for y in x]
-        vals = [y for x in [i.values() for i in self.methods] for y in x]
+        keys = [y for x in [aff.keys() for aff in self.methods] for y in x]
+        vals = [y for x in [aff.values() for aff in self.methods] for y in x]
         d = dict(zip(keys, vals))
 
+#         self.ActionType = str(self.action_type_combo.GetStringSelection())
+#         self.MethodID = d[str(self.method_combo.GetStringSelection())]
+#         self.BeginDT = self._getTime(self.m_datePicker5, self.m_timePicker1)
+#         self.BeginDTUTC = self.spinUTCBegin.GetValue()
+#         self.AffiliationList = [i.affiliationID for i in self.affList.GetSelectedObjects()]
+#
+#         for i in self.affList.selected():
+#             if self.affList.IsChecked(i):
+#                 self.actionLead = i.affiliationID
+# # =======
         self.actionType = str(self.action_type_combo.GetStringSelection())
         self.methodID = d[str(self.method_combo.GetStringSelection())]
-        self.beginDT = self._getTime(self.m_datePicker5, self.m_timePicker1)
+        self.beginDT = self._getTime(self.dateBegin, self.m_timePicker1)
         self.beginDTUTC = self.spinUTCBegin.GetValue()
         self.affiliationList = [aff.AffiliationID for aff in self.affList.GetSelectedObjects()]
 
-        for i in self.affList.GetSelectedObjects():
-            if self.affList.IsChecked(i):
-                self.actionLead = i.affiliationID
+        for aff in self.affList.GetSelectedObjects():
+            if self.affList.IsChecked(aff):
+                self.actionLead = aff.AffiliationID
+
                 break
         if self.m_datePicker51.GetValue().IsValid():
             self.endDT = self._getTime(self.m_datePicker51, self.m_timePicker2)

src/wizard/controller/frmAddNewMethodPanel.py

@@ -10,7 +10,10 @@ def __init__(self, daddy, db, **kwargs):
             **kwargs)
         self.parent = daddy
         self.db = db
-
+        self.method = None# This is our method return value
+        self.methodCode = None
+        self.methodName = None
+        self.desc = None
         self.populateFields()
 
     def setTypeFilter(self, t):
@@ -42,8 +45,10 @@ def onOK(self, event):
                     OrganizationID=self.orgId,
                     MethodDescription=self.desc)
                 self.method = write.createMethod(meth)
+
             except Exception as e:
-                wx.MessageBox(e, 'Error saving method to database.')
+                wx.MessageBox(e, "Error saving method to database")
+
         event.Skip()
 
     def getFieldValues(self):

src/wizard/controller/frmAddNewUnitPanel.py

@@ -4,21 +4,25 @@
 from odm2api.ODM2.models import Units
 
 
+
 class AddNewUnitPanelController(AddNewUnitPanelView):
-    def __init__(self, daddy, db):
+    def __init__(self, daddy, db, type = None):
         super(AddNewUnitPanelController, self).__init__(daddy)
         self.parent = daddy
         self.db = db
 
         self.units = None
 
-        self.populateFields()
+        self.populateFields(type)
 
-    def populateFields(self):
+    def populateFields(self, type = None):
        read = self.db.getReadSession()
 
        # units = [i.Name for i in read.getCVUnitsTypes()]
        units = [i.Name for i in read.getCVs(type="Units Type")]
+       if type:
+           if type in units:
+               units = [type]
        self.m_comboBox13.AppendItems(units)
 
     def onOK(self, event):
@@ -33,12 +37,14 @@ def onOK(self, event):
                     UnitsAbbreviation=self.unitsAbr,
                     UnitsName=self.unitsName,
                     UnitsLink=self.unitsLink)
-                write.createUnit(unit)
-                self.parent.parent.list_ctrl.SetObjects(self.parent.parent.getSeriesData())
-                length = self.parent.parent.list_ctrl.GetItemCount.im_self.ItemCount
-                length = length - 1
-                self.parent.parent.list_ctrl.Focus(length)
-                self.parent.parent.list_ctrl.Select(length, 1)
+                self.unit=write.createUnit(unit)
+
+                if type(self.parent.parent).__name__ =="UnitSelectPanel":
+                    #this part is for selecting the new value in a list. should only be run when generated fromx
+                    self.parent.parent.list_ctrl.SetObjects(self.parent.parent.getSeriesData())
+                    length = self.parent.parent.list_ctrl.GetItemCount.im_self.ItemCount - 1
+                    self.parent.parent.list_ctrl.Focus(length)
+                    self.parent.parent.list_ctrl.Select(length, 1)
 
             except Exception as e:
                 print e