From 608b5e404e9ead070f7484c742280a71dafa2535 Mon Sep 17 00:00:00 2001
From: JOSHUA Robert Boverhof <jrboverhof@lbl.gov>
Date: Wed, 18 Jan 2023 19:13:27 -0800
Subject: [PATCH 01/25] Replace time.process_time() with time.time()

---
 foqus_lib/framework/sim/turbineConfiguration.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/foqus_lib/framework/sim/turbineConfiguration.py b/foqus_lib/framework/sim/turbineConfiguration.py
index 0289131da..e8835eeb8 100644
--- a/foqus_lib/framework/sim/turbineConfiguration.py
+++ b/foqus_lib/framework/sim/turbineConfiguration.py
@@ -453,10 +453,10 @@ def stopConsumer(self, name, maxWait=150):
             try:
                 is_stopping = _tcon.post_consumer_stop(url, auth, str(ci.cid))
                 _log.info("Stop Consumer {} Requested: {}".format(ci.cid, is_stopping))
-                swt = time.process_time()
+                swt = time.time()
                 while proc.poll() is None:
                     # wait for consumer to go down
-                    if time.process_time() - swt > maxWait:
+                    if time.time() - swt > maxWait:
                         _log.error(
                             "Error stopping consumer {} "
                             "Subprocess still running?".format(ci.cid)

From 4096db3d80c19379c38e0a8ba1cbe9c6b433c0b7 Mon Sep 17 00:00:00 2001
From: JOSHUA Robert Boverhof <jrboverhof@lbl.gov>
Date: Tue, 7 Mar 2023 14:03:00 -0800
Subject: [PATCH 02/25] Exception handling

---
 cloud/aws/README.md            | 10 ++++++++++
 foqus_lib/service/flowsheet.py | 10 +++++-----
 2 files changed, 15 insertions(+), 5 deletions(-)

diff --git a/cloud/aws/README.md b/cloud/aws/README.md
index 2bc0535f6..6d2e346fe 100644
--- a/cloud/aws/README.md
+++ b/cloud/aws/README.md
@@ -101,6 +101,16 @@ C:\tools\Anaconda3\envs\foqus;C:\tools\Anaconda3\envs\foqus\Library\mingw-w64\bi
 Open a powershell terminal and copy the path from above to set the machine system PATH environment variable to allow the `foqus_service.py` to run as a Windows Service.
 ```
 PS C:\Users\Administrator\Desktop> $P1 = "C:\ProgramData\Anaconda3\envs\foqus;C:\ProgramData\Anaconda3\envs\foqus\Library\mingw-w64\bin;C:\ProgramData\Anaconda3\envs\foqus\Library\usr\bin;C:\ProgramData\Anaconda3\envs\foqus\Library\bin;C:\ProgramData\Anaconda3\envs\foqus\Scripts;C:\ProgramData\Anaconda3\envs\foqus\bin;C:\ProgramData\Anaconda3\condabin;C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Windows\System32\WindowsPowerShell\v1.0;C:\Windows\System32\WindowsPowerShell\v1.0;C:\Windows\System32\WindowsPowerShell\v1.0;C:\Program Files\Amazon\cfn-bootstrap;C:\Program Files (x86)\Microsoft SQL Server\110\Tools\Binn;C:\Program Files\Microsoft SQL Server\110\Tools\Binn;C:\Program Files\Microsoft SQL Server\110\DTS\Binn;C:\Program Files (x86)\Common Files\AspenTech Shared"
+
+PS C:\Users\Administrator\Desktop> $P1 =
+"C:\tools\Anaconda3\envs\foqus;C:\tools\Anaconda3\envs\foqus\Library\mingw-w64\bin;C:\tools\Anaconda3\envs\foqus\Library\usr\bin;C:\tools\Anaconda3\envs\foqus\Library\bin;C:\tools\Anaconda3\envs\foqus\Scripts;C:\tools\Anaconda3\envs\foqus\bin;C:\tools\Anaconda3\envs\foqus\condabin"
+
+$P2 = "C:\tools\Anaconda3;C:\tools\Anaconda3\Library\mingw-w64\bin;C:\tools\Anaconda3\Library\usr\bin;C:\tools\Anaconda3\Library\bin;C:\tools\Anaconda3\Scripts;C:\tools\Anaconda3\bin;C:\tools\Anaconda3\condabin"
+
+
+$P3 = "C:\Windows\system32;C:\Windows;C:\Windows\System32\Wbem;C:\Windows\System32\WindowsPowerShell\v1.0;C:\Program Files\Amazon\cfn-bootstrap;C:\ProgramData\chocolatey\bin;C:\Users\Administrator\AppData\Local\Microsoft\WindowsApps"
+
+[Environment]::SetEnvironmentVariable('path', "$P1;$P2;$P3", 'Machine');
 ```
 Second, add several Anaconda3 Library paths.
 ```
diff --git a/foqus_lib/service/flowsheet.py b/foqus_lib/service/flowsheet.py
index 77b6e045a..fe2917db6 100644
--- a/foqus_lib/service/flowsheet.py
+++ b/foqus_lib/service/flowsheet.py
@@ -856,14 +856,14 @@ def _run(self):
             self._delete_sqs_job()
             try:
                 self.run_foqus(db, job_desc)
-            except foqusException as ex:
-                _log.exception("run_foqus foqusException")
-                self.increment_metric_job_finished(event="error.job.run")
-                self.close()
-                raise
             except Exception as ex:
                 _log.exception("run_foqus Exception")
                 self.increment_metric_job_finished(event="error.job.run")
+                msg = traceback.format_exc()
+                db.job_change_status(job_desc, "error", message=msg)
+                db.add_message(
+                    "job failed in setup: %r" % (ex), job_desc["Id"], exception=msg
+                )
                 self.close()
                 raise
 

From 7c0231084e372ae86ee4fbd9791553890145ecb5 Mon Sep 17 00:00:00 2001
From: Joshua Boverhof <jrboverhof@lbl.gov>
Date: Fri, 10 Mar 2023 00:32:46 +0000
Subject: [PATCH 03/25] Backend updates for user plugins

---
 foqus_lib/framework/plugins/pluginSearch.py |  10 +-
 foqus_lib/service/flowsheet.py              | 145 +++++++++++++++++++-
 2 files changed, 151 insertions(+), 4 deletions(-)

diff --git a/foqus_lib/framework/plugins/pluginSearch.py b/foqus_lib/framework/plugins/pluginSearch.py
index a0bd47216..d0774b0f4 100644
--- a/foqus_lib/framework/plugins/pluginSearch.py
+++ b/foqus_lib/framework/plugins/pluginSearch.py
@@ -30,6 +30,7 @@
 import logging
 import imp
 import re
+import traceback
 
 _log = logging.getLogger("foqus." + __name__)
 
@@ -44,12 +45,14 @@ def __init__(self, idString, pathList, charLimit=1150):
         self.pathList = pathList
         self.charLimit = charLimit
         self.plugins = {}
+        self.check_available_error_d = None
         self.importPlugins()
 
     def importPlugins(self):
         """
         check files in self.pathList to see if they are plugins
         """
+        self.check_available_error_d = dict()
         for p in self.pathList:
             if os.path.exists(p):
                 sys.path.append(p)
@@ -96,10 +99,15 @@ def importPlugins(self):
                             )
         # Now check that the plugins have what they need to be used
         for pkey, p in list(self.plugins.items()):
+            _log.debug("check plugin available: %s" %(pkey))
             try:
                 av = p.checkAvailable()
-            except:
+            except Exception as ex:
+                _log.exception("Exception Plugin checkAvailable")
+                self.check_available_error_d[pkey] = traceback.format_exc()
                 av = False
             if not av:
                 del self.plugins[pkey]
                 _log.info("Removing plugin, due to missing dependency: " + pkey)
+        
+
diff --git a/foqus_lib/service/flowsheet.py b/foqus_lib/service/flowsheet.py
index fe2917db6..c9d2bc7f1 100644
--- a/foqus_lib/service/flowsheet.py
+++ b/foqus_lib/service/flowsheet.py
@@ -31,6 +31,8 @@
 from foqus_lib.framework.graph.nodeVars import NodeVarListEx, NodeVarEx
 from foqus_lib.framework.foqusException.foqusException import *
 from foqus_lib.framework.graph.graph import Graph
+from foqus_lib.framework.plugins import pluginSearch
+from foqus_lib.framework.pymodel import pymodel
 from turbine.commands import turbine_simulation_script
 import logging
 import logging.config
@@ -106,7 +108,96 @@ def scrub_empty_string_values_for_dynamo(db):
         else:
             scrub_empty_string_values_for_dynamo(v)
 
+def _setup_foqus_user_plugin(dat, nkey, user_name, user_plugin_dir):
+    assert len(dat.flowsheet.nodes[nkey].turbApp) == 2, (
+        "DAT Flowsheet nodes turbApp is %s" % dat.flowsheet.nodes[nkey].turbApp
+    )
+    node = dat.flowsheet.nodes[nkey]
+    turb_app = node.turbApp[0]
+    model_name = node.modelName
+    assert turb_app is not None
+    turb_app = turb_app.lower()
+    assert turb_app == "foqus-user-plugin", (
+        'unknown foqus-user-plugin: "%s"' % turb_app
+    )
+    _log.debug(
+        'Turbine Node Key="%s", Model="%s", Application="%s"',
+        nkey,
+        model_name,
+        turb_app,
+    )
+    model_user_plugin_dir = os.path.join(user_plugin_dir, model_name)
+    try:
+        os.makedirs(model_user_plugin_dir)
+    except OSError as e:
+        if e.errno != errno.EEXIST:
+            raise
+            
+    s3 = boto3.client("s3", region_name=FOQUSAWSConfig.get_instance().get_region())
+    bucket_name = FOQUSAWSConfig.get_instance().get_simulation_bucket_name()
+    prefix = "%s/%s/" % (user_name, model_name)
+    l = s3.list_objects(Bucket=bucket_name, Prefix=prefix)
+    
+    assert (
+        "Contents" in l
+    ), 'Node %s failure: S3 Bucket %s is missing simulation files for "%s"' % (
+        nkey,
+        bucket_name,
+        prefix,
+    )
+    key_sinter_filename = None
+    key_plugin_filename = None
+    s3_key_list = [i["Key"] for i in l["Contents"]]
+    _log.debug("Node model %s staged-input files %s" % (model_name, s3_key_list))
+    
+    for k in s3_key_list:
+        if k.endswith("/foqus-user-plugin.json"):
+            key_sinter_filename = k
+        elif  k.endswith("/%s.py" %(model_name)):
+            assert key_plugin_filename is None, "detected multiple plugin files"
+            key_plugin_filename = k
+
+    assert (
+        key_sinter_filename is not None
+    ), "Flowsheet node=%s simulation=%s sinter configuration not in %s" % (
+        nkey,
+        model_name,
+        str(s3_key_list),
+    )
+    assert (
+        key_plugin_filename is not None
+    ), "Flowsheet node=%s simulation=%s model file not in %s" % (
+        nkey,
+        model_name,
+        str(s3_key_list),
+    )
+   
+    entry_list = [
+        i for i in l["Contents"] if i["Key"] != prefix and i["Key"].startswith(prefix)
+    ]
+    for entry in entry_list:
+        _log.debug("s3 staged input: %s", entry)
+        key = entry["Key"]
+        etag = entry.get("ETag", "").strip('"')
+        target_file_path = None
+        assert key.startswith(prefix)
+        if key == key_sinter_filename:
+            target_file_path = os.path.join(model_user_plugin_dir, key.split("/")[-1])
+            s3.download_file(bucket_name, key, target_file_path)
+        elif key == key_plugin_filename:
+            target_file_path = os.path.join(user_plugin_dir, key.split("/")[-1])
+            s3.download_file(bucket_name, key, target_file_path)
+        else:
+            args = [i for i in key[len(prefix) :].split("/") if i]
+            args.insert(0, model_user_plugin_dir)
+            target_file_path = os.path.join(*args)
+            s3.download_file(bucket_name, key, target_file_path)
 
+        _log.debug(
+            'model="%s" key="%s" staged-in file="%s"'
+            % (model_name, key, target_file_path)
+        )
+        
 def _setup_flowsheet_turbine_node(dat, nkey, user_name):
     """From s3 download all simulation files into AspenSinterComsumer cache directory '{working_directory\test\{simulation_guid}'.  If
     Simulation does not exist create one.  If Simulation does exist just s3 download all simulation files into the above cache directory.
@@ -840,9 +931,19 @@ def _run(self):
                     event="error.job.setup.NotImplementedError"
                 )
                 raise
+            except foqusException as ex:
+                # TODO:
+                _log.exception("setup foqus exception: job fails, continue running")
+                msg = traceback.format_exc()
+                db.job_change_status(job_desc, "error", message=msg)
+                db.add_message(
+                    "job failed in setup: %r" % (ex), job_desc["Id"], exception=msg
+                )
+                self.increment_metric_job_finished(event="error.job.setup")
+                self._delete_sqs_job()
             except Exception as ex:
                 # TODO:
-                _log.exception("setup foqus exception")
+                _log.exception("setup foqus exception:  fatal error")
                 msg = traceback.format_exc()
                 db.job_change_status(job_desc, "error", message=msg)
                 db.add_message(
@@ -1104,7 +1205,7 @@ def setup_foqus(self, db, user_name, job_desc):
         _log.debug("setup foqus")
         db.job_change_status(job_desc, "setup")
         configContent = db.get_configuration_file(simulation_name)
-        logging.getLogger("foqus." + __name__).info("Job {0} is submitted".format(jid))
+        logging.getLogger("foqus." + __name__).info("Job {0} is submitted".format(guid))
 
         reset = job_desc.get("Reset", False)
         assert type(reset) is bool, "Bad type for reset %s" % type(reset)
@@ -1121,7 +1222,9 @@ def setup_foqus(self, db, user_name, job_desc):
             _log.debug("Reset Flowsheet")
             self.close()
             self._dat = dat = Session(useCurrentWorkingDir=True)
+            _log.debug("New Session Created: next load")
             dat.load(sfile, stopConsumers=True)
+            _log.debug("Session load finished")
             self._simulation_name = simulation_name
         else:
             _log.debug("No Reset Flowsheet")
@@ -1131,10 +1234,42 @@ def setup_foqus(self, db, user_name, job_desc):
         _log.debug("Process Flowsheet nodes")
         count_turb_apps = 0
         nkey = None
+        foqus_user_plugins = []
         for i in dat.flowsheet.nodes:
-            if dat.flowsheet.nodes[i].turbApp is not None:
+            node = dat.flowsheet.nodes[i]
+            if node.turbApp is not None:
+                turb_app = node.turbApp[0]
+                turb_app = turb_app.lower()
+                if turb_app == 'foqus-user-plugin':
+                    foqus_user_plugins.append(i)
+                    continue
                 nkey = i
                 count_turb_apps += 1
+                
+        user_plugin_dir = os.path.join(WORKING_DIRECTORY, "user_plugins")
+        for nkey in foqus_user_plugins:
+            _setup_foqus_user_plugin(dat, nkey, user_name=user_name, 
+                                     user_plugin_dir=user_plugin_dir)
+        
+        pymodels = pluginSearch.plugins(
+            idString="#\s?FOQUS_PYMODEL_PLUGIN",
+            pathList=[
+                user_plugin_dir,
+                os.path.dirname(pymodel.__file__),
+            ],
+        )
+        dat.pymodels = pymodels
+        dat.flowsheet.pymodels = pymodels
+        # Check if Plugin Available
+        for nkey in foqus_user_plugins:
+            #node = dat.flowsheet.nodes[i]
+            if nkey not in pymodels.plugins:
+                for key in pymodels.plugins: _log.debug("PLUGIN: %s" %(key))
+                if nkey in pymodels.check_available_error_d:
+                    msg = pymodels.check_available_error_d.get(nkey)
+                    raise foqusException("FOQUS User Plugin %s:  Failed to load on check available: %s" %(nkey,msg))
+                raise foqusException("FOQUS User Plugin %s:  Failed to load" %(nkey))
+        
         if count_turb_apps > 1:
             self.close()
             raise RuntimeError(
@@ -1148,6 +1283,10 @@ def setup_foqus(self, db, user_name, job_desc):
                 db.job_change_status(
                     job_desc, "error", message="Error in job setup: %s" % ex
                 )
+                
+
+        
+        
         return dat
 
     def run_foqus(self, db, job_desc):

From 9382917281107ff16d667e66106ce49d6a8f9597 Mon Sep 17 00:00:00 2001
From: Joshua Boverhof <jrboverhof@lbl.gov>
Date: Fri, 10 Mar 2023 01:20:23 +0000
Subject: [PATCH 04/25] If setttings bad throw runtime error

---
 foqus_lib/framework/sim/turbineConfiguration.py |  2 ++
 foqus_lib/service/flowsheet.py                  | 15 +--------------
 2 files changed, 3 insertions(+), 14 deletions(-)

diff --git a/foqus_lib/framework/sim/turbineConfiguration.py b/foqus_lib/framework/sim/turbineConfiguration.py
index e8835eeb8..4869277dd 100644
--- a/foqus_lib/framework/sim/turbineConfiguration.py
+++ b/foqus_lib/framework/sim/turbineConfiguration.py
@@ -291,6 +291,7 @@ def updateSettings(self, altConfig=None):
             self.reloadTurbine()
         except:
             _log.exception("Could not load FOQUS settings.")
+            raise RuntimeError("Failed to load FOQUS Settings")
 
     def makeCopy(self):
         """
@@ -577,6 +578,7 @@ def readConfig(self, address=True, logging=False):
         configuration file.
         """
         path = self.getFile()
+        _log.debug('turbine configuration="%s"', path)
         config = configparser.ConfigParser()
         config.optionxform = str  # makes options case sensitive
         try:
diff --git a/foqus_lib/service/flowsheet.py b/foqus_lib/service/flowsheet.py
index c9d2bc7f1..52c87a691 100644
--- a/foqus_lib/service/flowsheet.py
+++ b/foqus_lib/service/flowsheet.py
@@ -920,17 +920,6 @@ def _run(self):
                 )
                 self._delete_sqs_job()
                 raise
-                _log.exception("setup foqus URLError")
-                msg = traceback.format_exc()
-                db.job_change_status(job_desc, "error", message=msg)
-                db.add_message(
-                    "job failed in setup URLError", job_desc["Id"], exception=msg
-                )
-                self._delete_sqs_job()
-                self.increment_metric_job_finished(
-                    event="error.job.setup.NotImplementedError"
-                )
-                raise
             except foqusException as ex:
                 # TODO:
                 _log.exception("setup foqus exception: job fails, continue running")
@@ -941,6 +930,7 @@ def _run(self):
                 )
                 self.increment_metric_job_finished(event="error.job.setup")
                 self._delete_sqs_job()
+                continue
             except Exception as ex:
                 # TODO:
                 _log.exception("setup foqus exception:  fatal error")
@@ -1284,9 +1274,6 @@ def setup_foqus(self, db, user_name, job_desc):
                     job_desc, "error", message="Error in job setup: %s" % ex
                 )
                 
-
-        
-        
         return dat
 
     def run_foqus(self, db, job_desc):

From 176aa5d95f15d8a18a73ccd9b311393debdcef69 Mon Sep 17 00:00:00 2001
From: Joshua Boverhof <jrboverhof@lbl.gov>
Date: Sat, 11 Mar 2023 00:19:43 +0000
Subject: [PATCH 05/25] code improvements

---
 foqus_lib/framework/graph/graph.py | 42 +++++++++++++++++++++---------
 foqus_lib/service/flowsheet.py     | 22 ++++++++--------
 2 files changed, 41 insertions(+), 23 deletions(-)

diff --git a/foqus_lib/framework/graph/graph.py b/foqus_lib/framework/graph/graph.py
index 65385875a..bd5cd1a1a 100644
--- a/foqus_lib/framework/graph/graph.py
+++ b/foqus_lib/framework/graph/graph.py
@@ -870,11 +870,13 @@ def run(self):
             try:
                 # run a single simulation or a list of simulations
                 if not self.runList:
+                    _log.debug("run: solve single simulation")
                     self.solve()
                     with self.resLock:
                         self.res[0] = self.saveValues()
                         self.res_fin[0] = self.errorStat
                 else:
+                    _log.debug("run: solve list simulations")
                     self.solveListVal(self.runList)
             except Exception as e:
                 self.setErrorCode(19)
@@ -978,14 +980,14 @@ def solve(self):
         more than one tear they are converged simultaneously.
         I know that is probably not best but it is easy for now
         """
-        for key, node in self.nodes.items():
-            if node.modelType == nodeModelTypes.MODEL_DMF_LITE:
-                _log.debug("DMF will sync node {}".format(key))
-                node.synced = False
-            elif node.modelType == nodeModelTypes.MODEL_DMF_SERV:
-                node.synced = False
-            else:
-                pass  # Doesn't matter synced is a DMF thing
+        # for key, node in self.nodes.items():
+        #     if node.modelType == nodeModelTypes.MODEL_DMF_LITE:
+        #         _log.debug("DMF will sync node {}".format(key))
+        #         node.synced = False
+        #     elif node.modelType == nodeModelTypes.MODEL_DMF_SERV:
+        #         node.synced = False
+        #     else:
+        #         pass  # Doesn't matter synced is a DMF thing
         tstart = time.time()
         self.setAsNotRun()
         self.setErrorCode(0)
@@ -997,6 +999,7 @@ def solve(self):
         # the graph but only evaluates a single node.
         if self.onlySingleNode in list(self.nodes.keys()):
             name = self.onlySingleNode
+            _log.debug("solve: onlySingleNode %s", name)
             if self.runNode(name) != 0:
                 self.setErrorCode(1)
             else:
@@ -1007,8 +1010,10 @@ def solve(self):
             self.solTime = time.time() - tstart
             return self.solTime
         elif self.onlySingleNode != None:
+            _log.error("solve: onlySingleNode not in node keys")
             self.setErrorCode(4)
             return
+        
         # Take the pre and post solve nodes out of the calculation order
         fs_sub = []
         for nkey, node in self.nodes.items():
@@ -1023,14 +1028,17 @@ def solve(self):
                 fs_sub.append(nkey)
         # Run presove nodes in order
         for nkey in self.pre_solve_nodes:
+            _log.debug("solve: run pre-solve node %s", nkey)
             node = self.nodes[nkey]
             node.runCalc()
             if node.calcError != 0:
+                _log.debug("solve: pre-solve calcError %d", node.calcError)
                 self.setErrorCode(16)
                 self.solTime = time.time() - tstart
                 return self.solTime
         # Run graph, order is based on tree with tears removed
         order = self.calculationOrder(subNodes=fs_sub)
+        _log.debug('solve: runGraph')
         self.runGraph(order)
         # Check if there are any tears if no tears we are done
         solveTear = False
@@ -1039,11 +1047,13 @@ def solve(self):
                 solveTear = True
                 break
         if not solveTear:
-            # no tears flowsheet done run post solve nodes
+            _log.debug('solve: no tears flowsheet done run post solve nodes')
             for nkey in self.post_solve_nodes:
+                _log.debug("solve: run post-solve node %s", nkey)
                 node = self.nodes[nkey]
                 node.runCalc()
                 if node.calcError != 0:
+                    _log.debug("solve: post-solve calcError %d", node.calcError)
                     self.setErrorCode(17)
                     self.solTime = time.time() - tstart
                     return self.solTime
@@ -1089,15 +1099,18 @@ def solve(self):
                     self.setErrorCode(errCode)
                     self.solTime = time.time() - tstart
                     return self.solTime
-        # No errors so far and flowsheet converged, run post nodes
+        _log.debug('solve: No errors so far and flowsheet converged, run post nodes')
         for nkey in self.post_solve_nodes:
+            _log.debug("solve: run post-solve node %s", nkey)
             node = self.nodes[nkey]
             node.runCalc()
             if node.calcError != 0:
+                _log.debug("solve: post-solve calcError %d", node.calcError)
                 self.setErrorCode(17)
                 self.solTime = time.time() - tstart
                 return self.solTime
-        # pre, flowsheet, post all done return success
+        
+        _log.debug('solve: pre, flowsheet, post all done return success')
         self.setErrorCode(0)
         self.solTime = time.time() - tstart
         return self.solTime
@@ -1340,13 +1353,18 @@ def runGraph(self, order):
         for namelst in order:
             for name in namelst:
                 if self.stop.isSet():
+                    _log.error('runGraph(%s): error 20 stop is set', name)
                     self.setErrorCode(20)
                     return
-                elif self.runNode(name) != 0:
+                calcError = self.runNode(name)
+                if calcError != 0:
+                    _log.error('runGraph(%s): calcError=%d', name, calcError)
                     self.setErrorCode(1)
                     return
+                
                 for e in self.edges:
                     if e.start == name and e.tear == False and e.active == True:
+                        _log.debug('runGraph(%s): transfer edge', name)
                         e.transferInformation(self)
 
     def runNode(self, name):
diff --git a/foqus_lib/service/flowsheet.py b/foqus_lib/service/flowsheet.py
index 52c87a691..ef056c291 100644
--- a/foqus_lib/service/flowsheet.py
+++ b/foqus_lib/service/flowsheet.py
@@ -1223,7 +1223,7 @@ def setup_foqus(self, db, user_name, job_desc):
         dat.loadFlowsheetValues(vfile)
         _log.debug("Process Flowsheet nodes")
         count_turb_apps = 0
-        nkey = None
+        turb_app_nkey = None
         foqus_user_plugins = []
         for i in dat.flowsheet.nodes:
             node = dat.flowsheet.nodes[i]
@@ -1233,12 +1233,12 @@ def setup_foqus(self, db, user_name, job_desc):
                 if turb_app == 'foqus-user-plugin':
                     foqus_user_plugins.append(i)
                     continue
-                nkey = i
+                turb_app_nkey = i
                 count_turb_apps += 1
                 
         user_plugin_dir = os.path.join(WORKING_DIRECTORY, "user_plugins")
-        for nkey in foqus_user_plugins:
-            _setup_foqus_user_plugin(dat, nkey, user_name=user_name, 
+        for i in foqus_user_plugins:
+            _setup_foqus_user_plugin(dat, i, user_name=user_name, 
                                      user_plugin_dir=user_plugin_dir)
         
         pymodels = pluginSearch.plugins(
@@ -1251,14 +1251,14 @@ def setup_foqus(self, db, user_name, job_desc):
         dat.pymodels = pymodels
         dat.flowsheet.pymodels = pymodels
         # Check if Plugin Available
-        for nkey in foqus_user_plugins:
+        for i in foqus_user_plugins:
             #node = dat.flowsheet.nodes[i]
-            if nkey not in pymodels.plugins:
+            if i not in pymodels.plugins:
                 for key in pymodels.plugins: _log.debug("PLUGIN: %s" %(key))
-                if nkey in pymodels.check_available_error_d:
-                    msg = pymodels.check_available_error_d.get(nkey)
-                    raise foqusException("FOQUS User Plugin %s:  Failed to load on check available: %s" %(nkey,msg))
-                raise foqusException("FOQUS User Plugin %s:  Failed to load" %(nkey))
+                if i in pymodels.check_available_error_d:
+                    msg = pymodels.check_available_error_d.get(i)
+                    raise foqusException("FOQUS User Plugin %s:  Failed to load on check available: %s" %(i,msg))
+                raise foqusException("FOQUS User Plugin %s:  Failed to load" %(i))
         
         if count_turb_apps > 1:
             self.close()
@@ -1267,7 +1267,7 @@ def setup_foqus(self, db, user_name, job_desc):
             )
         if count_turb_apps:
             try:
-                _setup_flowsheet_turbine_node(dat, nkey, user_name=user_name)
+                _setup_flowsheet_turbine_node(dat, turb_app_nkey, user_name=user_name)
             except AssertionError as ex:
                 _log.error("Job Setup Failure: %s", str(ex))
                 db.job_change_status(

From 14c33e7a93daa1099262ec23f5dcbd07eabb3697 Mon Sep 17 00:00:00 2001
From: JOSHUA Robert Boverhof <jrboverhof@lbl.gov>
Date: Mon, 13 Mar 2023 09:12:05 -0700
Subject: [PATCH 06/25] Ran Black

---
 foqus_lib/framework/graph/graph.py          | 20 ++++-----
 foqus_lib/framework/plugins/pluginSearch.py |  4 +-
 foqus_lib/service/flowsheet.py              | 45 ++++++++++++---------
 3 files changed, 36 insertions(+), 33 deletions(-)

diff --git a/foqus_lib/framework/graph/graph.py b/foqus_lib/framework/graph/graph.py
index bd5cd1a1a..7157a06eb 100644
--- a/foqus_lib/framework/graph/graph.py
+++ b/foqus_lib/framework/graph/graph.py
@@ -1013,7 +1013,7 @@ def solve(self):
             _log.error("solve: onlySingleNode not in node keys")
             self.setErrorCode(4)
             return
-        
+
         # Take the pre and post solve nodes out of the calculation order
         fs_sub = []
         for nkey, node in self.nodes.items():
@@ -1038,7 +1038,7 @@ def solve(self):
                 return self.solTime
         # Run graph, order is based on tree with tears removed
         order = self.calculationOrder(subNodes=fs_sub)
-        _log.debug('solve: runGraph')
+        _log.debug("solve: runGraph")
         self.runGraph(order)
         # Check if there are any tears if no tears we are done
         solveTear = False
@@ -1047,7 +1047,7 @@ def solve(self):
                 solveTear = True
                 break
         if not solveTear:
-            _log.debug('solve: no tears flowsheet done run post solve nodes')
+            _log.debug("solve: no tears flowsheet done run post solve nodes")
             for nkey in self.post_solve_nodes:
                 _log.debug("solve: run post-solve node %s", nkey)
                 node = self.nodes[nkey]
@@ -1099,7 +1099,7 @@ def solve(self):
                     self.setErrorCode(errCode)
                     self.solTime = time.time() - tstart
                     return self.solTime
-        _log.debug('solve: No errors so far and flowsheet converged, run post nodes')
+        _log.debug("solve: No errors so far and flowsheet converged, run post nodes")
         for nkey in self.post_solve_nodes:
             _log.debug("solve: run post-solve node %s", nkey)
             node = self.nodes[nkey]
@@ -1109,8 +1109,8 @@ def solve(self):
                 self.setErrorCode(17)
                 self.solTime = time.time() - tstart
                 return self.solTime
-        
-        _log.debug('solve: pre, flowsheet, post all done return success')
+
+        _log.debug("solve: pre, flowsheet, post all done return success")
         self.setErrorCode(0)
         self.solTime = time.time() - tstart
         return self.solTime
@@ -1353,18 +1353,18 @@ def runGraph(self, order):
         for namelst in order:
             for name in namelst:
                 if self.stop.isSet():
-                    _log.error('runGraph(%s): error 20 stop is set', name)
+                    _log.error("runGraph(%s): error 20 stop is set", name)
                     self.setErrorCode(20)
                     return
                 calcError = self.runNode(name)
                 if calcError != 0:
-                    _log.error('runGraph(%s): calcError=%d', name, calcError)
+                    _log.error("runGraph(%s): calcError=%d", name, calcError)
                     self.setErrorCode(1)
                     return
-                
+
                 for e in self.edges:
                     if e.start == name and e.tear == False and e.active == True:
-                        _log.debug('runGraph(%s): transfer edge', name)
+                        _log.debug("runGraph(%s): transfer edge", name)
                         e.transferInformation(self)
 
     def runNode(self, name):
diff --git a/foqus_lib/framework/plugins/pluginSearch.py b/foqus_lib/framework/plugins/pluginSearch.py
index d0774b0f4..8e38fcbee 100644
--- a/foqus_lib/framework/plugins/pluginSearch.py
+++ b/foqus_lib/framework/plugins/pluginSearch.py
@@ -99,7 +99,7 @@ def importPlugins(self):
                             )
         # Now check that the plugins have what they need to be used
         for pkey, p in list(self.plugins.items()):
-            _log.debug("check plugin available: %s" %(pkey))
+            _log.debug("check plugin available: %s" % (pkey))
             try:
                 av = p.checkAvailable()
             except Exception as ex:
@@ -109,5 +109,3 @@ def importPlugins(self):
             if not av:
                 del self.plugins[pkey]
                 _log.info("Removing plugin, due to missing dependency: " + pkey)
-        
-
diff --git a/foqus_lib/service/flowsheet.py b/foqus_lib/service/flowsheet.py
index ef056c291..8284ab901 100644
--- a/foqus_lib/service/flowsheet.py
+++ b/foqus_lib/service/flowsheet.py
@@ -108,6 +108,7 @@ def scrub_empty_string_values_for_dynamo(db):
         else:
             scrub_empty_string_values_for_dynamo(v)
 
+
 def _setup_foqus_user_plugin(dat, nkey, user_name, user_plugin_dir):
     assert len(dat.flowsheet.nodes[nkey].turbApp) == 2, (
         "DAT Flowsheet nodes turbApp is %s" % dat.flowsheet.nodes[nkey].turbApp
@@ -117,9 +118,7 @@ def _setup_foqus_user_plugin(dat, nkey, user_name, user_plugin_dir):
     model_name = node.modelName
     assert turb_app is not None
     turb_app = turb_app.lower()
-    assert turb_app == "foqus-user-plugin", (
-        'unknown foqus-user-plugin: "%s"' % turb_app
-    )
+    assert turb_app == "foqus-user-plugin", 'unknown foqus-user-plugin: "%s"' % turb_app
     _log.debug(
         'Turbine Node Key="%s", Model="%s", Application="%s"',
         nkey,
@@ -132,12 +131,12 @@ def _setup_foqus_user_plugin(dat, nkey, user_name, user_plugin_dir):
     except OSError as e:
         if e.errno != errno.EEXIST:
             raise
-            
+
     s3 = boto3.client("s3", region_name=FOQUSAWSConfig.get_instance().get_region())
     bucket_name = FOQUSAWSConfig.get_instance().get_simulation_bucket_name()
     prefix = "%s/%s/" % (user_name, model_name)
     l = s3.list_objects(Bucket=bucket_name, Prefix=prefix)
-    
+
     assert (
         "Contents" in l
     ), 'Node %s failure: S3 Bucket %s is missing simulation files for "%s"' % (
@@ -149,11 +148,11 @@ def _setup_foqus_user_plugin(dat, nkey, user_name, user_plugin_dir):
     key_plugin_filename = None
     s3_key_list = [i["Key"] for i in l["Contents"]]
     _log.debug("Node model %s staged-input files %s" % (model_name, s3_key_list))
-    
+
     for k in s3_key_list:
         if k.endswith("/foqus-user-plugin.json"):
             key_sinter_filename = k
-        elif  k.endswith("/%s.py" %(model_name)):
+        elif k.endswith("/%s.py" % (model_name)):
             assert key_plugin_filename is None, "detected multiple plugin files"
             key_plugin_filename = k
 
@@ -171,7 +170,7 @@ def _setup_foqus_user_plugin(dat, nkey, user_name, user_plugin_dir):
         model_name,
         str(s3_key_list),
     )
-   
+
     entry_list = [
         i for i in l["Contents"] if i["Key"] != prefix and i["Key"].startswith(prefix)
     ]
@@ -197,7 +196,8 @@ def _setup_foqus_user_plugin(dat, nkey, user_name, user_plugin_dir):
             'model="%s" key="%s" staged-in file="%s"'
             % (model_name, key, target_file_path)
         )
-        
+
+
 def _setup_flowsheet_turbine_node(dat, nkey, user_name):
     """From s3 download all simulation files into AspenSinterComsumer cache directory '{working_directory\test\{simulation_guid}'.  If
     Simulation does not exist create one.  If Simulation does exist just s3 download all simulation files into the above cache directory.
@@ -1230,17 +1230,18 @@ def setup_foqus(self, db, user_name, job_desc):
             if node.turbApp is not None:
                 turb_app = node.turbApp[0]
                 turb_app = turb_app.lower()
-                if turb_app == 'foqus-user-plugin':
+                if turb_app == "foqus-user-plugin":
                     foqus_user_plugins.append(i)
                     continue
                 turb_app_nkey = i
                 count_turb_apps += 1
-                
+
         user_plugin_dir = os.path.join(WORKING_DIRECTORY, "user_plugins")
         for i in foqus_user_plugins:
-            _setup_foqus_user_plugin(dat, i, user_name=user_name, 
-                                     user_plugin_dir=user_plugin_dir)
-        
+            _setup_foqus_user_plugin(
+                dat, i, user_name=user_name, user_plugin_dir=user_plugin_dir
+            )
+
         pymodels = pluginSearch.plugins(
             idString="#\s?FOQUS_PYMODEL_PLUGIN",
             pathList=[
@@ -1252,14 +1253,18 @@ def setup_foqus(self, db, user_name, job_desc):
         dat.flowsheet.pymodels = pymodels
         # Check if Plugin Available
         for i in foqus_user_plugins:
-            #node = dat.flowsheet.nodes[i]
+            # node = dat.flowsheet.nodes[i]
             if i not in pymodels.plugins:
-                for key in pymodels.plugins: _log.debug("PLUGIN: %s" %(key))
+                for key in pymodels.plugins:
+                    _log.debug("PLUGIN: %s" % (key))
                 if i in pymodels.check_available_error_d:
                     msg = pymodels.check_available_error_d.get(i)
-                    raise foqusException("FOQUS User Plugin %s:  Failed to load on check available: %s" %(i,msg))
-                raise foqusException("FOQUS User Plugin %s:  Failed to load" %(i))
-        
+                    raise foqusException(
+                        "FOQUS User Plugin %s:  Failed to load on check available: %s"
+                        % (i, msg)
+                    )
+                raise foqusException("FOQUS User Plugin %s:  Failed to load" % (i))
+
         if count_turb_apps > 1:
             self.close()
             raise RuntimeError(
@@ -1273,7 +1278,7 @@ def setup_foqus(self, db, user_name, job_desc):
                 db.job_change_status(
                     job_desc, "error", message="Error in job setup: %s" % ex
                 )
-                
+
         return dat
 
     def run_foqus(self, db, job_desc):

From 70fdb66140e87269008e2b38e7808cb656eeb299 Mon Sep 17 00:00:00 2001
From: unknown <jrboverhof@lbl.gov>
Date: Thu, 30 May 2024 21:38:10 +0000
Subject: [PATCH 07/25] Log to CloudWatch using watchtower
 /user/{username}/ec2/{instance_id}

---
 foqus_lib/service/flowsheet.py | 29 +++++++++++++++++++++++++----
 1 file changed, 25 insertions(+), 4 deletions(-)

diff --git a/foqus_lib/service/flowsheet.py b/foqus_lib/service/flowsheet.py
index a02b7989a..4535a42c4 100644
--- a/foqus_lib/service/flowsheet.py
+++ b/foqus_lib/service/flowsheet.py
@@ -20,6 +20,7 @@
 """
 import errno
 import json
+import yaml
 import logging
 import logging.config
 import optparse
@@ -38,8 +39,10 @@
 
 import boto3
 import botocore.exceptions
-from turbine.commands import turbine_simulation_script
+import watchtower
+import functools
 
+from turbine.commands import turbine_simulation_script
 from foqus_lib.framework.foqusException.foqusException import *
 from foqus_lib.framework.graph.graph import Graph
 from foqus_lib.framework.graph.nodeVars import NodeVarEx, NodeVarListEx
@@ -51,11 +54,30 @@
 WORKING_DIRECTORY = os.path.abspath(
     os.environ.get("FOQUS_SERVICE_WORKING_DIR", "\\ProgramData\\foqus_service")
 )
-DEBUG = False
+
 CURRENT_JOB_DIR = None
 _log = logging.getLogger("foqus.foqus_lib.service.flowsheet")
 
 
+class FoqusCloudWatchLogHandler(watchtower.CloudWatchLogHandler):
+    @functools.lru_cache(maxsize=0)
+    def _get_machine_name(self):
+       return FOQUSAWSConfig.get_instance().instance_id
+    @functools.lru_cache(maxsize=0)
+    def _get_user(self):
+       return FOQUSAWSConfig.get_instance().get_user()
+    def _get_stream_name(self, message):
+        return '/user/%s/ec2/%s' %(self._get_user(), self._get_machine_name())
+
+def _applyLogSettings(self_gs):
+    # Short circuit FOQUS logging setup
+    region_name = FOQUSAWSConfig.get_instance().get_region()
+    os.environ['AWS_DEFAULT_REGION'] = region_name
+    with open(os.path.join(WORKING_DIRECTORY, "logging.yaml")) as log_config:
+        config_yml = log_config.read()
+        config_dict = yaml.safe_load(config_yml)
+        logging.config.dictConfig(config_dict)
+
 def _set_working_dir(wdir):
     global _log, WORKING_DIRECTORY
     WORKING_DIRECTORY = wdir
@@ -69,7 +91,6 @@ def _set_working_dir(wdir):
     FoqusSettings().applyLogSettings()
 
     _log = logging.getLogger("foqus.foqus_lib.service.flowsheet")
-    _log.setLevel(logging.DEBUG)
     _log.info("Working Directory: %s", WORKING_DIRECTORY)
     logging.getLogger("boto3").setLevel(logging.ERROR)
     logging.getLogger("botocore").setLevel(logging.ERROR)
@@ -82,6 +103,7 @@ def _get_user_config_location(*args, **kw):
 
 
 FoqusSettings.getUserConfigLocation = _get_user_config_location
+FoqusSettings.applyLogSettings = _applyLogSettings
 
 
 def getfilenames(jid):
@@ -463,7 +485,6 @@ def __init__(self):
     def _get(self, key):
         v = self._d.get(key)
         assert v, "UserData/MetaData Missing Key(%s): %s" % (key, str(self._d))
-        _log.debug("FOQUSAWSConfig._get: %s = %s" % (key, v))
         return v
 
     def get_region(self):

From 830a6c2fce42539fcf890eb91fc4338b1e55c577 Mon Sep 17 00:00:00 2001
From: JOSHUA Robert Boverhof <jrboverhof@lbl.gov>
Date: Thu, 30 May 2024 14:57:19 -0700
Subject: [PATCH 08/25] Log out to CloudWatch using watchtower

---
 foqus_lib/service/flowsheet.py | 29 +++++++++++++++++++++++++----
 1 file changed, 25 insertions(+), 4 deletions(-)

diff --git a/foqus_lib/service/flowsheet.py b/foqus_lib/service/flowsheet.py
index a02b7989a..4535a42c4 100644
--- a/foqus_lib/service/flowsheet.py
+++ b/foqus_lib/service/flowsheet.py
@@ -20,6 +20,7 @@
 """
 import errno
 import json
+import yaml
 import logging
 import logging.config
 import optparse
@@ -38,8 +39,10 @@
 
 import boto3
 import botocore.exceptions
-from turbine.commands import turbine_simulation_script
+import watchtower
+import functools
 
+from turbine.commands import turbine_simulation_script
 from foqus_lib.framework.foqusException.foqusException import *
 from foqus_lib.framework.graph.graph import Graph
 from foqus_lib.framework.graph.nodeVars import NodeVarEx, NodeVarListEx
@@ -51,11 +54,30 @@
 WORKING_DIRECTORY = os.path.abspath(
     os.environ.get("FOQUS_SERVICE_WORKING_DIR", "\\ProgramData\\foqus_service")
 )
-DEBUG = False
+
 CURRENT_JOB_DIR = None
 _log = logging.getLogger("foqus.foqus_lib.service.flowsheet")
 
 
+class FoqusCloudWatchLogHandler(watchtower.CloudWatchLogHandler):
+    @functools.lru_cache(maxsize=0)
+    def _get_machine_name(self):
+       return FOQUSAWSConfig.get_instance().instance_id
+    @functools.lru_cache(maxsize=0)
+    def _get_user(self):
+       return FOQUSAWSConfig.get_instance().get_user()
+    def _get_stream_name(self, message):
+        return '/user/%s/ec2/%s' %(self._get_user(), self._get_machine_name())
+
+def _applyLogSettings(self_gs):
+    # Short circuit FOQUS logging setup
+    region_name = FOQUSAWSConfig.get_instance().get_region()
+    os.environ['AWS_DEFAULT_REGION'] = region_name
+    with open(os.path.join(WORKING_DIRECTORY, "logging.yaml")) as log_config:
+        config_yml = log_config.read()
+        config_dict = yaml.safe_load(config_yml)
+        logging.config.dictConfig(config_dict)
+
 def _set_working_dir(wdir):
     global _log, WORKING_DIRECTORY
     WORKING_DIRECTORY = wdir
@@ -69,7 +91,6 @@ def _set_working_dir(wdir):
     FoqusSettings().applyLogSettings()
 
     _log = logging.getLogger("foqus.foqus_lib.service.flowsheet")
-    _log.setLevel(logging.DEBUG)
     _log.info("Working Directory: %s", WORKING_DIRECTORY)
     logging.getLogger("boto3").setLevel(logging.ERROR)
     logging.getLogger("botocore").setLevel(logging.ERROR)
@@ -82,6 +103,7 @@ def _get_user_config_location(*args, **kw):
 
 
 FoqusSettings.getUserConfigLocation = _get_user_config_location
+FoqusSettings.applyLogSettings = _applyLogSettings
 
 
 def getfilenames(jid):
@@ -463,7 +485,6 @@ def __init__(self):
     def _get(self, key):
         v = self._d.get(key)
         assert v, "UserData/MetaData Missing Key(%s): %s" % (key, str(self._d))
-        _log.debug("FOQUSAWSConfig._get: %s = %s" % (key, v))
         return v
 
     def get_region(self):

From 8bb96d10212fae3312d91adbccb1694d4ce0372a Mon Sep 17 00:00:00 2001
From: JOSHUA Robert Boverhof <jrboverhof@lbl.gov>
Date: Thu, 30 May 2024 14:58:33 -0700
Subject: [PATCH 09/25] Add Watchtower to installs

---
 setup.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/setup.py b/setup.py
index fa38bdcf4..a4fbb7c34 100644
--- a/setup.py
+++ b/setup.py
@@ -108,6 +108,7 @@
         "scipy",
         "tqdm",
         "TurbineClient ~= 4.0, >= 4.0.3",
+        "watchtower",
         "winshell; sys_platform == 'win32'",
         "websocket_client>=1.1.0",
     ],

From b1584603f5cd109cf0701b470f838d3fad9d1ba6 Mon Sep 17 00:00:00 2001
From: JOSHUA Robert Boverhof <jrboverhof@lbl.gov>
Date: Thu, 30 May 2024 15:01:35 -0700
Subject: [PATCH 10/25] logging file for cloudwatch aws

---
 cloud/aws/logging.yaml | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)
 create mode 100644 cloud/aws/logging.yaml

diff --git a/cloud/aws/logging.yaml b/cloud/aws/logging.yaml
new file mode 100644
index 000000000..d3f8571f6
--- /dev/null
+++ b/cloud/aws/logging.yaml
@@ -0,0 +1,32 @@
+# Default AWS Config
+version: 1
+disable_existing_loggers: False
+handlers:
+  console:
+    class: logging.StreamHandler
+    level: DEBUG
+    stream: ext://sys.stdout
+  logfile:
+    class: logging.handlers.RotatingFileHandler
+    level: DEBUG
+    filename: watchtower.log
+    maxBytes: 1000000
+    backupCount: 3
+  watchtower:
+    #class: watchtower.CloudWatchLogHandler
+    class: foqus_lib.service.flowsheet.FoqusCloudWatchLogHandler
+    level: DEBUG
+    log_group_name: foqus-cloud-service
+    #log_stream_name: "{machine_name}-{strftime:%y-%m-%d}"
+    log_stream_name: "{machine_name}"
+    send_interval: 10
+    create_log_group: True
+root:
+  level: DEBUG
+  propagate: True
+  handlers: [console, logfile, watchtower]
+loggers:
+  botocore:
+    level: INFO
+  urllib3:
+    level: INFO

From 970d82d9a8c6fb46e003bfd3e9632fa69fe6ee84 Mon Sep 17 00:00:00 2001
From: JOSHUA Robert Boverhof <jrboverhof@lbl.gov>
Date: Wed, 26 Jun 2024 16:55:04 -0700
Subject: [PATCH 11/25] Use core for simsinter and turbineLite home paths

---
 foqus_lib/framework/session/session.py          | 5 +++--
 foqus_lib/framework/sim/turbineConfiguration.py | 3 ++-
 foqus_lib/framework/sim/turbineLiteDB.py        | 4 ++--
 3 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/foqus_lib/framework/session/session.py b/foqus_lib/framework/session/session.py
index ce6176c19..8282faedb 100644
--- a/foqus_lib/framework/session/session.py
+++ b/foqus_lib/framework/session/session.py
@@ -28,6 +28,7 @@
 import sys
 import uuid
 
+from foqus_lib import core
 import foqus_lib.framework.optimizer.problem as oprob
 from foqus_lib.framework.graph.graph import *
 from foqus_lib.framework.graph.node import nodeModelTypes
@@ -770,7 +771,7 @@ def __init__(self):
         self.working_dir_override = False
         self.working_dir = ""
         self.new_working_dir = ""
-        self.simsinter_path = "C:/Program Files (x86)/CCSI/SimSinter"
+        self.simsinter_path = core.SimSinterDependencyTracker.load().path
         self.psuade_path = (
             shutil.which("psuade")
             or "C:/Program Files (x86)/psuade_project 1.7.5/bin/psuade.exe"
@@ -791,7 +792,7 @@ def __init__(self):
         self.turbineRemoteReSub = 0  # number of times to resubmit failed
         # jobs to Turbine when running remote
         self.aspenVersion = 2  # 0 = none, 1 = 7.3, 2 = 7.3.2 or higher
-        self.turbLiteHome = "C:\\Program Files (x86)\\Turbine\\Lite"
+        self.turbLiteHome = core.TurbineLiteDependencyTracker.load().path
         self.rScriptPath = "C:\\Program Files\\R\\R-3.1.2\\bin\\x64\\Rscript.exe"
         self.logFormat = "%(asctime)s - %(levelname)s - %(name)s - %(message)s"
         self.logRotate = False
diff --git a/foqus_lib/framework/sim/turbineConfiguration.py b/foqus_lib/framework/sim/turbineConfiguration.py
index ced186c18..f11bac743 100644
--- a/foqus_lib/framework/sim/turbineConfiguration.py
+++ b/foqus_lib/framework/sim/turbineConfiguration.py
@@ -53,6 +53,7 @@
     read_configuration,
 )
 
+from foqus_lib import core
 import foqus_lib.framework.sim.process_management as _pm
 from foqus_lib.framework.foqusException.foqusException import *
 
@@ -238,7 +239,7 @@ def __init__(self, path="turbine.cfg"):
         self.user = ""
         self.pwd = ""
         self.turbVer = "Lite"  # Lite, Remote or ....
-        self.turbineLiteHome = "C:\\Program Files (x86)\\Turbine\\Lite"
+        self.turbineLiteHome = core.TurbineLiteDependencyTracker.load().path
         self.consumers = {}
         self.consumerCountDict = {}
         self.reloadTurbine()
diff --git a/foqus_lib/framework/sim/turbineLiteDB.py b/foqus_lib/framework/sim/turbineLiteDB.py
index 23071a806..ecb74aa45 100644
--- a/foqus_lib/framework/sim/turbineLiteDB.py
+++ b/foqus_lib/framework/sim/turbineLiteDB.py
@@ -58,8 +58,8 @@ def __init__(self, close_after=True):
         self.conn = None
         self.close_after = close_after
         self.dbFile = (
-            "C:\\Program Files (x86)"
-            "\\Turbine\\Lite\\Data\\TurbineCompactDatabase.sdf"
+            core.TurbineLiteDependencyTracker.load().path
+            "\\Data\\TurbineCompactDatabase.sdf"
         )
 
     def __del__(self):

From 270396a4b10fe488af20a38ebf9b4fb5c409bce8 Mon Sep 17 00:00:00 2001
From: JOSHUA Robert Boverhof <jrboverhof@lbl.gov>
Date: Wed, 26 Jun 2024 16:56:26 -0700
Subject: [PATCH 12/25] add import

---
 foqus_lib/framework/sim/turbineLiteDB.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/foqus_lib/framework/sim/turbineLiteDB.py b/foqus_lib/framework/sim/turbineLiteDB.py
index ecb74aa45..bae5bb7fc 100644
--- a/foqus_lib/framework/sim/turbineLiteDB.py
+++ b/foqus_lib/framework/sim/turbineLiteDB.py
@@ -24,6 +24,7 @@
 
 import adodbapi
 import adodbapi.apibase
+from foqus_lib import core
 
 adodbapi.adodbapi.defaultCursorLocation = 2  # adodbapi.adUseServer
 

From 843d202a8c9c3d841c9a7a2c7d60c279c6d687db Mon Sep 17 00:00:00 2001
From: JOSHUA Robert Boverhof <jrboverhof@lbl.gov>
Date: Wed, 26 Jun 2024 16:58:08 -0700
Subject: [PATCH 13/25] add core

---
 foqus_lib/core.py | 199 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 199 insertions(+)
 create mode 100644 foqus_lib/core.py

diff --git a/foqus_lib/core.py b/foqus_lib/core.py
new file mode 100644
index 000000000..e42fe08c1
--- /dev/null
+++ b/foqus_lib/core.py
@@ -0,0 +1,199 @@
+###############################################################################
+# FOQUS Copyright (c) 2012 - 2021, by the software owners: Oak Ridge Institute
+# for Science and Education (ORISE), TRIAD National Security, LLC., Lawrence
+# Livermore National Security, LLC., The Regents of the University of
+# California, through Lawrence Berkeley National Laboratory, Battelle Memorial
+# Institute, Pacific Northwest Division through Pacific Northwest National
+# Laboratory, Carnegie Mellon University, West Virginia University, Boston
+# University, the Trustees of Princeton University, The University of Texas at
+# Austin, URS Energy & Construction, Inc., et al.  All rights reserved.
+#
+# Please see the file LICENSE.md for full copyright and license information,
+# respectively. This file is also available online at the URL
+# "https://github.com/CCSI-Toolset/FOQUS".
+#
+###############################################################################
+"""
+Joshua Boverhof, Lawrence Berkeley National Labs, 2024
+"""
+import os, shutil, logging
+from foqus_lib.framework.session.session import generalSettings as FoqusSettings
+
+
+class DependencyTracker:
+    @classmethod
+    def available(cls):
+        """ Returns set of available packages
+        """
+        raise NotImplementedError()
+
+    @classmethod
+    def unavailable(cls):
+        """ Returns set of unavailable packages
+        """
+        raise NotImplementedError()
+
+class ModuleDependencyTracker:
+    """tracks imported python modules"""
+
+    python_modules_available = {}
+    python_modules_unavailable = {}
+    module = None
+
+    @classmethod
+    def available(cls):
+        return tuple(cls.python_modules_available.values())
+
+    @classmethod
+    def unavailable(cls):
+        return tuple(cls.python_modules_unavailable.values())
+
+    @classmethod
+    def load(cls):
+        instance = cls.python_modules_available.get(cls.python_module_name)
+        if instance is not None:
+            return instance
+        instance = cls()
+        try:
+            exec("import %s" % (instance.python_module_name))
+        except ModuleNotFoundError:
+            cls.python_modules_unavailable[instance.python_module_name] = instance
+            raise
+        self._module = eval(instance.python_module_name)
+        cls.python_modules_available[instance.python_module_name] = instance
+        return instance
+
+    @classmethod
+    def load_capture_error(cls):
+        instance = None
+        try:
+            instance = cls.load()
+        except ModuleNotFoundError:
+            pass
+        return instance
+
+    def __init__(self):
+        self._module = None
+
+    @property
+    def module(self):
+        return self._module
+
+class ExecutableDependencyTracker(DependencyTracker):
+    """tracks optional executables"""
+    executables_available = dict()
+    executables_unavailable = dict()
+    executable_name = None
+    default_path = None
+    required = False
+
+    @classmethod
+    def available(cls):
+        return cls.executables_available.values()
+
+    @classmethod
+    def unavailable(cls):
+        return cls.executables_unavailable.values()
+
+    @classmethod
+    def path(cls):
+        raise NotImplementedError()
+
+    @classmethod
+    def load(cls):
+        assert cls.executable_name is not None
+        instance = cls.executables_available.get(cls.executable_name)
+        if instance is not None:
+            return instance
+        instance = cls()
+        if not os.path.isfile(instance.path()):
+            raise RuntimeError("%r:  Failed to Load Dependency" %(instance))
+        if not os.access(filename, os.X_OK):
+            raise RuntimeError("%r:  Dependency Path is not Executable:  %s" %(instance.path()))
+        cls.executables_available[instance.executable_name] = instance
+
+class PsuadeDependencyTracker(ExecutableDependencyTracker):
+    """
+    plugin = PsuadeDependencyTracker.load()
+    if plugin == None:  print("unavailable")
+    elif plugin.nomad is False: print("nomand unavailable")
+    """
+    required = False
+    executable_name = "psuade"
+    default_path = "C:/Program Files (x86)/psuade_project 1.7.5/bin/psuade.exe"
+    def path(self):
+        return (
+            shutil.which("psuade")
+            or self.default_path
+        )
+
+class RScriptDependencyTracker(ExecutableDependencyTracker):
+    required = False
+    executable_name = "Rscript.exe"
+    default_path = "C:\\Program Files\\R\\R-3.1.2\\bin\\x64\\Rscript.exe"
+    @classmethod
+    def path(self):
+        return shutil.which(cls.executable_name) or cls.default_path
+
+
+class WindowsPackageDependencyTracker(DependencyTracker):
+    """tracks installed Windows Packages"""
+
+    windows_packages_available = {}
+    windows_packages_unavailable = {}
+    package_name = None
+    install_path = None
+    required = False
+
+    @classmethod
+    def available(cls):
+        return cls.windows_packages_available.values()
+
+    @classmethod
+    def unavailable(cls):
+        return cls.windows_packages_unavailable.values()
+
+    @classmethod
+    def load(cls):
+        instance = cls.windows_packages_available.get(cls.package_name)
+        instance = instance or cls()
+        if not os.path.isfile(instance.path):
+            if cls.required:
+                raise RuntimeError("Install Path Does Not Exist: %s" %(instance.path))
+            if instance.package_name not in cls.windows_packages_unavailable:
+                cls.windows_packages_unavailable[instance.package_name] = cls.windows_packages_unavailable
+            logging.getLogger().warning("Install Path Does Not Exist: %s" %(instance.path))
+        cls.windows_packages_available[instance.package_name] = instance
+        return instance
+
+    @classmethod
+    def path(cls):
+        raise NotImplementedError()
+
+class SimSinterDependencyTracker(WindowsPackageDependencyTracker):
+    """
+    plugin = PsuadeDependencyTracker.load()
+    if plugin == None:  print("unavailable")
+    elif plugin.nomad is False: print("nomand unavailable")
+    """
+    package_name = "SimSinter"
+    install_path = "C:/Program Files/CCSI/SimSinter"
+
+    @property
+    def path(self):
+        return self.install_path
+
+
+class TurbineLiteDependencyTracker(WindowsPackageDependencyTracker):
+    """
+    plugin = PsuadeDependencyTracker.load()
+    if plugin == None:  print("unavailable")
+    elif plugin.nomad is False: print("nomand unavailable")
+    """
+    package_name = "TurbineLite"
+    install_path = "C:\\Program Files\\Turbine\\Lite"
+
+
+    @property
+    def path(self):
+        return self.install_path

From 1b6495a729a0103649c29ffe1b380823674cef32 Mon Sep 17 00:00:00 2001
From: JOSHUA Robert Boverhof <jrboverhof@lbl.gov>
Date: Thu, 27 Jun 2024 10:45:59 -0700
Subject: [PATCH 14/25] change isfile to isdir for install path

---
 foqus_lib/core.py                      | 8 ++++----
 foqus_lib/framework/session/session.py | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/foqus_lib/core.py b/foqus_lib/core.py
index e42fe08c1..323edff5c 100644
--- a/foqus_lib/core.py
+++ b/foqus_lib/core.py
@@ -17,7 +17,7 @@
 Joshua Boverhof, Lawrence Berkeley National Labs, 2024
 """
 import os, shutil, logging
-from foqus_lib.framework.session.session import generalSettings as FoqusSettings
+#from foqus_lib.framework.session.session import generalSettings as FoqusSettings
 
 
 class DependencyTracker:
@@ -106,7 +106,7 @@ def load(cls):
         if instance is not None:
             return instance
         instance = cls()
-        if not os.path.isfile(instance.path()):
+        if not os.path.isdir(instance.path()):
             raise RuntimeError("%r:  Failed to Load Dependency" %(instance))
         if not os.access(filename, os.X_OK):
             raise RuntimeError("%r:  Dependency Path is not Executable:  %s" %(instance.path()))
@@ -157,7 +157,7 @@ def unavailable(cls):
     def load(cls):
         instance = cls.windows_packages_available.get(cls.package_name)
         instance = instance or cls()
-        if not os.path.isfile(instance.path):
+        if not os.path.isdir(instance.path):
             if cls.required:
                 raise RuntimeError("Install Path Does Not Exist: %s" %(instance.path))
             if instance.package_name not in cls.windows_packages_unavailable:
@@ -191,7 +191,7 @@ class TurbineLiteDependencyTracker(WindowsPackageDependencyTracker):
     elif plugin.nomad is False: print("nomand unavailable")
     """
     package_name = "TurbineLite"
-    install_path = "C:\\Program Files\\Turbine\\Lite"
+    install_path = "C:/Program Files/Turbine/Lite"
 
 
     @property
diff --git a/foqus_lib/framework/session/session.py b/foqus_lib/framework/session/session.py
index 8282faedb..389cd6f37 100644
--- a/foqus_lib/framework/session/session.py
+++ b/foqus_lib/framework/session/session.py
@@ -30,7 +30,7 @@
 
 from foqus_lib import core
 import foqus_lib.framework.optimizer.problem as oprob
-from foqus_lib.framework.graph.graph import *
+from foqus_lib.framework.graph.graph import Graph,GraphEx
 from foqus_lib.framework.graph.node import nodeModelTypes
 from foqus_lib.framework.ml_ai_models import mlaiSearch
 from foqus_lib.framework.optimizer import problem

From 7ec236d4572507b92c989c0364bdbbd10f6c7c6e Mon Sep 17 00:00:00 2001
From: JOSHUA Robert Boverhof <jrboverhof@lbl.gov>
Date: Thu, 27 Jun 2024 11:06:49 -0700
Subject: [PATCH 15/25] less logging messages

---
 foqus_lib/service/flowsheet.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/foqus_lib/service/flowsheet.py b/foqus_lib/service/flowsheet.py
index 4535a42c4..c247bb6a5 100644
--- a/foqus_lib/service/flowsheet.py
+++ b/foqus_lib/service/flowsheet.py
@@ -858,7 +858,6 @@ def _run(self):
         dat = None
         while not self._stop:
             ret = None
-            _log.debug("pop job")
             try:
                 ret = self.pop_job(db, VisibilityTimeout=VisibilityTimeout)
             except FOQUSJobException as ex:
@@ -1072,7 +1071,7 @@ def pop_job(self, db, VisibilityTimeout=300):
         )
 
         if not response.get("Messages", None):
-            _log.info("Job Queue is Empty")
+            _log.debug("Job Queue is Empty")
             self.increment_metric_queue_peeks(state="empty")
             return
 

From ef445bf35c2a799b8f8c8a181a6ada323bf60c21 Mon Sep 17 00:00:00 2001
From: JOSHUA Robert Boverhof <jrboverhof@lbl.gov>
Date: Thu, 27 Jun 2024 11:39:21 -0700
Subject: [PATCH 16/25] update logging

---
 cloud/aws/logging.yaml                          | 10 ++++++++--
 foqus_lib/framework/sim/turbineConfiguration.py |  1 +
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/cloud/aws/logging.yaml b/cloud/aws/logging.yaml
index d3f8571f6..218b51a0a 100644
--- a/cloud/aws/logging.yaml
+++ b/cloud/aws/logging.yaml
@@ -1,17 +1,22 @@
 # Default AWS Config
 version: 1
 disable_existing_loggers: False
+formatters:
+  simple_formatter:
+      format: "%(levelname)-8s - %(asctime)12s - %(message)s"
 handlers:
   console:
     class: logging.StreamHandler
     level: DEBUG
     stream: ext://sys.stdout
+    formatter: simple_formatter
   logfile:
     class: logging.handlers.RotatingFileHandler
     level: DEBUG
     filename: watchtower.log
     maxBytes: 1000000
     backupCount: 3
+    formatter: simple_formatter
   watchtower:
     #class: watchtower.CloudWatchLogHandler
     class: foqus_lib.service.flowsheet.FoqusCloudWatchLogHandler
@@ -21,12 +26,13 @@ handlers:
     log_stream_name: "{machine_name}"
     send_interval: 10
     create_log_group: True
+    formatter: simple_formatter
 root:
   level: DEBUG
   propagate: True
   handlers: [console, logfile, watchtower]
 loggers:
   botocore:
-    level: INFO
+    level: WARN
   urllib3:
-    level: INFO
+    level: WARN
diff --git a/foqus_lib/framework/sim/turbineConfiguration.py b/foqus_lib/framework/sim/turbineConfiguration.py
index f11bac743..f0536b19c 100644
--- a/foqus_lib/framework/sim/turbineConfiguration.py
+++ b/foqus_lib/framework/sim/turbineConfiguration.py
@@ -369,6 +369,7 @@ def startConsumer(self, nodeName, simName):
 
         sinter_process_log = open("%s_sinter_log.txt" % app, "a")
         sinter_process_log.write("starting consumer\n")
+        _log.info("executing process: %s", f)
         proc = subprocess.Popen(
             [f],
             stdout=sinter_process_log,

From c59a84c895d21eb4b273ce58a6308bdb2f80e74b Mon Sep 17 00:00:00 2001
From: JOSHUA Robert Boverhof <jrboverhof@lbl.gov>
Date: Thu, 27 Jun 2024 12:29:07 -0700
Subject: [PATCH 17/25] clean code

---
 foqus_lib/framework/session/session.py   | 2 +-
 foqus_lib/framework/sim/turbineLiteDB.py | 7 ++++---
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/foqus_lib/framework/session/session.py b/foqus_lib/framework/session/session.py
index 389cd6f37..c333ca8dc 100644
--- a/foqus_lib/framework/session/session.py
+++ b/foqus_lib/framework/session/session.py
@@ -41,7 +41,7 @@
 from foqus_lib.framework.plugins import pluginSearch
 from foqus_lib.framework.pymodel import pymodel
 from foqus_lib.framework.sampleResults.results import Results
-from foqus_lib.framework.sim.turbineConfiguration import *
+from foqus_lib.framework.sim.turbineConfiguration import TurbineConfiguration
 from foqus_lib.framework.surrogate import surrogate
 from foqus_lib.framework.surrogate.surrogate import surrogate as junk2
 from foqus_lib.framework.uq.LocalExecutionModule import *
diff --git a/foqus_lib/framework/sim/turbineLiteDB.py b/foqus_lib/framework/sim/turbineLiteDB.py
index bae5bb7fc..e1a5a25f7 100644
--- a/foqus_lib/framework/sim/turbineLiteDB.py
+++ b/foqus_lib/framework/sim/turbineLiteDB.py
@@ -18,6 +18,7 @@
 John Eslick, Carnegie Mellon University, 2014
 """
 import os
+import os.path
 import threading
 import time
 import uuid
@@ -58,9 +59,9 @@ class turbineLiteDB:
     def __init__(self, close_after=True):
         self.conn = None
         self.close_after = close_after
-        self.dbFile = (
-            core.TurbineLiteDependencyTracker.load().path
-            "\\Data\\TurbineCompactDatabase.sdf"
+        self.dbFile = os.path.join(
+            core.TurbineLiteDependencyTracker.load().path,
+            "/Data/TurbineCompactDatabase.sdf"
         )
 
     def __del__(self):

From 1142414492c5db3d3698284d27eddfa4973d3743 Mon Sep 17 00:00:00 2001
From: JOSHUA Robert Boverhof <jrboverhof@lbl.gov>
Date: Tue, 24 Sep 2024 12:16:34 -0700
Subject: [PATCH 18/25] format with black

---
 foqus_lib/core.py                        | 42 +++++++++++++++---------
 foqus_lib/framework/session/session.py   |  2 +-
 foqus_lib/framework/sim/turbineLiteDB.py |  2 +-
 foqus_lib/service/flowsheet.py           | 12 ++++---
 4 files changed, 37 insertions(+), 21 deletions(-)

diff --git a/foqus_lib/core.py b/foqus_lib/core.py
index 323edff5c..132c4959c 100644
--- a/foqus_lib/core.py
+++ b/foqus_lib/core.py
@@ -17,22 +17,22 @@
 Joshua Boverhof, Lawrence Berkeley National Labs, 2024
 """
 import os, shutil, logging
-#from foqus_lib.framework.session.session import generalSettings as FoqusSettings
+
+# from foqus_lib.framework.session.session import generalSettings as FoqusSettings
 
 
 class DependencyTracker:
     @classmethod
     def available(cls):
-        """ Returns set of available packages
-        """
+        """Returns set of available packages"""
         raise NotImplementedError()
 
     @classmethod
     def unavailable(cls):
-        """ Returns set of unavailable packages
-        """
+        """Returns set of unavailable packages"""
         raise NotImplementedError()
 
+
 class ModuleDependencyTracker:
     """tracks imported python modules"""
 
@@ -79,8 +79,10 @@ def __init__(self):
     def module(self):
         return self._module
 
+
 class ExecutableDependencyTracker(DependencyTracker):
     """tracks optional executables"""
+
     executables_available = dict()
     executables_unavailable = dict()
     executable_name = None
@@ -107,30 +109,34 @@ def load(cls):
             return instance
         instance = cls()
         if not os.path.isdir(instance.path()):
-            raise RuntimeError("%r:  Failed to Load Dependency" %(instance))
+            raise RuntimeError("%r:  Failed to Load Dependency" % (instance))
         if not os.access(filename, os.X_OK):
-            raise RuntimeError("%r:  Dependency Path is not Executable:  %s" %(instance.path()))
+            raise RuntimeError(
+                "%r:  Dependency Path is not Executable:  %s" % (instance.path())
+            )
         cls.executables_available[instance.executable_name] = instance
 
+
 class PsuadeDependencyTracker(ExecutableDependencyTracker):
     """
     plugin = PsuadeDependencyTracker.load()
     if plugin == None:  print("unavailable")
     elif plugin.nomad is False: print("nomand unavailable")
     """
+
     required = False
     executable_name = "psuade"
     default_path = "C:/Program Files (x86)/psuade_project 1.7.5/bin/psuade.exe"
+
     def path(self):
-        return (
-            shutil.which("psuade")
-            or self.default_path
-        )
+        return shutil.which("psuade") or self.default_path
+
 
 class RScriptDependencyTracker(ExecutableDependencyTracker):
     required = False
     executable_name = "Rscript.exe"
     default_path = "C:\\Program Files\\R\\R-3.1.2\\bin\\x64\\Rscript.exe"
+
     @classmethod
     def path(self):
         return shutil.which(cls.executable_name) or cls.default_path
@@ -159,10 +165,14 @@ def load(cls):
         instance = instance or cls()
         if not os.path.isdir(instance.path):
             if cls.required:
-                raise RuntimeError("Install Path Does Not Exist: %s" %(instance.path))
+                raise RuntimeError("Install Path Does Not Exist: %s" % (instance.path))
             if instance.package_name not in cls.windows_packages_unavailable:
-                cls.windows_packages_unavailable[instance.package_name] = cls.windows_packages_unavailable
-            logging.getLogger().warning("Install Path Does Not Exist: %s" %(instance.path))
+                cls.windows_packages_unavailable[instance.package_name] = (
+                    cls.windows_packages_unavailable
+                )
+            logging.getLogger().warning(
+                "Install Path Does Not Exist: %s" % (instance.path)
+            )
         cls.windows_packages_available[instance.package_name] = instance
         return instance
 
@@ -170,12 +180,14 @@ def load(cls):
     def path(cls):
         raise NotImplementedError()
 
+
 class SimSinterDependencyTracker(WindowsPackageDependencyTracker):
     """
     plugin = PsuadeDependencyTracker.load()
     if plugin == None:  print("unavailable")
     elif plugin.nomad is False: print("nomand unavailable")
     """
+
     package_name = "SimSinter"
     install_path = "C:/Program Files/CCSI/SimSinter"
 
@@ -190,10 +202,10 @@ class TurbineLiteDependencyTracker(WindowsPackageDependencyTracker):
     if plugin == None:  print("unavailable")
     elif plugin.nomad is False: print("nomand unavailable")
     """
+
     package_name = "TurbineLite"
     install_path = "C:/Program Files/Turbine/Lite"
 
-
     @property
     def path(self):
         return self.install_path
diff --git a/foqus_lib/framework/session/session.py b/foqus_lib/framework/session/session.py
index 447ad0599..0fa1d0cee 100644
--- a/foqus_lib/framework/session/session.py
+++ b/foqus_lib/framework/session/session.py
@@ -30,7 +30,7 @@
 
 from foqus_lib import core
 import foqus_lib.framework.optimizer.problem as oprob
-from foqus_lib.framework.graph.graph import Graph,GraphEx
+from foqus_lib.framework.graph.graph import Graph, GraphEx
 from foqus_lib.framework.graph.node import nodeModelTypes
 from foqus_lib.framework.ml_ai_models import mlaiSearch
 from foqus_lib.framework.optimizer import problem
diff --git a/foqus_lib/framework/sim/turbineLiteDB.py b/foqus_lib/framework/sim/turbineLiteDB.py
index e1a5a25f7..36ca0f3b4 100644
--- a/foqus_lib/framework/sim/turbineLiteDB.py
+++ b/foqus_lib/framework/sim/turbineLiteDB.py
@@ -61,7 +61,7 @@ def __init__(self, close_after=True):
         self.close_after = close_after
         self.dbFile = os.path.join(
             core.TurbineLiteDependencyTracker.load().path,
-            "/Data/TurbineCompactDatabase.sdf"
+            "/Data/TurbineCompactDatabase.sdf",
         )
 
     def __del__(self):
diff --git a/foqus_lib/service/flowsheet.py b/foqus_lib/service/flowsheet.py
index c247bb6a5..57545aba9 100644
--- a/foqus_lib/service/flowsheet.py
+++ b/foqus_lib/service/flowsheet.py
@@ -62,22 +62,26 @@
 class FoqusCloudWatchLogHandler(watchtower.CloudWatchLogHandler):
     @functools.lru_cache(maxsize=0)
     def _get_machine_name(self):
-       return FOQUSAWSConfig.get_instance().instance_id
+        return FOQUSAWSConfig.get_instance().instance_id
+
     @functools.lru_cache(maxsize=0)
     def _get_user(self):
-       return FOQUSAWSConfig.get_instance().get_user()
+        return FOQUSAWSConfig.get_instance().get_user()
+
     def _get_stream_name(self, message):
-        return '/user/%s/ec2/%s' %(self._get_user(), self._get_machine_name())
+        return "/user/%s/ec2/%s" % (self._get_user(), self._get_machine_name())
+
 
 def _applyLogSettings(self_gs):
     # Short circuit FOQUS logging setup
     region_name = FOQUSAWSConfig.get_instance().get_region()
-    os.environ['AWS_DEFAULT_REGION'] = region_name
+    os.environ["AWS_DEFAULT_REGION"] = region_name
     with open(os.path.join(WORKING_DIRECTORY, "logging.yaml")) as log_config:
         config_yml = log_config.read()
         config_dict = yaml.safe_load(config_yml)
         logging.config.dictConfig(config_dict)
 
+
 def _set_working_dir(wdir):
     global _log, WORKING_DIRECTORY
     WORKING_DIRECTORY = wdir

From ed05b389b717337ce9337ddd56394b7efd789c6c Mon Sep 17 00:00:00 2001
From: Joshua Boverhof <jrboverhof@lbl.gov>
Date: Fri, 4 Oct 2024 11:58:22 -0700
Subject: [PATCH 19/25] comment out this override code so tests will pass

---
 foqus_lib/service/flowsheet.py | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/foqus_lib/service/flowsheet.py b/foqus_lib/service/flowsheet.py
index 57545aba9..7af98438a 100644
--- a/foqus_lib/service/flowsheet.py
+++ b/foqus_lib/service/flowsheet.py
@@ -82,9 +82,17 @@ def _applyLogSettings(self_gs):
         logging.config.dictConfig(config_dict)
 
 
-def _set_working_dir(wdir):
+def _set_working_dir(wdir, override=False):
+    """Set working directory, apply settings and log configuration.
+    Parameters:
+        override: Change the user configuration location and log settings.
+    """
     global _log, WORKING_DIRECTORY
     WORKING_DIRECTORY = wdir
+    # if override:
+    #     FoqusSettings.getUserConfigLocation = _get_user_config_location
+    #     FoqusSettings.applyLogSettings = _applyLogSettings
+        
     log_dir = os.path.join(wdir, "logs")
     try:
         os.makedirs(log_dir)
@@ -106,10 +114,6 @@ def _get_user_config_location(*args, **kw):
     return os.path.join(WORKING_DIRECTORY, "foqus.cfg")
 
 
-FoqusSettings.getUserConfigLocation = _get_user_config_location
-FoqusSettings.applyLogSettings = _applyLogSettings
-
-
 def getfilenames(jid):
     global CURRENT_JOB_DIR
     CURRENT_JOB_DIR = os.path.join(WORKING_DIRECTORY, str(jid))
@@ -770,7 +774,7 @@ def __init__(self):
     def _set_working_directory(cls, working_dir=WORKING_DIRECTORY):
         if cls._is_set_working_directory:
             return
-        _set_working_dir(working_dir)
+        _set_working_dir(working_dir, override=True)
         cls._is_set_working_directory = True
 
     def stop(self):

From 30b94d3754058cc64333e15cb03956b5e702059e Mon Sep 17 00:00:00 2001
From: JOSHUA Robert Boverhof <jrboverhof@lbl.gov>
Date: Fri, 4 Oct 2024 12:22:49 -0700
Subject: [PATCH 20/25] run black

---
 foqus_lib/service/flowsheet.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/foqus_lib/service/flowsheet.py b/foqus_lib/service/flowsheet.py
index 7af98438a..8e8e43c9e 100644
--- a/foqus_lib/service/flowsheet.py
+++ b/foqus_lib/service/flowsheet.py
@@ -92,7 +92,7 @@ def _set_working_dir(wdir, override=False):
     # if override:
     #     FoqusSettings.getUserConfigLocation = _get_user_config_location
     #     FoqusSettings.applyLogSettings = _applyLogSettings
-        
+
     log_dir = os.path.join(wdir, "logs")
     try:
         os.makedirs(log_dir)

From 76fdaebc300414897f99339f8635341390c9386a Mon Sep 17 00:00:00 2001
From: JOSHUA Robert Boverhof <jrboverhof@lbl.gov>
Date: Fri, 4 Oct 2024 12:33:33 -0700
Subject: [PATCH 21/25] pylint fixes

---
 foqus_lib/core.py              | 10 +++++-----
 foqus_lib/foqus.py             |  2 ++
 foqus_lib/service/flowsheet.py |  3 +--
 3 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/foqus_lib/core.py b/foqus_lib/core.py
index 132c4959c..8681d9f6a 100644
--- a/foqus_lib/core.py
+++ b/foqus_lib/core.py
@@ -38,7 +38,7 @@ class ModuleDependencyTracker:
 
     python_modules_available = {}
     python_modules_unavailable = {}
-    module = None
+    python_module_name = None
 
     @classmethod
     def available(cls):
@@ -59,7 +59,7 @@ def load(cls):
         except ModuleNotFoundError:
             cls.python_modules_unavailable[instance.python_module_name] = instance
             raise
-        self._module = eval(instance.python_module_name)
+        instance._module = eval(instance.python_module_name)
         cls.python_modules_available[instance.python_module_name] = instance
         return instance
 
@@ -108,9 +108,9 @@ def load(cls):
         if instance is not None:
             return instance
         instance = cls()
-        if not os.path.isdir(instance.path()):
+        if not os.path.isfile(instance.path()):
             raise RuntimeError("%r:  Failed to Load Dependency" % (instance))
-        if not os.access(filename, os.X_OK):
+        if not os.access(instance.path(), os.X_OK):
             raise RuntimeError(
                 "%r:  Dependency Path is not Executable:  %s" % (instance.path())
             )
@@ -138,7 +138,7 @@ class RScriptDependencyTracker(ExecutableDependencyTracker):
     default_path = "C:\\Program Files\\R\\R-3.1.2\\bin\\x64\\Rscript.exe"
 
     @classmethod
-    def path(self):
+    def path(cls):
         return shutil.which(cls.executable_name) or cls.default_path
 
 
diff --git a/foqus_lib/foqus.py b/foqus_lib/foqus.py
index cb56c0656..76efe032b 100644
--- a/foqus_lib/foqus.py
+++ b/foqus_lib/foqus.py
@@ -29,6 +29,8 @@
 import sys
 import time
 import uuid
+import traceback
+import turbine
 
 # FOQUS imports
 import foqus_lib.version.version as ver  # foqus version and other info
diff --git a/foqus_lib/service/flowsheet.py b/foqus_lib/service/flowsheet.py
index 8e8e43c9e..17c1e4ea8 100644
--- a/foqus_lib/service/flowsheet.py
+++ b/foqus_lib/service/flowsheet.py
@@ -20,7 +20,6 @@
 """
 import errno
 import json
-import yaml
 import logging
 import logging.config
 import optparse
@@ -36,11 +35,11 @@
 import urllib.request
 import uuid
 from os.path import expanduser
-
 import boto3
 import botocore.exceptions
 import watchtower
 import functools
+import yaml
 
 from turbine.commands import turbine_simulation_script
 from foqus_lib.framework.foqusException.foqusException import *

From 407d9cb28598a5d15e9964875c19600d3bac634a Mon Sep 17 00:00:00 2001
From: JOSHUA Robert Boverhof <jrboverhof@lbl.gov>
Date: Fri, 4 Oct 2024 12:39:20 -0700
Subject: [PATCH 22/25] move functools in front of boto3

---
 foqus_lib/service/flowsheet.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/foqus_lib/service/flowsheet.py b/foqus_lib/service/flowsheet.py
index 17c1e4ea8..b2e71da51 100644
--- a/foqus_lib/service/flowsheet.py
+++ b/foqus_lib/service/flowsheet.py
@@ -35,10 +35,10 @@
 import urllib.request
 import uuid
 from os.path import expanduser
+import functools
 import boto3
 import botocore.exceptions
 import watchtower
-import functools
 import yaml
 
 from turbine.commands import turbine_simulation_script

From 6863a4340b4ac7b2ea4195231891581ffda00670 Mon Sep 17 00:00:00 2001
From: JOSHUA Robert Boverhof <jrboverhof@lbl.gov>
Date: Mon, 7 Oct 2024 08:54:18 -0700
Subject: [PATCH 23/25] replace header

---
 foqus_lib/core.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/foqus_lib/core.py b/foqus_lib/core.py
index 8681d9f6a..fa1ee5c6f 100644
--- a/foqus_lib/core.py
+++ b/foqus_lib/core.py
@@ -1,5 +1,5 @@
-###############################################################################
-# FOQUS Copyright (c) 2012 - 2021, by the software owners: Oak Ridge Institute
+#################################################################################
+# FOQUS Copyright (c) 2012 - 2024, by the software owners: Oak Ridge Institute
 # for Science and Education (ORISE), TRIAD National Security, LLC., Lawrence
 # Livermore National Security, LLC., The Regents of the University of
 # California, through Lawrence Berkeley National Laboratory, Battelle Memorial
@@ -11,8 +11,7 @@
 # Please see the file LICENSE.md for full copyright and license information,
 # respectively. This file is also available online at the URL
 # "https://github.com/CCSI-Toolset/FOQUS".
-#
-###############################################################################
+#################################################################################
 """
 Joshua Boverhof, Lawrence Berkeley National Labs, 2024
 """

From ee9980f3f27e527d768441993fd36d610d0af9c4 Mon Sep 17 00:00:00 2001
From: JOSHUA Robert Boverhof <jrboverhof@lbl.gov>
Date: Mon, 7 Oct 2024 08:58:12 -0700
Subject: [PATCH 24/25] turbine client is required and installed during setup

---
 foqus_lib/foqus.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/foqus_lib/foqus.py b/foqus_lib/foqus.py
index 76efe032b..ee260ffc3 100644
--- a/foqus_lib/foqus.py
+++ b/foqus_lib/foqus.py
@@ -256,10 +256,7 @@ def main(args_to_parse=None):
     logging.getLogger("foqus").setLevel(logging.DEBUG)
     logging.getLogger("turbine").setLevel(logging.DEBUG)
     sys.excepthook = logException  # for unhandled exception logging
-    try:
-        turbine.commands._setup_logging.done = True
-    except:
-        _logger.exception("Cannot find turbine module")
+    turbine.commands._setup_logging.done = True
     app = None  # Qt application if I need to display message boxes.
     ## Setup the command line arguments
     parser = argparse.ArgumentParser()

From 855f93a9eb0c0eebfa9396b63c24f96a52942a5a Mon Sep 17 00:00:00 2001
From: JOSHUA Robert Boverhof <jrboverhof@lbl.gov>
Date: Wed, 9 Oct 2024 14:10:30 -0700
Subject: [PATCH 25/25] remove sphinx deprecation

---
 docs/source/conf.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/source/conf.py b/docs/source/conf.py
index 3bf47ccbf..240fb7c43 100644
--- a/docs/source/conf.py
+++ b/docs/source/conf.py
@@ -168,7 +168,7 @@
 # Add any paths that contain custom themes here, relative to this directory.
 # html_theme_path = []
 # html_theme_path = ['_themes', ]
-html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
+# html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
 
 # The name for this set of Sphinx documents.
 # "<project> v<release> documentation" by default.