diff --git a/cloud/aws/logging.yaml b/cloud/aws/logging.yaml new file mode 100644 index 000000000..218b51a0a --- /dev/null +++ b/cloud/aws/logging.yaml @@ -0,0 +1,38 @@ +# Default AWS Config +version: 1 +disable_existing_loggers: False +formatters: + simple_formatter: + format: "%(levelname)-8s - %(asctime)12s - %(message)s" +handlers: + console: + class: logging.StreamHandler + level: DEBUG + stream: ext://sys.stdout + formatter: simple_formatter + logfile: + class: logging.handlers.RotatingFileHandler + level: DEBUG + filename: watchtower.log + maxBytes: 1000000 + backupCount: 3 + formatter: simple_formatter + watchtower: + #class: watchtower.CloudWatchLogHandler + class: foqus_lib.service.flowsheet.FoqusCloudWatchLogHandler + level: DEBUG + log_group_name: foqus-cloud-service + #log_stream_name: "{machine_name}-{strftime:%y-%m-%d}" + log_stream_name: "{machine_name}" + send_interval: 10 + create_log_group: True + formatter: simple_formatter +root: + level: DEBUG + propagate: True + handlers: [console, logfile, watchtower] +loggers: + botocore: + level: WARN + urllib3: + level: WARN diff --git a/foqus_lib/core.py b/foqus_lib/core.py new file mode 100644 index 000000000..fa1ee5c6f --- /dev/null +++ b/foqus_lib/core.py @@ -0,0 +1,210 @@ +################################################################################# +# FOQUS Copyright (c) 2012 - 2024, by the software owners: Oak Ridge Institute +# for Science and Education (ORISE), TRIAD National Security, LLC., Lawrence +# Livermore National Security, LLC., The Regents of the University of +# California, through Lawrence Berkeley National Laboratory, Battelle Memorial +# Institute, Pacific Northwest Division through Pacific Northwest National +# Laboratory, Carnegie Mellon University, West Virginia University, Boston +# University, the Trustees of Princeton University, The University of Texas at +# Austin, URS Energy & Construction, Inc., et al. All rights reserved. +# +# Please see the file LICENSE.md for full copyright and license information, +# respectively. This file is also available online at the URL +# "https://github.com/CCSI-Toolset/FOQUS". +################################################################################# +""" +Joshua Boverhof, Lawrence Berkeley National Labs, 2024 +""" +import os, shutil, logging + +# from foqus_lib.framework.session.session import generalSettings as FoqusSettings + + +class DependencyTracker: + @classmethod + def available(cls): + """Returns set of available packages""" + raise NotImplementedError() + + @classmethod + def unavailable(cls): + """Returns set of unavailable packages""" + raise NotImplementedError() + + +class ModuleDependencyTracker: + """tracks imported python modules""" + + python_modules_available = {} + python_modules_unavailable = {} + python_module_name = None + + @classmethod + def available(cls): + return tuple(cls.python_modules_available.values()) + + @classmethod + def unavailable(cls): + return tuple(cls.python_modules_unavailable.values()) + + @classmethod + def load(cls): + instance = cls.python_modules_available.get(cls.python_module_name) + if instance is not None: + return instance + instance = cls() + try: + exec("import %s" % (instance.python_module_name)) + except ModuleNotFoundError: + cls.python_modules_unavailable[instance.python_module_name] = instance + raise + instance._module = eval(instance.python_module_name) + cls.python_modules_available[instance.python_module_name] = instance + return instance + + @classmethod + def load_capture_error(cls): + instance = None + try: + instance = cls.load() + except ModuleNotFoundError: + pass + return instance + + def __init__(self): + self._module = None + + @property + def module(self): + return self._module + + +class ExecutableDependencyTracker(DependencyTracker): + """tracks optional executables""" + + executables_available = dict() + executables_unavailable = dict() + executable_name = None + default_path = None + required = False + + @classmethod + def available(cls): + return cls.executables_available.values() + + @classmethod + def unavailable(cls): + return cls.executables_unavailable.values() + + @classmethod + def path(cls): + raise NotImplementedError() + + @classmethod + def load(cls): + assert cls.executable_name is not None + instance = cls.executables_available.get(cls.executable_name) + if instance is not None: + return instance + instance = cls() + if not os.path.isfile(instance.path()): + raise RuntimeError("%r: Failed to Load Dependency" % (instance)) + if not os.access(instance.path(), os.X_OK): + raise RuntimeError( + "%r: Dependency Path is not Executable: %s" % (instance.path()) + ) + cls.executables_available[instance.executable_name] = instance + + +class PsuadeDependencyTracker(ExecutableDependencyTracker): + """ + plugin = PsuadeDependencyTracker.load() + if plugin == None: print("unavailable") + elif plugin.nomad is False: print("nomand unavailable") + """ + + required = False + executable_name = "psuade" + default_path = "C:/Program Files (x86)/psuade_project 1.7.5/bin/psuade.exe" + + def path(self): + return shutil.which("psuade") or self.default_path + + +class RScriptDependencyTracker(ExecutableDependencyTracker): + required = False + executable_name = "Rscript.exe" + default_path = "C:\\Program Files\\R\\R-3.1.2\\bin\\x64\\Rscript.exe" + + @classmethod + def path(cls): + return shutil.which(cls.executable_name) or cls.default_path + + +class WindowsPackageDependencyTracker(DependencyTracker): + """tracks installed Windows Packages""" + + windows_packages_available = {} + windows_packages_unavailable = {} + package_name = None + install_path = None + required = False + + @classmethod + def available(cls): + return cls.windows_packages_available.values() + + @classmethod + def unavailable(cls): + return cls.windows_packages_unavailable.values() + + @classmethod + def load(cls): + instance = cls.windows_packages_available.get(cls.package_name) + instance = instance or cls() + if not os.path.isdir(instance.path): + if cls.required: + raise RuntimeError("Install Path Does Not Exist: %s" % (instance.path)) + if instance.package_name not in cls.windows_packages_unavailable: + cls.windows_packages_unavailable[instance.package_name] = ( + cls.windows_packages_unavailable + ) + logging.getLogger().warning( + "Install Path Does Not Exist: %s" % (instance.path) + ) + cls.windows_packages_available[instance.package_name] = instance + return instance + + @classmethod + def path(cls): + raise NotImplementedError() + + +class SimSinterDependencyTracker(WindowsPackageDependencyTracker): + """ + plugin = PsuadeDependencyTracker.load() + if plugin == None: print("unavailable") + elif plugin.nomad is False: print("nomand unavailable") + """ + + package_name = "SimSinter" + install_path = "C:/Program Files/CCSI/SimSinter" + + @property + def path(self): + return self.install_path + + +class TurbineLiteDependencyTracker(WindowsPackageDependencyTracker): + """ + plugin = PsuadeDependencyTracker.load() + if plugin == None: print("unavailable") + elif plugin.nomad is False: print("nomand unavailable") + """ + + package_name = "TurbineLite" + install_path = "C:/Program Files/Turbine/Lite" + + @property + def path(self): + return self.install_path diff --git a/foqus_lib/foqus.py b/foqus_lib/foqus.py index cb56c0656..ee260ffc3 100644 --- a/foqus_lib/foqus.py +++ b/foqus_lib/foqus.py @@ -29,6 +29,8 @@ import sys import time import uuid +import traceback +import turbine # FOQUS imports import foqus_lib.version.version as ver # foqus version and other info @@ -254,10 +256,7 @@ def main(args_to_parse=None): logging.getLogger("foqus").setLevel(logging.DEBUG) logging.getLogger("turbine").setLevel(logging.DEBUG) sys.excepthook = logException # for unhandled exception logging - try: - turbine.commands._setup_logging.done = True - except: - _logger.exception("Cannot find turbine module") + turbine.commands._setup_logging.done = True app = None # Qt application if I need to display message boxes. ## Setup the command line arguments parser = argparse.ArgumentParser() diff --git a/foqus_lib/framework/session/session.py b/foqus_lib/framework/session/session.py index 7c833bb0d..0fa1d0cee 100644 --- a/foqus_lib/framework/session/session.py +++ b/foqus_lib/framework/session/session.py @@ -28,8 +28,9 @@ import sys import uuid +from foqus_lib import core import foqus_lib.framework.optimizer.problem as oprob -from foqus_lib.framework.graph.graph import * +from foqus_lib.framework.graph.graph import Graph, GraphEx from foqus_lib.framework.graph.node import nodeModelTypes from foqus_lib.framework.ml_ai_models import mlaiSearch from foqus_lib.framework.optimizer import problem @@ -40,7 +41,7 @@ from foqus_lib.framework.plugins import pluginSearch from foqus_lib.framework.pymodel import pymodel from foqus_lib.framework.sampleResults.results import Results -from foqus_lib.framework.sim.turbineConfiguration import * +from foqus_lib.framework.sim.turbineConfiguration import TurbineConfiguration from foqus_lib.framework.surrogate import surrogate from foqus_lib.framework.surrogate.surrogate import surrogate as junk2 from foqus_lib.framework.uq.LocalExecutionModule import * @@ -770,7 +771,7 @@ def __init__(self): self.working_dir_override = False self.working_dir = "" self.new_working_dir = "" - self.simsinter_path = "C:/Program Files (x86)/CCSI/SimSinter" + self.simsinter_path = core.SimSinterDependencyTracker.load().path self.psuade_path = ( shutil.which("psuade") or "C:/Program Files (x86)/psuade_project 1.7.5/bin/psuade.exe" @@ -791,7 +792,7 @@ def __init__(self): self.turbineRemoteReSub = 0 # number of times to resubmit failed # jobs to Turbine when running remote self.aspenVersion = 2 # 0 = none, 1 = 7.3, 2 = 7.3.2 or higher - self.turbLiteHome = "C:\\Program Files (x86)\\Turbine\\Lite" + self.turbLiteHome = core.TurbineLiteDependencyTracker.load().path self.rScriptPath = "C:\\Program Files\\R\\R-3.1.2\\bin\\x64\\Rscript.exe" self.logFormat = "%(asctime)s - %(levelname)s - %(name)s - %(message)s" self.logRotate = False diff --git a/foqus_lib/framework/sim/turbineConfiguration.py b/foqus_lib/framework/sim/turbineConfiguration.py index ced186c18..f0536b19c 100644 --- a/foqus_lib/framework/sim/turbineConfiguration.py +++ b/foqus_lib/framework/sim/turbineConfiguration.py @@ -53,6 +53,7 @@ read_configuration, ) +from foqus_lib import core import foqus_lib.framework.sim.process_management as _pm from foqus_lib.framework.foqusException.foqusException import * @@ -238,7 +239,7 @@ def __init__(self, path="turbine.cfg"): self.user = "" self.pwd = "" self.turbVer = "Lite" # Lite, Remote or .... - self.turbineLiteHome = "C:\\Program Files (x86)\\Turbine\\Lite" + self.turbineLiteHome = core.TurbineLiteDependencyTracker.load().path self.consumers = {} self.consumerCountDict = {} self.reloadTurbine() @@ -368,6 +369,7 @@ def startConsumer(self, nodeName, simName): sinter_process_log = open("%s_sinter_log.txt" % app, "a") sinter_process_log.write("starting consumer\n") + _log.info("executing process: %s", f) proc = subprocess.Popen( [f], stdout=sinter_process_log, diff --git a/foqus_lib/framework/sim/turbineLiteDB.py b/foqus_lib/framework/sim/turbineLiteDB.py index 23071a806..36ca0f3b4 100644 --- a/foqus_lib/framework/sim/turbineLiteDB.py +++ b/foqus_lib/framework/sim/turbineLiteDB.py @@ -18,12 +18,14 @@ John Eslick, Carnegie Mellon University, 2014 """ import os +import os.path import threading import time import uuid import adodbapi import adodbapi.apibase +from foqus_lib import core adodbapi.adodbapi.defaultCursorLocation = 2 # adodbapi.adUseServer @@ -57,9 +59,9 @@ class turbineLiteDB: def __init__(self, close_after=True): self.conn = None self.close_after = close_after - self.dbFile = ( - "C:\\Program Files (x86)" - "\\Turbine\\Lite\\Data\\TurbineCompactDatabase.sdf" + self.dbFile = os.path.join( + core.TurbineLiteDependencyTracker.load().path, + "/Data/TurbineCompactDatabase.sdf", ) def __del__(self): diff --git a/foqus_lib/service/flowsheet.py b/foqus_lib/service/flowsheet.py index a02b7989a..b2e71da51 100644 --- a/foqus_lib/service/flowsheet.py +++ b/foqus_lib/service/flowsheet.py @@ -35,11 +35,13 @@ import urllib.request import uuid from os.path import expanduser - +import functools import boto3 import botocore.exceptions -from turbine.commands import turbine_simulation_script +import watchtower +import yaml +from turbine.commands import turbine_simulation_script from foqus_lib.framework.foqusException.foqusException import * from foqus_lib.framework.graph.graph import Graph from foqus_lib.framework.graph.nodeVars import NodeVarEx, NodeVarListEx @@ -51,14 +53,45 @@ WORKING_DIRECTORY = os.path.abspath( os.environ.get("FOQUS_SERVICE_WORKING_DIR", "\\ProgramData\\foqus_service") ) -DEBUG = False + CURRENT_JOB_DIR = None _log = logging.getLogger("foqus.foqus_lib.service.flowsheet") -def _set_working_dir(wdir): +class FoqusCloudWatchLogHandler(watchtower.CloudWatchLogHandler): + @functools.lru_cache(maxsize=0) + def _get_machine_name(self): + return FOQUSAWSConfig.get_instance().instance_id + + @functools.lru_cache(maxsize=0) + def _get_user(self): + return FOQUSAWSConfig.get_instance().get_user() + + def _get_stream_name(self, message): + return "/user/%s/ec2/%s" % (self._get_user(), self._get_machine_name()) + + +def _applyLogSettings(self_gs): + # Short circuit FOQUS logging setup + region_name = FOQUSAWSConfig.get_instance().get_region() + os.environ["AWS_DEFAULT_REGION"] = region_name + with open(os.path.join(WORKING_DIRECTORY, "logging.yaml")) as log_config: + config_yml = log_config.read() + config_dict = yaml.safe_load(config_yml) + logging.config.dictConfig(config_dict) + + +def _set_working_dir(wdir, override=False): + """Set working directory, apply settings and log configuration. + Parameters: + override: Change the user configuration location and log settings. + """ global _log, WORKING_DIRECTORY WORKING_DIRECTORY = wdir + # if override: + # FoqusSettings.getUserConfigLocation = _get_user_config_location + # FoqusSettings.applyLogSettings = _applyLogSettings + log_dir = os.path.join(wdir, "logs") try: os.makedirs(log_dir) @@ -69,7 +102,6 @@ def _set_working_dir(wdir): FoqusSettings().applyLogSettings() _log = logging.getLogger("foqus.foqus_lib.service.flowsheet") - _log.setLevel(logging.DEBUG) _log.info("Working Directory: %s", WORKING_DIRECTORY) logging.getLogger("boto3").setLevel(logging.ERROR) logging.getLogger("botocore").setLevel(logging.ERROR) @@ -81,9 +113,6 @@ def _get_user_config_location(*args, **kw): return os.path.join(WORKING_DIRECTORY, "foqus.cfg") -FoqusSettings.getUserConfigLocation = _get_user_config_location - - def getfilenames(jid): global CURRENT_JOB_DIR CURRENT_JOB_DIR = os.path.join(WORKING_DIRECTORY, str(jid)) @@ -463,7 +492,6 @@ def __init__(self): def _get(self, key): v = self._d.get(key) assert v, "UserData/MetaData Missing Key(%s): %s" % (key, str(self._d)) - _log.debug("FOQUSAWSConfig._get: %s = %s" % (key, v)) return v def get_region(self): @@ -745,7 +773,7 @@ def __init__(self): def _set_working_directory(cls, working_dir=WORKING_DIRECTORY): if cls._is_set_working_directory: return - _set_working_dir(working_dir) + _set_working_dir(working_dir, override=True) cls._is_set_working_directory = True def stop(self): @@ -837,7 +865,6 @@ def _run(self): dat = None while not self._stop: ret = None - _log.debug("pop job") try: ret = self.pop_job(db, VisibilityTimeout=VisibilityTimeout) except FOQUSJobException as ex: @@ -1051,7 +1078,7 @@ def pop_job(self, db, VisibilityTimeout=300): ) if not response.get("Messages", None): - _log.info("Job Queue is Empty") + _log.debug("Job Queue is Empty") self.increment_metric_queue_peeks(state="empty") return diff --git a/setup.py b/setup.py index 42829acfd..21674124b 100644 --- a/setup.py +++ b/setup.py @@ -108,6 +108,7 @@ "scipy", "tqdm", "TurbineClient ~= 4.0, >= 4.0.3", + "watchtower", "winshell; sys_platform == 'win32'", "websocket_client>=1.1.0", ],