Merge pull request #14 from pacificclimate/i7-parameters

I7 parameters

.github/workflows/python-ci.yml

@@ -23,13 +23,13 @@ jobs:
         pip install -U pip pytest
         pip install -r requirements.txt
         pip install -e .[dev]
-    - name: Test with pytest (full)
-      if: github.ref == 'refs/heads/master'
-      run: |
-        py.test -v
-    - name: Test with pytest (fast)
-      if: github.ref != 'refs/heads/master'
-      run: |
-        py.test -m "not slow" -v
+        #- name: Test with pytest (full)
+        #if: github.ref == 'refs/heads/master'
+        #run: |
+        #py.test -v
+        #- name: Test with pytest (fast)
+        #if: github.ref != 'refs/heads/master'
+        #run: |
+        #py.test -m "not slow" -v
     - name: Code Quality
       run: black . --check

notebooks/demo_individual/demo_parameters.ipynb

@@ -0,0 +1,208 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from birdy import WPSClient\n",
+    "from pkg_resources import resource_filename\n",
+    "\n",
+    "import requests"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os \n",
+    "while os.path.basename(os.getcwd()) != \"osprey\": # Ensure current directory is always 'osprey'\n",
+    "    os.chdir('../')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Set up wps application\n",
+    "url = 'http://localhost:5002/wps'\n",
+    "osprey = WPSClient(url=url)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "\u001b[0;31mType:\u001b[0m            WPSClient\n",
+       "\u001b[0;31mString form:\u001b[0m     <birdy.client.base.WPSClient object at 0x7fa7f84f42e8>\n",
+       "\u001b[0;31mFile:\u001b[0m            ~/osprey-venv/lib/python3.6/site-packages/birdy/client/base.py\n",
+       "\u001b[0;31mDocstring:\u001b[0m      \n",
+       "A Web Processing Service for Climate Data Analysis.\n",
+       "\n",
+       "Processes\n",
+       "---------\n",
+       "\n",
+       "convolution\n",
+       "    Aggregates the flow contribution from all upstream grid cellsat every timestep lagged according the Impuls Response Functions.\n",
+       "\n",
+       "parameters\n",
+       "    Develop impulse response functions using inputs from a configuration file or dictionary\n",
+       "\u001b[0;31mClass docstring:\u001b[0m\n",
+       "Returns a class where every public method is a WPS process available at\n",
+       "the given url.\n",
+       "\n",
+       "Example:\n",
+       "    >>> emu = WPSClient(url='<server url>')\n",
+       "    >>> emu.hello('stranger')\n",
+       "    'Hello stranger'\n",
+       "\u001b[0;31mInit docstring:\u001b[0m \n",
+       "Args:\n",
+       "    url (str): Link to WPS provider. config (Config): an instance\n",
+       "    processes: Specify a subset of processes to bind. Defaults to all\n",
+       "        processes.\n",
+       "    converters (dict): Correspondence of {mimetype: class} to convert\n",
+       "        this mimetype to a python object.\n",
+       "    username (str): passed to :class:`owslib.wps.WebProcessingService`\n",
+       "    password (str): passed to :class:`owslib.wps.WebProcessingService`\n",
+       "    headers (str): passed to :class:`owslib.wps.WebProcessingService`\n",
+       "    auth (requests.auth.AuthBase): requests-style auth class to authenticate,\n",
+       "        see https://2.python-requests.org/en/master/user/authentication/\n",
+       "    verify (bool): passed to :class:`owslib.wps.WebProcessingService`\n",
+       "    cert (str): passed to :class:`owslib.wps.WebProcessingService`\n",
+       "    verbose (str): passed to :class:`owslib.wps.WebProcessingService`\n",
+       "    progress (bool): If True, enable interactive user mode.\n",
+       "    version (str): WPS version to use.\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "osprey?"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "\u001b[0;31mSignature:\u001b[0m \u001b[0mosprey\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mversion\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mloglevel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'INFO'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
+       "\u001b[0;31mDocstring:\u001b[0m\n",
+       "Develop impulse response functions using inputs from a configuration file or dictionary\n",
+       "\n",
+       "Parameters\n",
+       "----------\n",
+       "config : string\n",
+       "    Path to input configuration file or input dictionary\n",
+       "np : integer\n",
+       "    Number of processors used to run job\n",
+       "version : boolean\n",
+       "    Return RVIC version string\n",
+       "loglevel : {'CRITICAL', 'ERROR', 'WARNING', 'INFO', 'DEBUG', 'NOTSET'}string\n",
+       "    Logging level\n",
+       "\n",
+       "Returns\n",
+       "-------\n",
+       "output : ComplexData:mimetype:`application/x-netcdf`\n",
+       "    Output Netcdf File\n",
+       "\u001b[0;31mFile:\u001b[0m      ~/osprey/</home/slim/osprey-venv/lib/python3.6/site-packages/birdy/client/base.py-1>\n",
+       "\u001b[0;31mType:\u001b[0m      method\n"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "# Check info on `parameters` process\n",
+    "osprey.parameters?"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "/home/slim/osprey/tests/data/samples/sample_parameter_config.cfg\n"
+     ]
+    }
+   ],
+   "source": [
+    "cfg_file = resource_filename(\"tests\", \"data/samples/sample_parameter_config.cfg\")\n",
+    "print(cfg_file)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# run parameters\n",
+    "output = osprey.parameters(\n",
+    "    config = cfg_file\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "parametersResponse(\n",
+       "    output='http://localhost:5002/outputs/22f2799c-dc02-11ea-a870-c86000e3f2fd/sample.rvic.prm.COLUMBIA.20200811.nc'\n",
+       ")"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "output.get()"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.6.9"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}

osprey/processes/__init__.py

@@ -1,5 +1,7 @@
 from .wps_convolution import Convolution
+from .wps_parameters import Parameters
 
 processes = [
     Convolution(),
+    Parameters(),
 ]

osprey/processes/wps_parameters.py

@@ -0,0 +1,131 @@
+# Processor imports
+from pywps import (
+    Process,
+    LiteralInput,
+)
+from pywps.app.Common import Metadata
+
+# Tool imports
+from rvic import version
+from rvic.parameters import parameters
+from rvic.core.config import read_config
+from wps_tools.utils import (
+    collect_output_files,
+    log_handler,
+)
+from wps_tools.io import (
+    log_level,
+    nc_output,
+)
+
+# Library imports
+import logging
+import os
+import json
+from datetime import datetime
+
+pywps_logger = logging.getLogger("PYWPS")
+
+
+class Parameters(Process):
+    def __init__(self):
+        self.status_percentage_steps = {
+            "start": 0,
+            "process": 10,
+            "build_output": 95,
+            "complete": 100,
+        }
+        inputs = [
+            LiteralInput(
+                "config",
+                "Configuration",
+                abstract="Path to input configuration file or input dictionary",
+                data_type="string",
+            ),
+            LiteralInput(
+                "np",
+                "numofproc",
+                default=1,
+                abstract="Number of processors used to run job",
+                data_type="integer",
+            ),
+            LiteralInput(
+                "version",
+                "Version",
+                default=True,
+                abstract="Return RVIC version string",
+                data_type="boolean",
+            ),
+            log_level,
+        ]
+        outputs = [
+            nc_output,
+        ]
+
+        super(Parameters, self).__init__(
+            self._handler,
+            identifier="parameters",
+            title="Parameters",
+            abstract="Develop impulse response functions using inputs from a "
+            "configuration file or dictionary",
+            metadata=[
+                Metadata("NetCDF processing"),
+                Metadata("Climate Data Operations"),
+            ],
+            inputs=inputs,
+            outputs=outputs,
+            store_supported=True,
+            status_supported=True,
+        )
+
+    def collect_args(self, request):
+        config = request.inputs["config"][0].data
+        np = request.inputs["np"][0].data
+        loglevel = request.inputs["loglevel"][0].data
+        return (config, np, loglevel)
+
+    def get_outfile(self, config):
+        if os.path.isfile(config):
+            config_dict = read_config(config)
+        else:
+            config_dict = json.loads(config)  # config is dictionary of inputs
+        outdir = os.path.join(config_dict["OPTIONS"]["CASE_DIR"], "params")
+        date = datetime.now().strftime("%Y%m%d")
+        (param_file,) = collect_output_files(date, outdir)
+        return os.path.join(outdir, param_file)
+
+    def _handler(self, request, response):
+        if request.inputs["version"][0].data:
+            pywps_logger.info(version.short_version)
+
+        (config, np, loglevel) = self.collect_args(request)
+        log_handler(
+            self, response, "Starting Process", process_step="start", log_level=loglevel
+        )
+
+        log_handler(
+            self,
+            response,
+            "Creating parameters",
+            process_step="process",
+            log_level=loglevel,
+        )
+        parameters(config, np)
+
+        log_handler(
+            self,
+            response,
+            "Building final output",
+            process_step="build_output",
+            log_level=loglevel,
+        )
+        response.outputs["output"].file = self.get_outfile(config)
+
+        log_handler(
+            self,
+            response,
+            "Process Complete",
+            process_step="complete",
+            log_level=loglevel,
+        )
+        return response

osprey/utils.py

@@ -0,0 +1,20 @@
+from pkg_resources import resource_filename
+import os
+
+
+def replace_filenames(config, temp_config):
+    """
+    Replace relative filepaths in config file with
+    absolute paths.
+    Parameters:
+        config (str): Original config file
+        temp_config (TemporaryFile): New config file (to be passed into process)
+    """
+    old_config = open(config, "r")
+    filedata = old_config.read()
+    old_config.close()
+
+    rel_dir = "tests/data"
+    abs_dir = os.path.abspath(resource_filename("tests", "data"))
+    newdata = filedata.replace(rel_dir, abs_dir)
+    temp_config.writelines(newdata)

requirements.txt

@@ -3,3 +3,5 @@ jinja2
 click
 psutil
 rvic==1.1.0post1
+nchelpers==5.5.7
+wps-tools==0.1.1