IMMM-SFA · rg727 · Aug 23, 2023 · Jan 5, 2023 · Jan 5, 2023 · Jan 5, 2023
diff --git a/.gitignore b/.gitignore
@@ -1,6 +1,9 @@
 # pycharm
 .idea/
 
+# other
+_static
+
 # system
 .DS_Store
 

diff --git a/MANIFEST.in b/MANIFEST.in
@@ -1 +1,2 @@
 include statemodify/data/*.*
+include statemodify/tests/data/*.*
diff --git a/README.md b/README.md
@@ -46,3 +46,24 @@ stm.modify_ddm(modify_dict=setup_dict,
                query_field=query_field,
                template_file=template_file)
 ```
+
+### Convert output .xdd files to .parquet files
+Parquet files are efficient columnar data stores that easily interoperate with pandas dataframes.
+```python
+from statemodify.xdd import XddConverter
+
+# set up the converter
+converter = XddConverter(
+    output_path='./output',
+    allow_overwrite=False,
+    xdd_files='**/*.xdd',
+    id_subset=None,
+    parallel_jobs=4,
+)
+
+# convert the files
+converter.convert()
+
+# look for your parquet files in './output'!
+
+```
diff --git a/statemodify/sample.py → docs/.nojekyll b/statemodify/sample.py → docs/.nojekyll
diff --git a/docs/Makefile b/docs/Makefile
@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = source
+BUILDDIR      = build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
diff --git a/docs/make.bat b/docs/make.bat
@@ -0,0 +1,35 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=source
+set BUILDDIR=build
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.https://www.sphinx-doc.org/
+	exit /b 1
+)
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O%
+
+:end
+popd
diff --git a/docs/source/conf.py b/docs/source/conf.py
@@ -0,0 +1,104 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# For the full list of built-in configuration values, see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+import sys
+import os
+
+sys.path.insert(0, os.path.abspath('.'))
+sys.path.insert(0, os.path.abspath('../'))
+sys.path.insert(0, os.path.abspath('../../'))
+
+# -- Project information -----------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
+
+project = 'statemodify'
+copyright = '2023, Battelle Memorial Institute'
+author = 'Rohini S. Gupta'
+
+# The version info for the project you're documenting, acts as replacement for
+# |version| and |release|, also used in various other places throughout the
+# built documents.
+from statemodify import __version__ as version
+
+# The full version, including alpha/beta/rc tags.
+release = version
+
+# -- General configuration ---------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
+
+extensions = [
+    'sphinx.ext.autodoc',
+    'sphinx.ext.autosectionlabel',
+    'sphinx.ext.autosummary',
+    'sphinx.ext.duration',
+    'sphinx.ext.doctest',
+    'sphinx.ext.intersphinx',
+    'sphinx.ext.napoleon',
+    'sphinx.ext.viewcode',
+    'sphinx_autodoc_typehints',
+    'sphinx_click',
+    'sphinx_copybutton',
+]
+
+intersphinx_mapping = {
+    'python': ('https://docs.python.org/3/', None),
+    'sphinx': ('https://www.sphinx-doc.org/en/master/', None),
+}
+intersphinx_disabled_domains = ['std']
+
+templates_path = ['_templates']
+
+# The main toctree document.
+main_doc = 'index'
+
+pygments_style = None
+autosummary_generate = True
+napolean_use_rtype = False
+
+# -- Options for autodoc
+
+# Python methods should be presented in source code order
+autodoc_member_order = 'bysource'
+
+# -- Options for HTML output -------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
+
+
+html_theme = 'sphinx_book_theme'
+
+html_theme_options = {
+    'logo_only': True,
+    'repository_url': 'https://github.com/IMMM-SFA/statemodify',
+    'use_repository_button': True,
+    'use_issues_button': True,
+    'use_edit_page_button': True,
+    'path_to_docs': 'docs/source',
+}
+
+# The name for this set of Sphinx documents.  If None, it defaults to
+# "<project> v<release> documentation".
+html_title = 'statemodify documentation'
+
+# -- Options for EPUB output
+epub_show_urls = 'footnote'
+
+# -- Options for sphinx-copybutton
+copybutton_prompt_text = r'\$ '
+copybutton_prompt_is_regexp = True
+
+html_show_sourcelink = False
+
+# The name of an image file (relative to this directory) to place at the top
+# of the sidebar.
+html_logo = 'images/logo_statemodify.png'
+
+# The name of an image file (within the static path) to use as favicon of the
+# docs. This file should be a Windows icon file (.ico), 16x16 or 32x32 pixels.
+html_favicon = '_static/favicon.ico'
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named 'default.css' will overwrite the builtin 'default.css'.
+html_static_path = ['_static']
diff --git a/docs/source/examples/modify-inputs.rst b/docs/source/examples/modify-inputs.rst
@@ -0,0 +1,2 @@
+Modify Inputs
+=================
diff --git a/docs/source/getting-started/installation.rst b/docs/source/getting-started/installation.rst
@@ -0,0 +1,17 @@
+Installation
+============
+
+Install ```statemodify``` using pip:
+
+.. code-block:: console
+
+  $ # statemodify requires python >= 3.8.
+  $ pip install statemodify
+
+You may also install ```statemodify``` from source.
+
+.. code-block:: console
+
+  $ git clone https://github.com/IMMM-SFA/statemodify.git
+  $ cd statemodify
+  $ pip install .
diff --git a/docs/source/getting-started/quickstarter.rst b/docs/source/getting-started/quickstarter.rst
@@ -0,0 +1,2 @@
+Quickstarter
+==============
diff --git a/docs/source/getting-started/tutorial.rst b/docs/source/getting-started/tutorial.rst
@@ -0,0 +1,2 @@
+Tutorial
+============
diff --git a/docs/source/images/figure_1.png b/docs/source/images/figure_1.png
diff --git a/docs/source/images/logo_statemodify.png b/docs/source/images/logo_statemodify.png
diff --git a/docs/source/index.rst b/docs/source/index.rst
@@ -0,0 +1,94 @@
+Welcome to **statemodify**!
+--------------------------------
+
+**statemodify** is a python-based framework that provides users with a way to easily interact with StateMod in a linux-based environment. As demonstrated in Figure 1, the **statemodify** framework is comprised of 4 key steps:
+
+.. raw:: html
+
+  <ol>
+  <li>The user will first be able to initialize a cluster in a clean environment, download and compile a StateMod executable, and download/import all other python packages that are required.</li>
+  <li><p>Next, the user will step through a series of methods that are created to facilitate manipulating and overwriting the historical StateMod files to represent plausible future conditions. For example:</p>
+  <ul>
+
+    <li><b>.xbm</b> files (monthly streamflow): can be overwritten with synthetic streamflows that are simultaneously generated across five key Colorado west-slope basins that represent plausible future flood/drought conditions. We currently implement a multi-basin Hidden Markov Model-based approach to generate these flows.</li>
+    <li><b>.iwr</b> files (monthly irrigation): can be overwritten in conjunction with <b>.xbm</b> files to maintain that irrigation demands will decrease with increasing flow.</li>
+    <li><b>.ddm</b>/<b>.eva</b>/<b>.opr</b> (demand, evaporation, operating rules): can be manipulated with multipliers (i.e. impose a 20% increase in municipal demands, increase reservoir evaporation, adjust operating rules)</li>
+    <li><b>.ddr</b> (water rights): can be manipulated to turn on/off users, and to change the seniority of users)</li>
+    <li><b>.res</b> (reservoir): can be manipulated to increase/decrease storage at specific reservoirs in each basin.</li>
+  </ul>
+
+  <p>The above are examples of methods that have been applied within our studies, but <b>statemodify</b> is flexible enough to accommodate new methods.</p></li>
+
+  <li>Once the input files are generated, we supply scripts that allow the user to run StateMod with a set of input files that represent plausible futures in parallel instances.</li>
+  <li>Finally, we store all relevant output in highly compressible parquet files that can be easily opened in Pandas dataframes. We also provide simple code to inspect shortages for specified users across futures.</li>
+  </ol>
+
+.. figure:: images/figure_1.png
+  :alt: **statemodify** workflow
+
+  Figure 1: The statemodify framework is comprised of four key steps (1) initializing an environment to run StateMod, (2) creating alternative plausible futures across five key West Slope Basins in Colorado, (3) running StateMod across all generated scenarios and (4) compiling results in compact data frames for easy visualization.
+
+Why do we need **statemodify**?
+--------------------------------
+
+Current Conditions
+++++++++++++++++++++++++++++
+
+In the Western United States, and particularly the Colorado River Basin, a recent study used tree-ring reconstructions to suggest that the megadrought that has been occurring in the Southwest over the past 22 years is the regions worst drought since about 800 AD (Williams et al., 2022). The recent trend of warming and reduced soil moisture in the Southwest US is becoming highly challenging from a water systems planning and management perspective for the Colorado River Basin. Given the wide recognition that the river is completely over-allocated, the most recent drought highlights the difficulty of sustaining the flow requirements as dictated by the Colorado Compact. Thus, there has been an increasing focus in utilizing exploratory modeling efforts to help actively inform Compact discussions and to clarify how vulnerable water systems in this region are to plausible future scenarios that may be characterized by changing climate and human-system dynamics.
+
+Planning and Management Models
++++++++++++++++++++++++++++++++
+
+The Colorado Water Conservation Board (CWCB) and the Division of Water Resources (DWR) have jointly developed Colorado's Decision Support System (CDSS), a collection of databases, data viewing and management tools, and models to support water resources planning in Colorado's major water basins (Malers et al., 2001). The CDSS is made up of a central database with water resources data (HydroBase), a public website where the data can be accessed, a Geographic Information System (GIS) for viewing and analyzing the data, and a consumptive use model (StateCU) that estimates consumptive use by each irrigation unit in a basin. The outputs from StateCU are then input to the State of Colorado's Stream Simulation Model (StateMod), a generic network-based water system model for water accounting and allocation, and the final component of CDSS. StateMod was developed to support comprehensive assessments of water demand, allocation, and use, as well as reservoir operations. It represents all of the major sub-basins within the state of Colorado (i.e., Parsons & Bennett, 2006; White, Yampa, Upper Colorado, Gunnison, Dolores, San Juan, and San Miguel CWCB, 2012). StateMod replicates each basin's unique application and enforcement of the prior appropriation doctrine and accounts for all of the consumptive use within the basins. To do so, it relies on the detailed historic demand and operation records contained in HydroBase that include individual water right information for all consumptive use, data on water structures (wells, ditches, reservoirs, and tunnels), and streamflow data. Further, StateMod uses irrigation consumptive use data output from StateCU, which calculates water consumption based on soil moisture, crop type, irrigated acreage, and conveyance and application efficiencies for each individual irrigation unit in the region.
+
+The Utility of StateMod
++++++++++++++++++++++++++++++++
+
+StateMod is one of the few highly resolved, open source, regional water allocation models that currently exists. Detailed, fine-scale inputs allow StateMod to resolve the effect of all users and water rights on water availability, which can, in turn, be used to simulate hypothetical scenarios to assess the impacts of changes in hydrology, water rights, or infrastructure on any and all represented water rights. A variety of studies have been conducted utilizing StateMod in an exploratory analysis (see Hadjimichael et al., 2020a, b). In order to facilitate these analyses, many methods have been developed to create templates from historical StateMod input files and adjust them (and their dependencies) to create new input files that represent plausible futures. The user can then run many StateMod simulations (i.e. thousands of future scenarios) in parallel and analyze the results.
+
+Prior to these studies, StateMod’s utility to be run efficiently in exploratory settings was limited, primarily due to the difficulty of adjusting the input files to represent future scenarios. Thus, we are proposing the development of a package called **statemodify** that neatly deploys this functionality and allows users of all coding levels to be able to interact with StateMod at a deeper and more exploratory level. By making StateMod more accessible, we also provide a better avenue for it to be used to actively inform current planning and management in the Colorado River basin by more general planners and decision-makers.
+
+
+References
+---------------------------
+
+CWCB (2012). Colorado River water availability study Phase I Report: Colorado Water Conservation Board.
+
+Hadjimichael, A., Quinn, J., Wilson, E., Reed, P., Basdekas, L., Yates, D., & Garrison, M. (2020). Defining robustness, vulnerabilities, and consequential scenarios for diverse stakeholder interests in institutionally complex river basins. Earth's Future, 8(7), e2020EF001503.
+
+Hadjimichael, A., Quinn, J., & Reed, P. (2020). Advancing diagnostic model evaluation to better understand water shortage mechanisms in institutionally complex river basins. Water Resources Research, 56(10), e2020WR028079.
+
+Malers, S. A., Bennett, R. R., & Nutting-Lane, C. (2000). Colorado's decision support systems: Data-centered water resources planning and administration. In Watershed Management and Operations Management 2000 (pp. 1-9).
+
+Parsons, R., & Bennett, R. (2006). Reservoir operations management using a water resources model. In Operating Reservoirs in Changing Conditions (pp. 304-311).
+
+
+
+Documentation
+--------------------------
+
+.. toctree::
+   :maxdepth: 1
+   :caption: Getting Started
+
+   getting-started/installation
+   getting-started/quickstarter
+   getting-started/tutorial
+
+.. toctree::
+   :maxdepth: 1
+   :caption: User Guides
+
+   examples/modify-inputs
+
+.. toctree::
+   :maxdepth: 1
+   :caption: Python API
+
+   reference/api
+
+.. toctree::
+   :maxdepth: 1
+   :caption: Contributing
+
+   reference/contributing