UW-523 Support Jinja2 template loading (#426)

docs/sections/user_guide/cli/tools/mode_template.rst

@@ -30,8 +30,9 @@ The ``uw`` mode for handling :jinja2:`Jinja2 templates<templates>`.
 
    $ uw template render --help
    usage: uw template render [-h] [--input-file PATH] [--output-file PATH] [--values-file PATH]
-                             [--values-format {ini,nml,sh,yaml}] [--env] [--values-needed]
-                             [--partial] [--dry-run] [--quiet] [--verbose]
+                             [--values-format {ini,nml,sh,yaml}] [--env]
+                             [--search-path PATH[:PATH:...]] [--values-needed] [--partial]
+                             [--dry-run] [--quiet] [--verbose]
                              [KEY=VALUE ...]
 
    Render a template
@@ -49,6 +50,8 @@ The ``uw`` mode for handling :jinja2:`Jinja2 templates<templates>`.
          Values format
      --env
          Use environment variables
+     --search-path PATH[:PATH:...]
+         Colon-separated paths to search for extra templates
      --values-needed
          Print report of values needed to render template
      --partial
@@ -186,9 +189,9 @@ and a YAML file called ``values.yaml`` with the following contents:
      [2023-12-18T23:27:04]    DEBUG ---------------------------------------------------------------------
      [2023-12-18T23:27:04]    DEBUG Read initial values from values.yaml
 
-**NB**: The following examples are based on a ``values.yaml`` file with ``recipient: World`` removed.
+* **NB**: This set of examples is based on a ``values.yaml`` file with ``recipient: World`` removed.
 
-* It is an error to render a template without providing all needed values.
+  It is an error to render a template without providing all needed values.
 
   .. code-block:: text
 
@@ -228,6 +231,42 @@ and a YAML file called ``values.yaml`` with the following contents:
 
   Note that ``recipient=Sunshine`` is shell syntax for exporting environment variable ``recipient`` only for the duration of the command that follows. It should not be confused with the two ``key=value`` pairs later on the command line, which are arguments to ``uw``.
 
+* Jinja2 supports references to additional templates via, for example, `import <https://jinja.palletsprojects.com/en/latest/templates/#import>`_ expressions, and ``uw`` provides support as follows:
+
+  #. By default, the directory containing the primary template file is used as the search path for additional templates.
+  #. The optional ``--search-path`` flag overrides the default search path with any number of explicitly specified, colon-separated paths.
+
+  For example, given file ``template``
+
+  .. code-block:: text
+
+     {% import "macros" as m -%}
+     {{ m.double(11) }}
+
+  and file ``macros`` (in the same directory as ``template``)
+
+  .. code-block:: text
+
+     {% macro double(n) -%}
+     {{ n * 2 }}
+     {%- endmacro %}
+
+  the template is rendered as
+
+  .. code-block:: text
+
+     $ uw template render --input-file template
+     22
+
+  The invocation ``uw template render --input-file template --search-path $PWD`` would behave identically. Alternatively, ``--search-path`` could be specified with a colon-separated set of directories to be searched for templates.
+
+  **NB**: Reading the primary template from ``stdin`` requires use of ``--search-path``, as there is no implicit directory related to the input. For example, given the existence of ``/path/to/macros``:
+
+  .. code-block:: text
+
+     $ cat template | uw template render --search-path /path/to
+     22
+
 * Non-YAML-formatted files may also be used as value sources. For example, ``template``
 
   .. code-block:: jinja

src/uwtools/api/template.py

@@ -3,7 +3,7 @@
 """
 import os
 from pathlib import Path
-from typing import Dict, Optional, Union
+from typing import Dict, List, Optional, Union
 
 from uwtools.config.atparse_to_jinja2 import convert as _convert_atparse_to_jinja2
 from uwtools.config.jinja2 import render as _render
@@ -17,6 +17,7 @@ def render(
     output_file: Optional[Path] = None,
     overrides: Optional[Dict[str, str]] = None,
     env: bool = False,
+    searchpath: Optional[List[str]] = None,
     values_needed: bool = False,
     partial: bool = False,
     dry_run: bool = False,
@@ -38,6 +39,7 @@ def render(
         to ``stdout``)
     :param overrides: Supplemental override values
     :param env: Supplement values with environment variables?
+    :param searchpath: Paths to search for extra templates
     :param values_needed: Just report variables needed to render the template?
     :param partial: Permit unrendered Jinja2 variables/expressions in output?
     :param dry_run: Run in dry-run mode?
@@ -51,6 +53,7 @@ def render(
         output_file=output_file,
         overrides=overrides,
         env=env,
+        searchpath=searchpath,
         values_needed=values_needed,
         partial=partial,
         dry_run=dry_run,
@@ -66,6 +69,7 @@ def render_to_str(  # pylint: disable=unused-argument
     input_file: Optional[Path] = None,
     overrides: Optional[Dict[str, str]] = None,
     env: bool = False,
+    searchpath: Optional[List[str]] = None,
     values_needed: bool = False,
     partial: bool = False,
     dry_run: bool = False,

src/uwtools/cli.py

@@ -442,6 +442,7 @@ def _add_subparser_template_render(subparsers: Subparsers) -> ActionChecks:
     _add_arg_values_file(optional)
     _add_arg_values_format(optional, choices=FORMATS)
     _add_arg_env(optional)
+    _add_arg_search_path(optional)
     _add_arg_values_needed(optional)
     _add_arg_partial(optional)
     _add_arg_dry_run(optional)
@@ -478,6 +479,7 @@ def _dispatch_template_render(args: Args) -> bool:
             output_file=args[STR.outfile],
             overrides=_dict_from_key_eq_val_strings(args[STR.keyvalpairs]),
             env=args[STR.env],
+            searchpath=args[STR.searchpath],
             values_needed=args[STR.valsneeded],
             partial=args[STR.partial],
             dry_run=args[STR.dryrun],
@@ -661,6 +663,16 @@ def _add_arg_schema_file(group: Group) -> None:
     )
 
 
+def _add_arg_search_path(group: Group) -> None:
+    group.add_argument(
+        _switch(STR.searchpath),
+        help="Colon-separated paths to search for extra templates",
+        metavar="PATH[:PATH:...]",
+        required=False,
+        type=lambda s: s.split(":"),
+    )
+
+
 def _add_arg_supplemental_files(group: Group) -> None:
     group.add_argument(
         STR.suppfiles,
@@ -894,6 +906,7 @@ class STR:
     rocoto: str = "rocoto"
     run: str = "run"
     schemafile: str = "schema_file"
+    searchpath: str = "search_path"
     sfcclimogen: str = "sfc_climo_gen"
     suppfiles: str = "supplemental_files"
     task: str = "task"

src/uwtools/config/jinja2.py

@@ -3,19 +3,11 @@
 """
 
 import os
+from functools import cached_property
 from pathlib import Path
 from typing import Dict, List, Optional, Set, Union
 
-from jinja2 import (
-    BaseLoader,
-    DebugUndefined,
-    Environment,
-    FileSystemLoader,
-    StrictUndefined,
-    Template,
-    Undefined,
-    meta,
-)
+from jinja2 import DebugUndefined, Environment, FileSystemLoader, StrictUndefined, Undefined, meta
 from jinja2.exceptions import UndefinedError
 
 from uwtools.config.support import TaggedString, format_to_config
@@ -30,19 +22,44 @@ class J2Template:
     Reads Jinja2 templates from files or strings, and renders them using the user-provided values.
     """
 
-    def __init__(self, values: dict, template_source: Union[str, Path]) -> None:
+    def __init__(
+        self,
+        values: dict,
+        template_source: Optional[Union[str, Path]] = None,
+        searchpath: Optional[List[str]] = None,
+    ) -> None:
         """
         :param values: Values needed to render the provided template.
         :param template_source: Jinja2 string or template file path (None => read stdin).
+        :param searchpath: Colon-separated paths to search for extra templates.
         :raises: RuntimeError: If neither a template file or path is provided.
         """
         self._values = values
-        self._template = (
-            self._load_string(template_source)
-            if isinstance(template_source, str)
-            else self._load_file(template_source)
-        )
         self._template_source = template_source
+        self._j2env = Environment(
+            loader=FileSystemLoader(
+                searchpath=searchpath
+                if searchpath
+                else self._template_source.parent
+                if isinstance(self._template_source, Path)
+                else []
+            )
+        )
+        _register_filters(self._j2env)
+        self._template = self._j2env.from_string(self._template_str)
+
+    def __repr__(self):
+        return self._template_str
+
+    @cached_property
+    def _template_str(self):
+        """
+        A string containing the template.
+        """
+        if isinstance(self._template_source, str):
+            return self._template_source
+        with readable(self._template_source) as f:
+            return f.read()
 
     # Public methods
 
@@ -72,37 +89,9 @@ def undeclared_variables(self) -> Set[str]:
 
         :return: Names of variables needed to render the template.
         """
-        if isinstance(self._template_source, str):
-            j2_parsed = self._j2env.parse(self._template_source)
-        else:
-            with open(self._template_source, "r", encoding="utf-8") as f:
-                j2_parsed = self._j2env.parse(f.read())
+        j2_parsed = self._j2env.parse(self._template_str)
         return meta.find_undeclared_variables(j2_parsed)
 
-    # Private methods
-
-    def _load_file(self, template_path: Path) -> Template:
-        """
-        Load the Jinja2 template from the file provided.
-
-        :param template_path: Filesystem path to the Jinja2 template file.
-        :return: The Jinja2 template object.
-        """
-        self._j2env = Environment(loader=FileSystemLoader(searchpath="/"))
-        _register_filters(self._j2env)
-        return self._j2env.get_template(str(template_path))
-
-    def _load_string(self, template: str) -> Template:
-        """
-        Load the Jinja2 template from the string provided.
-
-        :param template: An in-memory Jinja2 template.
-        :return: The Jinja2 template object.
-        """
-        self._j2env = Environment(loader=BaseLoader())
-        _register_filters(self._j2env)
-        return self._j2env.from_string(template)
-
 
 # Public functions
 
@@ -150,6 +139,7 @@ def render(
     output_file: Optional[Path] = None,
     overrides: Optional[Dict[str, str]] = None,
     env: bool = False,
+    searchpath: Optional[List[str]] = None,
     values_needed: bool = False,
     partial: bool = False,
     dry_run: bool = False,
@@ -163,6 +153,7 @@ def render(
     :param output_file: Path to write rendered Jinja2 template to (None => write to stdout).
     :param overrides: Supplemental override values.
     :param env: Supplement values with environment variables?
+    :param searchpath: Paths to search for extra templates.
     :param values_needed: Just report variables needed to render the template?
     :param partial: Permit unrendered Jinja2 variables/expressions in output?
     :param dry_run: Run in dry-run mode?
@@ -172,9 +163,7 @@ def render(
     values = _supplement_values(
         values_src=values_src, values_format=values_format, overrides=overrides, env=env
     )
-    with readable(input_file) as f:
-        template_str = f.read()
-    template = J2Template(values=values, template_source=template_str)
+    template = J2Template(values=values, template_source=input_file, searchpath=searchpath)
     undeclared_variables = template.undeclared_variables
 
     # If a report of variables required to render the template was requested, make that report and
@@ -188,7 +177,7 @@ def render(
     # missing values and return an error to the caller.
 
     if partial:
-        rendered = Environment(undefined=DebugUndefined).from_string(template_str).render(values)
+        rendered = Environment(undefined=DebugUndefined).from_string(str(template)).render(values)
     else:
         missing = [var for var in undeclared_variables if var not in values.keys()]
         if missing:

src/uwtools/tests/api/test_template.py

@@ -18,6 +18,7 @@ def kwargs():
         "values_src": "valsfile",
         "values_format": "format",
         "overrides": {"key": "val"},
+        "searchpath": None,
         "env": True,
         "partial": True,
         "values_needed": True,