Merge pull request #87 from wwkimball/development

Prep 3.2.0 for release

CHANGES

@@ -1,3 +1,16 @@
+3.2.0:
+Enhancements:
+* Expanded YAML Path Search Expressions such that the OPERAND of a Search
+  Expression may be a sub-YAML Path.  This enables searching descendent nodes
+  -- without moving the document pointer -- to yield ancestors with matching
+  descendants.  This has more utility when searching against Arrays-of-Hashes.
+
+Bug Fixes:
+* Date values in YAML could not be written to JSON streams; the JSON renderer
+  would generate an incompatibility error.  Now, dates are written as Strings
+  to JSON.  This affected: yaml-get, yaml-set (in stream mode), yaml-merge, and
+  yaml-paths.
+
 3.1.0:
 Enhancements:
 * yaml-set can now delete nodes when using --delete rather than other input

README.md

@@ -195,6 +195,9 @@ YAML Path understands these segment types:
     `"` must be escaped lest they be deemed unmatched demarcation pairings)
   * Multi-level matching: `hash[name%admin].pass[encrypted!^ENC\[]` or
     `/hash[name%admin]/pass[encrypted!^ENC\[]`
+  * Descendent node searches:
+    `structure[has.descendant.with=something].has.another.field` or
+    `/structure[/has/descendant/with=something]/has/another/field`
 * Array element searches with all of the search methods above via `.` (yields
   any matching elements): `array[.>9000]`
 * Hash key-name searches with all of the search methods above via `.` (yields

tests/test_func.py

@@ -22,6 +22,7 @@
     get_yaml_editor,
     get_yaml_multidoc_data,
     make_new_node,
+    stringify_dates,
     wrap_type,
 )
 
@@ -355,3 +356,13 @@ def test_create_searchterms_from_pathattributes(self):
 
         with pytest.raises(AttributeError):
             _ = create_searchterms_from_pathattributes("nothing-to-see-here")
+
+
+    ###
+    # stringify_dates
+    ###
+    def test_stringify_dates(self):
+        from datetime import date
+        yaml_safe_data = { "string": "Value", "number": 1, "date": date(2020, 10, 19) }
+        json_safe_data = { "string": "Value", "number": 1, "date": "2020-10-19" }
+        assert stringify_dates(yaml_safe_data) == json_safe_data

tests/test_processor.py

@@ -1,4 +1,5 @@
 import pytest
+from datetime import date
 
 from ruamel.yaml import YAML
 
@@ -68,6 +69,8 @@ def test_get_none_data_nodes(self, quiet_logger):
         ("/**/Hey*", ["Hey, Number Two!"], True, None),
         ("lots_of_names.**.name", ["Name 1-1", "Name 2-1", "Name 3-1", "Name 4-1", "Name 4-2", "Name 4-3", "Name 4-4"], True, None),
         ("/array_of_hashes/**", [1, "one", 2, "two"], True, None),
+        ("products_hash.*[dimensions.weight==4].(availability.start.date)+(availability.stop.date)", [[date(2020, 8, 1), date(2020, 9, 25)], [date(2020, 1, 1), date(2020, 1, 1)]], True, None),
+        ("products_array[dimensions.weight==4].product", ["doohickey", "widget"], True, None),
     ])
     def test_get_nodes(self, quiet_logger, yamlpath, results, mustexist, default):
         yamldata = """---
@@ -124,6 +127,90 @@ def test_get_nodes(self, quiet_logger, yamlpath, results, mustexist, default):
           tag: Tag 4-4
           name: Name 4-4
           other: Other 4-4
+
+###############################################################################
+# For descendent searching:
+products_hash:
+  doodad:
+    availability:
+      start:
+        date: 2020-10-10
+        time: 08:00
+      stop:
+        date: 2020-10-29
+        time: 17:00
+    dimensions:
+      width: 5
+      height: 5
+      depth: 5
+      weight: 10
+  doohickey:
+    availability:
+      start:
+        date: 2020-08-01
+        time: 10:00
+      stop:
+        date: 2020-09-25
+        time: 10:00
+    dimensions:
+      width: 1
+      height: 2
+      depth: 3
+      weight: 4
+  widget:
+    availability:
+      start:
+        date: 2020-01-01
+        time: 12:00
+      stop:
+        date: 2020-01-01
+        time: 16:00
+    dimensions:
+      width: 9
+      height: 10
+      depth: 1
+      weight: 4
+products_array:
+  - product: doodad
+    availability:
+      start:
+        date: 2020-10-10
+        time: 08:00
+      stop:
+        date: 2020-10-29
+        time: 17:00
+    dimensions:
+      width: 5
+      height: 5
+      depth: 5
+      weight: 10
+  - product: doohickey
+    availability:
+      start:
+        date: 2020-08-01
+        time: 10:00
+      stop:
+        date: 2020-09-25
+        time: 10:00
+    dimensions:
+      width: 1
+      height: 2
+      depth: 3
+      weight: 4
+  - product: widget
+    availability:
+      start:
+        date: 2020-01-01
+        time: 12:00
+      stop:
+        date: 2020-01-01
+        time: 16:00
+    dimensions:
+      width: 9
+      height: 10
+      depth: 1
+      weight: 4
+###############################################################################
 """
         yaml = YAML()
         processor = Processor(quiet_logger, yaml.load(yamldata))

yamlpath/commands/yaml_get.py

@@ -15,7 +15,12 @@
 from os.path import isfile
 
 from yamlpath.common import YAMLPATH_VERSION
-from yamlpath.func import get_yaml_data, get_yaml_editor, unwrap_node_coords
+from yamlpath.func import (
+    get_yaml_data,
+    get_yaml_editor,
+    stringify_dates,
+    unwrap_node_coords,
+)
 from yamlpath import YAMLPath
 from yamlpath.exceptions import YAMLPathException
 from yamlpath.eyaml.exceptions import EYAMLCommandException
@@ -181,7 +186,7 @@ def main():
 
     for node in discovered_nodes:
         if isinstance(node, (dict, list)):
-            print(json.dumps(node))
+            print(json.dumps(stringify_dates(node)))
         else:
             print("{}".format(str(node).replace("\n", r"\n")))
 

yamlpath/commands/yaml_merge.py

@@ -22,7 +22,11 @@
     HashMergeOpts,
     OutputDocTypes,
 )
-from yamlpath.func import get_yaml_multidoc_data, get_yaml_editor
+from yamlpath.func import (
+    get_yaml_multidoc_data,
+    get_yaml_editor,
+    stringify_dates,
+)
 from yamlpath.merger.exceptions import MergeException
 from yamlpath.merger import Merger, MergerConfig
 from yamlpath.exceptions import YAMLPathException
@@ -294,12 +298,12 @@ def write_output_document(args, log, merger, yaml_editor):
     if args.output:
         with open(args.output, 'w') as out_fhnd:
             if document_is_json:
-                json.dump(merger.data, out_fhnd)
+                json.dump(stringify_dates(merger.data), out_fhnd)
             else:
                 yaml_editor.dump(merger.data, out_fhnd)
     else:
         if document_is_json:
-            json.dump(merger.data, sys.stdout)
+            json.dump(stringify_dates(merger.data), sys.stdout)
         else:
             yaml_editor.dump(merger.data, sys.stdout)
 

yamlpath/commands/yaml_paths.py

@@ -24,6 +24,7 @@
     get_yaml_editor,
     search_matches,
     search_anchor,
+    stringify_dates,
 )
 from yamlpath.exceptions import YAMLPathException
 from yamlpath.enums import (
@@ -699,7 +700,7 @@ def print_results(
             for node_coordinate in processor.get_nodes(result, mustexist=True):
                 node = node_coordinate.node
                 if isinstance(node, (dict, list)):
-                    resline += "{}".format(json.dumps(node))
+                    resline += "{}".format(json.dumps(stringify_dates(node)))
                 else:
                     resline += "{}".format(str(node).replace("\n", r"\n"))
                 break

yamlpath/commands/yaml_set.py

@@ -26,7 +26,8 @@
     clone_node,
     build_next_node,
     get_yaml_data,
-    get_yaml_editor
+    get_yaml_editor,
+    stringify_dates,
 )
 from yamlpath import YAMLPath
 from yamlpath.exceptions import YAMLPathException
@@ -248,7 +249,7 @@ def save_to_json_file(args, log, yaml_data):
     """Save to a JSON file."""
     log.verbose("Writing changed data as JSON to {}.".format(args.yaml_file))
     with open(args.yaml_file, 'w') as out_fhnd:
-        json.dump(yaml_data, out_fhnd)
+        json.dump(stringify_dates(yaml_data), out_fhnd)
 
 def save_to_yaml_file(args, log, yaml_parser, yaml_data, backup_file):
     """Save to a YAML file."""
@@ -321,7 +322,7 @@ def write_output_document(args, log, yaml, yaml_data):
         if write_document_as_yaml(args.yaml_file, yaml_data):
             yaml.dump(yaml_data, sys.stdout)
         else:
-            json.dump(yaml_data, sys.stdout)
+            json.dump(stringify_dates(yaml_data), sys.stdout)
     else:
         save_to_file(args, log, yaml, yaml_data, backup_file)
 

yamlpath/func.py

@@ -8,6 +8,7 @@
 import re
 from sys import maxsize, stdin
 from distutils.util import strtobool
+from datetime import date
 from typing import Any, Generator, List, Optional, Tuple
 
 from ruamel.yaml import YAML
@@ -720,3 +721,15 @@ def unwrap_node_coords(data: Any) -> Any:
         return stripped_nodes
 
     return data
+
+def stringify_dates(data: Any) -> Any:
+    """Recurse through a data structure, converting all dates to strings."""
+    if isinstance(data, dict):
+        for key, val in data.items():
+            data[key] = stringify_dates(val)
+    elif isinstance(data, list):
+        for idx, ele in enumerate(data):
+            data[idx] = stringify_dates(ele)
+    elif isinstance(data, date):
+        return str(data)
+    return data

yamlpath/merger/merger.py

@@ -15,6 +15,7 @@
 from yamlpath.func import (
     append_list_element,
     build_next_node,
+    stringify_dates,
 )
 from yamlpath.wrappers import ConsolePrinter, NodeCoords
 from yamlpath.merger.exceptions import MergeException
@@ -633,7 +634,7 @@ def prepare_for_dump(
             # Dump the document as true JSON and reload it; this automatically
             # exlodes all aliases.
             xfer_buffer = StringIO()
-            json.dump(self.data, xfer_buffer)
+            json.dump(stringify_dates(self.data), xfer_buffer)
             xfer_buffer.seek(0)
             self.data = yaml_writer.load(xfer_buffer)
 

yamlpath/processor.py

@@ -450,6 +450,7 @@ def _get_nodes_by_anchor(
                       and stripped_attrs == val.anchor.value):
                     yield NodeCoords(val, data, key)
 
+    # pylint: disable=too-many-statements
     def _get_nodes_by_search(
             self, data: Any, terms: SearchTerms, **kwargs: Any
     ) -> Generator[NodeCoords, None, None]:
@@ -487,6 +488,8 @@ def _get_nodes_by_search(
         method = terms.method
         attr = terms.attribute
         term = terms.term
+        matches = False
+        desc_path = YAMLPath(attr)
         if isinstance(data, list):
             if not traverse_lists:
                 return
@@ -496,6 +499,14 @@ def _get_nodes_by_search(
                     matches = search_matches(method, term, ele)
                 elif isinstance(ele, dict) and attr in ele:
                     matches = search_matches(method, term, ele[attr])
+                else:
+                    # Attempt a descendant search
+                    for desc_node in self._get_required_nodes(
+                        ele, desc_path, 0
+                    ):
+                        matches = search_matches(
+                            method, term, desc_node.node)
+                        break
 
                 if (matches and not invert) or (invert and not matches):
                     self.logger.debug(
@@ -528,6 +539,17 @@ def _get_nodes_by_search(
                         prefix="Processor::_get_nodes_by_search:  ")
                     yield NodeCoords(value, data, attr)
 
+            else:
+                # Attempt a descendant search
+                for desc_node in self._get_required_nodes(
+                    data, desc_path, 0, parent=parent, parentref=parentref
+                ):
+                    matches = search_matches(method, term, desc_node.node)
+                    break
+
+                if (matches and not invert) or (invert and not matches):
+                    yield NodeCoords(data, parent, parentref)
+
         else:
             # Check the passed data itself for a match
             matches = search_matches(method, term, data)