Merge pull request #94 from wwkimball/development

Prep release 3.3.0

.codacy.yaml

@@ -2,8 +2,11 @@
 engines:
   pylint:
     enabled: false
+  pylintpython3:
+    enabled: false
 
 exclude_paths:
   - 'tests/**'
   - '**/*.md'
   - '**/__init__.py'
+  - 'README.md'

.coveragerc

@@ -1,4 +1,11 @@
 [run]
+# Enable multi-threaded processing of tests and coverage scanning
+parallel = true
+
+# Move .coverage and all temporary .coverage* files to a temporary directory
+data_file = /tmp/yamlpath-python-coverage-data
+
+# Ignore these files from coverage analysis
 omit =
 	# Don't bother with ruamel.yaml patches (not my code)
 	yamlpath/patches/emitter_write_folded_fix.py

.gitignore

@@ -105,3 +105,6 @@ venv*.bak/
 
 # IDE configurations
 .vscode/
+.vscode-env*/
+.vscode-venv*/
+

.travis.yml

@@ -1,5 +1,3 @@
-sudo: false
-
 language:
   - python
 
@@ -8,12 +6,19 @@ python:
   - 3.7
   - 3.8
 
+os: linux
+dist: xenial
+
+# Note:  pytest-cov requires the module to be installed with --editable lest it
+# will not cover all code-paths!
 install:
-  - pip install --upgrade ruamel.yaml mypy pytest pytest-cov pytest-console-scripts pylint coveralls pep257
+  - python -m pip install --upgrade pip
+  - pip install --upgrade setuptools
+  - pip install --editable .
+  - pip install --upgrade mypy pytest pytest-cov pytest-console-scripts pylint coveralls pep257
   - gem install hiera-eyaml -v 2.1.0
 
 script:
-  - pip install -e .
   - pep257 yamlpath
   - mypy yamlpath
   - pylint yamlpath
@@ -22,7 +27,7 @@ script:
 
 deploy:
   provider: pypi
-  user: wwkimball
+  username: wwkimball
   password:
     secure: "cBWRVvQibxw2KTYJoruKeOA1vNDk2+ZuZsAzW9hq3bk2Ab4GDy7G8pYRhVhiHNMDY0w5+Ii66/i0ERpBC95bi7Mzy/NOzbhoYflr+lDOoC6YR76VpCyHJQ/I3YmyTjSeWqB86JLfmQNi9RJWBbZYWpsCuEjBRP7w6oOf2QdAefQUctJ9g69Gt5r7okNm4nfbiAwvUAjGBT7dSRKPrMmRhP8Qc5ugXrO3xNnh9xCBqBkvXfDQjlA3YCi3RiuWJQHkwIhHjfwoDXzis4bprBQUOAZ/+c6WiCvEsMsG1551Ilt7Q1x6aiu98m4hYkZMTIl5IjCMSXbnlOSdlv0XvO8jSq/+GGSdI6EbLAYC1MTnob7w1/NQSsVfz6dKl/pMCRC2J+OGNqv5a3Ln3rzeOTs9opZunHQ34Ro+MxEKSf+XdU48YJR1DlDLWYV9tbiVFG+tSybu3uxcs825EB4aNzL7pMBaIzMpEjhsDNEDZ90T6jDnLzJHYFgH3yj9ortpjYI3CZdvzsVqQrTT1zXzKzMbW6/o0U8azzGQWu24Bn6HSe3Cr0PQQpalSyN/usZziBwLYVj42d3tCtg1xGNUvzgX1mRst8Ltsag4g4OG54IJ4RihcviQWLY6Jgjnob5KU3JcCYzKJjKu5cXyuK2ec7lMCBRaVo6qaiP1iVPNLpXsqz4="
   skip_existing: true

CHANGES

@@ -1,3 +1,37 @@
+3.3.0:
+Bug Fixes:
+* It was impossible to install yamlpath 3.x without first installing
+  ruamel.yaml via pip for Python 3.x.  Not only has this been fixed but
+  explicit tests have been created to ensure this never happens again.
+
+Enhancements:
+* A new command-line tool, yaml-diff, now compares exactly two
+  YAML/JSON/Compatible documents, producing a GNU diff-like report of any
+  differences in the data they present to parsers.  Along with diff's "a"
+  (added), "c" (changed), and "d" (deleted) report entries, affected YAML Paths
+  are printed in lieu of line numbers.  Further, a report entry of "s" (same)
+  is available and can be enabled via command-line options.  This tool also
+  features optional special handling of Arrays and Arrays-of-Hashes, which can
+  be configured as CLI options or via an INI file for distinct settings per
+  YAML Path.  See --help or the Wiki for more detail.
+
+API Changes:
+* NodeCoords now employ a new `path` attribute.  This is an optional parameter
+  which is assigned during construction to later report the translated origin
+  YAML Path; this is where the node was found or created within the DOM.  Note
+  that Collector segments work against virtual DOMs, so the YAML Path of an
+  outer Collector will be virtual, relative to its parent at construction; when
+  nested, this will be a bare list index.  Any NodeCoords in the virtual
+  container which point to real nodes in the DOM will have their own concrete
+  YAML Paths.
+* YAMLPath instances now support nonmutating addition of individual segments
+  via the + operator.  Whereas the append() method mutates the YAMLPath being
+  acted upon, + creates a new YAMLPath that is the original plus the new
+  segment.  In both cases, the orignal YAMLPath's seperator is retained during
+  both operations.  As with .append(), new segments added via + must also be
+  properly escaped -- typically via path.escape_path_section -- before being
+  added.
+
 3.2.0:
 Enhancements:
 * Expanded YAML Path Search Expressions such that the OPERAND of a Search

README.md

@@ -30,11 +30,13 @@ for other projects to readily employ YAML Paths.
 7. [Basic Usage](#basic-usage)
    1. [Basic Usage:  Command-Line Tools](#basic-usage--command-line-tools)
       1. [Rotate Your EYAML Keys](#rotate-your-eyaml-keys)
-      2. [Get a YAML/JSON/Compatible Value](#get-a-yamljsoncompatible-value)
-      3. [Search For YAML Paths](#search-for-yaml-paths)
-      4. [Change a YAML/JSON/Compatible Value](#change-a-yamljsoncompatible-value)
-      5. [Merge YAML/JSON/Compatible Files](#merge-yamljsoncompatible-files)
-      6. [Validate YAML/JSON/Compatible Documents](#validate-yamljsoncompatible-documents)
+      2. [Get the Differences Between Two Documents](#get-the-differences-between-two-documents)
+      3. [Get a YAML/JSON/Compatible Value](#get-a-yamljsoncompatible-value)
+      4. [Search For YAML Paths](#search-for-yaml-paths)
+      5. [Change a YAML/JSON/Compatible Value](#change-a-yamljsoncompatible-value)
+      6. [Merge YAML/JSON/Compatible Files](#merge-yamljsoncompatible-files)
+      7. [Validate YAML/JSON/Compatible Documents](#validate-yamljsoncompatible-documents)
+
    2. [Basic Usage:  Libraries](#basic-usage--libraries)
       1. [Initialize ruamel.yaml and These Helpers](#initialize-ruamelyaml-and-these-helpers)
       2. [Searching for YAML Nodes](#searching-for-yaml-nodes)
@@ -345,6 +347,63 @@ Any YAML_FILEs lacking EYAML values will not be modified (or backed up, even
 when -b/--backup is specified).
 ```
 
+* [yaml-diff](yamlpath/commands/yaml_diff.py)
+
+```text
+usage: yaml-diff [-h] [-V] [-a] [-s | -o]
+                 [-t ['.', '/', 'auto', 'dot', 'fslash']] [-x EYAML]
+                 [-r PRIVATEKEY] [-u PUBLICKEY] [-E] [-d | -v | -q]
+                 YAML_FILE YAML_FILE
+
+Calculate the functional difference between two YAML/JSON/Compatible
+documents. Immaterial differences (which YAML/JSON parsers discard) are
+ignored. EYAML can be employed to compare encrypted values.
+
+positional arguments:
+  YAML_FILE             exactly two YAML/JSON/compatible files to compare; use
+                        - to read one document from STDIN
+
+optional arguments:
+  -h, --help            show this help message and exit
+  -V, --version         show program's version number and exit
+  -a, --sync-arrays     Synchronize array elements before comparing them,
+                        resulting only in ADD, DELETE, and SAME differences
+                        (no CHANGEs because the positions of elements are
+                        disregarded); Array-of-Hash elements must completely
+                        and perfectly match or they will be deemed additions
+                        or deletions
+  -s, --same            Show all nodes which are the same in addition to
+                        differences
+  -o, --onlysame        Show only nodes which are the same, still reporting
+                        that differences exist -- when they do -- with an
+                        exit-state of 1
+  -t ['.', '/', 'auto', 'dot', 'fslash'], --pathsep ['.', '/', 'auto', 'dot', 'fslash']
+                        indicate which YAML Path seperator to use when
+                        rendering results; default=dot
+  -d, --debug           output debugging details
+  -v, --verbose         increase output verbosity
+  -q, --quiet           suppress all output except system errors
+
+EYAML options:
+  Left unset, the EYAML keys will default to your system or user defaults.
+  Both keys must be set either here or in your system or user EYAML
+  configuration file when using EYAML.
+
+  -x EYAML, --eyaml EYAML
+                        the eyaml binary to use when it isn't on the PATH
+  -r PRIVATEKEY, --privatekey PRIVATEKEY
+                        EYAML private key
+  -u PUBLICKEY, --publickey PUBLICKEY
+                        EYAML public key
+  -E, --ignore-eyaml-values
+                        Do not use EYAML to compare encrypted data; rather,
+                        treat ENC[...] values as regular strings
+
+Only one YAML_FILE may be the - pseudo-file for reading from STDIN. For more
+information about YAML Paths, please visit
+https://github.com/wwkimball/yamlpath.
+```
+
 * [yaml-get](yamlpath/commands/yaml_get.py)
 
 ```text
@@ -749,6 +808,43 @@ fail to decrypt your data!**  This is *not* a problem with YAML Path.
 hiera-eyaml certificate compatibility is well outside the purview of YAML Path
 and its tools.
 
+#### Get the Differences Between Two Documents
+
+For routine use:
+
+```shell
+yaml-diff yaml_file1.yaml yaml_file2.yaml
+```
+
+Output is very similar to that of standard GNU diff against text files, except
+it is generated against the *data* within the input files.  This excludes
+evaluating purely structural and immaterial differences between them like value
+demarcation, white-space, and comments.  When you need to evaluate the two
+files as if they were just text files, use GNU diff or any of its clones.
+
+To see all identical entries along with differences:
+
+```shell
+yaml-diff --same yaml_file1.yaml yaml_file2.yaml
+```
+
+To see *only* entries which are identical between the documents:
+
+```shell
+yaml-diff --onlysame yaml_file1.yaml yaml_file2.yaml
+```
+
+Advanced:  Arrays can be evaluated such that they are synchronized before
+evaluation.  Rather than compare elements by identical index in both
+documents -- reporting differences between them as changes and any additional
+elements as additions or deletions -- they can instead be compared by matching
+up all identical elements and then reporting only those values which are unique
+to either document (and optionally identical matches).
+
+```shell
+yaml-diff --sync-arrays yaml_file1.yaml yaml_file2.yaml
+```
+
 #### Get a YAML/JSON/Compatible Value
 
 At its simplest:

run-tests.ps1

@@ -5,12 +5,24 @@ if (-Not $HasTestsDir -Or -Not $HasProjectDir) {
     exit 2
 }
 
+# Credit: https://stackoverflow.com/a/54935264
+function New-TemporaryDirectory {
+    [CmdletBinding(SupportsShouldProcess = $true)]
+    param()
+    $parent = [System.IO.Path]::GetTempPath()
+    do {
+        $name = [System.IO.Path]::GetRandomFileName()
+        $item = New-Item -Path $parent -Name $name -ItemType "directory" -ErrorAction SilentlyContinue
+    } while (-not $item)
+    return $Item
+}
+
 $EnvDirs = Get-ChildItem -Directory -Filter "env*"
 ForEach ($EnvDir in $EnvDirs) {
     & "$($EnvDir.FullName)\Scripts\Activate.ps1"
     if (!$?) {
         Write-Error "`nERROR:  Unable to activate $EnvDir!"
-        exit 20
+        continue
     }
 
     $PythonVersion = $(python --version)
@@ -21,43 +33,75 @@ ForEach ($EnvDir in $EnvDirs) {
         =========================================================================
 "@
 
+    Write-Output "...spawning a new temporary Virtual Environment..."
+    $TmpVEnv = New-TemporaryDirectory
+    python -m venv $TmpVEnv
+    if (!$?) {
+        Write-Error "`nERROR:  Unable to spawn a new temporary virtual environment at $TmpVEnv!"
+        exit 125
+    }
+    & deactivate
+    & "$($TmpVEnv.FullName)\Scripts\Activate.ps1"
+    if (!$?) {
+        Write-Error "`nERROR:  Unable to activate $TmpVEnv!"
+        continue
+    }
+
     Write-Output "...upgrading pip"
     python -m pip install --upgrade pip
-    Write-Output "...reinstalling ruamel.yaml (because pip upgrades break it)"
-    pip install --force-reinstall ruamel.yaml==0.15.96
-    Write-Output "...upgrading testing tools"
-    pip install --upgrade mypy pytest pytest-cov pytest-console-scripts pylint coveralls pep257
+
+    Write-Output "...upgrading setuptools"
+    pip install --upgrade setuptools
+
     Write-Output "...installing self"
     pip install -e .
+    if (!$?) {
+        & deactivate
+        Remove-Item -Recurse -Force $TmpVEnv
+        Write-Error "`nERROR:  Unable to install self!"
+        exit 124
+    }
+
+    Write-Output "...upgrading testing tools"
+    pip install --upgrade mypy pytest pytest-cov pytest-console-scripts pylint coveralls pep257
 
     Write-Output "`nPEP257..."
     pep257 yamlpath | Out-String
     if (!$?) {
+        & deactivate
+        Remove-Item -Recurse -Force $TmpVEnv
         Write-Error "PEP257 Error: $?"
         exit 9
     }
 
     Write-Output "`nMYPY..."
     mypy yamlpath | Out-String
     if (!$?) {
+        & deactivate
+        Remove-Item -Recurse -Force $TmpVEnv
         Write-Error "MYPY Error: $?"
         exit 10
     }
 
     Write-Output "`nPYLINT..."
     pylint yamlpath | Out-String
     if (!$?) {
+        & deactivate
+        Remove-Item -Recurse -Force $TmpVEnv
         Write-Error "PYLINT Error: $?"
         exit 11
     }
 
     Write-Output "`n PYTEST..."
     pytest -vv --cov=yamlpath --cov-report=term-missing --cov-fail-under=100 --script-launch-mode=subprocess tests
     if (!$?) {
+        & deactivate
+        Remove-Item -Recurse -Force $TmpVEnv
         Write-Error "PYTEST Error: $?"
         exit 12
     }
-}
 
-Write-Output "Deactivating virtual Python environment..."
-& deactivate
+    Write-Output "Deactivating virtual Python environment..."
+    & deactivate
+    Remove-Item -Recurse -Force $TmpVEnv
+}