feat: model.XsUri migrate control characters according to spec (#498)

fixes #497 --------- Signed-off-by: Jan Kowalleck <jan.kowalleck@gmail.com>
CycloneDX · Dec 2, 2023 · e490429 · e490429
1 parent 78957e6
commit e490429
Show file tree

Hide file tree

Showing 12 changed files with 409 additions and 6 deletions.
diff --git a/cyclonedx/model/__init__.py b/cyclonedx/model/__init__.py
@@ -16,6 +16,7 @@
 import re
 from datetime import datetime, timezone
 from enum import Enum
+from functools import reduce
 from hashlib import sha1
 from itertools import zip_longest
 from typing import Any, Iterable, Optional, Tuple, TypeVar
@@ -424,16 +425,47 @@ class XsUri(serializable.helpers.BaseHelper):
 
     .. note::
         See XSD definition for xsd:anyURI: http://www.datypic.com/sc/xsd/t-xsd_anyURI.html
+        See JSON Schema definition for iri-reference: https://tools.ietf.org/html/rfc3987
     """
 
     _INVALID_URI_REGEX = re.compile(r'%(?![0-9A-F]{2})|#.*#', re.IGNORECASE + re.MULTILINE)
 
+    __SPEC_REPLACEMENTS = (
+        (' ', '%20'),
+        ('[', '%5B'),
+        (']', '%5D'),
+        ('<', '%3C'),
+        ('>', '%3E'),
+        ('{', '%7B'),
+        ('}', '%7D'),
+    )
+
+    @staticmethod
+    def __spec_replace(v: str, r: Tuple[str, str]) -> str:
+        return v.replace(*r)
+
+    @classmethod
+    def _spec_migrate(cls, o: str) -> str:
+        """
+         Make a string valid to
+         - XML::anyURI spec.
+         - JSON::iri-reference spec.
+
+         BEST EFFORT IMPLEMENTATION
+
+         @see http://www.w3.org/TR/xmlschema-2/#anyURI
+         @see http://www.datypic.com/sc/xsd/t-xsd_anyURI.html
+         @see https://datatracker.ietf.org/doc/html/rfc2396
+         @see https://datatracker.ietf.org/doc/html/rfc3987
+        """
+        return reduce(cls.__spec_replace, cls.__SPEC_REPLACEMENTS, o)
+
     def __init__(self, uri: str) -> None:
         if re.search(XsUri._INVALID_URI_REGEX, uri):
             raise InvalidUriException(
                 f"Supplied value '{uri}' does not appear to be a valid URI."
             )
-        self._uri = uri
+        self._uri = self._spec_migrate(uri)
 
     @property
     @serializable.json_name('.')

diff --git a/tests/_data/models.py b/tests/_data/models.py
@@ -754,6 +754,31 @@ def get_bom_with_multiple_licenses() -> Bom:
     )
 
 
+def get_bom_for_issue_497_urls() -> Bom:
+    """regression test for issue #497
+    see https://github.com/CycloneDX/cyclonedx-python-lib/issues/497
+    """
+    return _make_bom(components=[
+        Component(name='dummy', bom_ref='dummy', external_references=[
+            ExternalReference(
+                type=ExternalReferenceType.OTHER,
+                comment='nothing special',
+                url=XsUri('https://acme.org')
+            ),
+            ExternalReference(
+                type=ExternalReferenceType.OTHER,
+                comment='control characters',
+                url=XsUri('https://acme.org/?foo=sp ace&bar[23]=42&lt=1<2&gt=3>2&cb={lol}')
+            ),
+            ExternalReference(
+                type=ExternalReferenceType.OTHER,
+                comment='pre-encoded',
+                url=XsUri('https://acme.org/?bar%5b23%5D=42')
+            ),
+        ])
+    ])
+
+
 def bom_all_same_bomref() -> Tuple[Bom, int]:
     bom = Bom()
     bom.metadata.component = Component(name='root', bom_ref='foo', components=[
@@ -774,13 +799,18 @@ def bom_all_same_bomref() -> Tuple[Bom, int]:
     if n.startswith('get_bom_') and not n.endswith('_invalid')
 )
 
+all_get_bom_funct_valid_immut = tuple(
+    (n, f) for n, f in getmembers(sys.modules[__name__], isfunction)
+    if n.startswith('get_bom_') and not n.endswith('_invalid') and not n.endswith('_migrate')
+)
+
 all_get_bom_funct_invalid = tuple(
     (n, f) for n, f in getmembers(sys.modules[__name__], isfunction)
     if n.startswith('get_bom_') and n.endswith('_invalid')
 )
 
 all_get_bom_funct_with_incomplete_deps = {
-    # List of functions that return BOM with an incomplte dependency graph.
+    # List of functions that return BOM with an incomplete dependency graph.
     # It is expected that some process auto-fixes this before actual serialization takes place.
     get_bom_just_complete_metadata,
     get_bom_with_component_setuptools_basic,
@@ -797,4 +827,5 @@ def bom_all_same_bomref() -> Tuple[Bom, int]:
     get_bom_with_services_simple,
     get_bom_with_licenses,
     get_bom_with_multiple_licenses,
+    get_bom_for_issue_497_urls,
 }
diff --git a/tests/_data/snapshots/get_bom_for_issue_497_urls-1.0.xml.bin b/tests/_data/snapshots/get_bom_for_issue_497_urls-1.0.xml.bin
@@ -0,0 +1,10 @@
+<?xml version="1.0" ?>
+<bom xmlns="http://cyclonedx.org/schema/bom/1.0" version="1">
+  <components>
+    <component type="library">
+      <name>dummy</name>
+      <version/>
+      <modified>false</modified>
+    </component>
+  </components>
+</bom>
diff --git a/tests/_data/snapshots/get_bom_for_issue_497_urls-1.1.xml.bin b/tests/_data/snapshots/get_bom_for_issue_497_urls-1.1.xml.bin
@@ -0,0 +1,23 @@
+<?xml version="1.0" ?>
+<bom xmlns="http://cyclonedx.org/schema/bom/1.1" serialNumber="urn:uuid:1441d33a-e0fc-45b5-af3b-61ee52a88bac" version="1">
+  <components>
+    <component type="library" bom-ref="dummy">
+      <name>dummy</name>
+      <version/>
+      <externalReferences>
+        <reference type="other">
+          <url>https://acme.org</url>
+          <comment>nothing special</comment>
+        </reference>
+        <reference type="other">
+          <url>https://acme.org/?bar%5b23%5D=42</url>
+          <comment>pre-encoded</comment>
+        </reference>
+        <reference type="other">
+          <url>https://acme.org/?foo=sp%20ace&amp;bar%5B23%5D=42&amp;lt=1%3C2&amp;gt=3%3E2&amp;cb=%7Blol%7D</url>
+          <comment>control characters</comment>
+        </reference>
+      </externalReferences>
+    </component>
+  </components>
+</bom>
diff --git a/tests/_data/snapshots/get_bom_for_issue_497_urls-1.2.json.bin b/tests/_data/snapshots/get_bom_for_issue_497_urls-1.2.json.bin
@@ -0,0 +1,47 @@
+{
+  "components": [
+    {
+      "bom-ref": "dummy",
+      "externalReferences": [
+        {
+          "comment": "nothing special",
+          "type": "other",
+          "url": "https://acme.org"
+        },
+        {
+          "comment": "pre-encoded",
+          "type": "other",
+          "url": "https://acme.org/?bar%5b23%5D=42"
+        },
+        {
+          "comment": "control characters",
+          "type": "other",
+          "url": "https://acme.org/?foo=sp%20ace&bar%5B23%5D=42&lt=1%3C2&gt=3%3E2&cb=%7Blol%7D"
+        }
+      ],
+      "name": "dummy",
+      "type": "library",
+      "version": ""
+    }
+  ],
+  "dependencies": [
+    {
+      "ref": "dummy"
+    }
+  ],
+  "metadata": {
+    "timestamp": "2023-01-07T13:44:32.312678+00:00",
+    "tools": [
+      {
+        "name": "cyclonedx-python-lib",
+        "vendor": "CycloneDX",
+        "version": "TESTING"
+      }
+    ]
+  },
+  "serialNumber": "urn:uuid:1441d33a-e0fc-45b5-af3b-61ee52a88bac",
+  "version": 1,
+  "$schema": "http://cyclonedx.org/schema/bom-1.2b.schema.json",
+  "bomFormat": "CycloneDX",
+  "specVersion": "1.2"
+}
diff --git a/tests/_data/snapshots/get_bom_for_issue_497_urls-1.2.xml.bin b/tests/_data/snapshots/get_bom_for_issue_497_urls-1.2.xml.bin
@@ -0,0 +1,36 @@
+<?xml version="1.0" ?>
+<bom xmlns="http://cyclonedx.org/schema/bom/1.2" serialNumber="urn:uuid:1441d33a-e0fc-45b5-af3b-61ee52a88bac" version="1">
+  <metadata>
+    <timestamp>2023-01-07T13:44:32.312678+00:00</timestamp>
+    <tools>
+      <tool>
+        <vendor>CycloneDX</vendor>
+        <name>cyclonedx-python-lib</name>
+        <version>TESTING</version>
+      </tool>
+    </tools>
+  </metadata>
+  <components>
+    <component type="library" bom-ref="dummy">
+      <name>dummy</name>
+      <version/>
+      <externalReferences>
+        <reference type="other">
+          <url>https://acme.org</url>
+          <comment>nothing special</comment>
+        </reference>
+        <reference type="other">
+          <url>https://acme.org/?bar%5b23%5D=42</url>
+          <comment>pre-encoded</comment>
+        </reference>
+        <reference type="other">
+          <url>https://acme.org/?foo=sp%20ace&amp;bar%5B23%5D=42&amp;lt=1%3C2&amp;gt=3%3E2&amp;cb=%7Blol%7D</url>
+          <comment>control characters</comment>
+        </reference>
+      </externalReferences>
+    </component>
+  </components>
+  <dependencies>
+    <dependency ref="dummy"/>
+  </dependencies>
+</bom>
diff --git a/tests/_data/snapshots/get_bom_for_issue_497_urls-1.3.json.bin b/tests/_data/snapshots/get_bom_for_issue_497_urls-1.3.json.bin
@@ -0,0 +1,47 @@
+{
+  "components": [
+    {
+      "bom-ref": "dummy",
+      "externalReferences": [
+        {
+          "comment": "nothing special",
+          "type": "other",
+          "url": "https://acme.org"
+        },
+        {
+          "comment": "pre-encoded",
+          "type": "other",
+          "url": "https://acme.org/?bar%5b23%5D=42"
+        },
+        {
+          "comment": "control characters",
+          "type": "other",
+          "url": "https://acme.org/?foo=sp%20ace&bar%5B23%5D=42&lt=1%3C2&gt=3%3E2&cb=%7Blol%7D"
+        }
+      ],
+      "name": "dummy",
+      "type": "library",
+      "version": ""
+    }
+  ],
+  "dependencies": [
+    {
+      "ref": "dummy"
+    }
+  ],
+  "metadata": {
+    "timestamp": "2023-01-07T13:44:32.312678+00:00",
+    "tools": [
+      {
+        "name": "cyclonedx-python-lib",
+        "vendor": "CycloneDX",
+        "version": "TESTING"
+      }
+    ]
+  },
+  "serialNumber": "urn:uuid:1441d33a-e0fc-45b5-af3b-61ee52a88bac",
+  "version": 1,
+  "$schema": "http://cyclonedx.org/schema/bom-1.3a.schema.json",
+  "bomFormat": "CycloneDX",
+  "specVersion": "1.3"
+}
diff --git a/tests/_data/snapshots/get_bom_for_issue_497_urls-1.3.xml.bin b/tests/_data/snapshots/get_bom_for_issue_497_urls-1.3.xml.bin
@@ -0,0 +1,36 @@
+<?xml version="1.0" ?>
+<bom xmlns="http://cyclonedx.org/schema/bom/1.3" serialNumber="urn:uuid:1441d33a-e0fc-45b5-af3b-61ee52a88bac" version="1">
+  <metadata>
+    <timestamp>2023-01-07T13:44:32.312678+00:00</timestamp>
+    <tools>
+      <tool>
+        <vendor>CycloneDX</vendor>
+        <name>cyclonedx-python-lib</name>
+        <version>TESTING</version>
+      </tool>
+    </tools>
+  </metadata>
+  <components>
+    <component type="library" bom-ref="dummy">
+      <name>dummy</name>
+      <version/>
+      <externalReferences>
+        <reference type="other">
+          <url>https://acme.org</url>
+          <comment>nothing special</comment>
+        </reference>
+        <reference type="other">
+          <url>https://acme.org/?bar%5b23%5D=42</url>
+          <comment>pre-encoded</comment>
+        </reference>
+        <reference type="other">
+          <url>https://acme.org/?foo=sp%20ace&amp;bar%5B23%5D=42&amp;lt=1%3C2&amp;gt=3%3E2&amp;cb=%7Blol%7D</url>
+          <comment>control characters</comment>
+        </reference>
+      </externalReferences>
+    </component>
+  </components>
+  <dependencies>
+    <dependency ref="dummy"/>
+  </dependencies>
+</bom>