diff --git a/.appveyor.yml b/.appveyor.yml deleted file mode 100644 index 65358bdb..00000000 --- a/.appveyor.yml +++ /dev/null @@ -1,33 +0,0 @@ -# TODO: update this from inside the build to use branch current version -version: '{build}' - -image: -- Visual Studio 2015 - -#cache: -#- 'C:\Python38\' -#- 'C:\Python38-x64' - -environment: - libyaml_repo_url: https://github.com/yaml/libyaml.git - libyaml_refspec: 0.2.5 - PYYAML_TEST_GROUP: all - -# matrix: -# - PYTHON_VER: Python36 -# - PYTHON_VER: Python36-x64 -# - PYTHON_VER: Python37 -# - PYTHON_VER: Python37-x64 -# - PYTHON_VER: Python38 -# - PYTHON_VER: Python38-x64 -# - PYTHON_VER: Python39 -# - PYTHON_VER: Python39-x64 - -#init: -#- ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) - -build_script: -- ps: packaging\build\appveyor.ps1 - -#on_finish: -#- ps: $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1')) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 777839cd..bb1b550b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -12,59 +12,59 @@ env: LIBYAML_REF: 0.2.5 jobs: - python_sdist: - name: PyYAML sdist - runs-on: ubuntu-latest - steps: - - name: Checkout pyyaml - uses: actions/checkout@v2 - - - name: Install a python - uses: actions/setup-python@v2 - with: - python-version: 3.x - - - name: Install build deps - run: | - python -V - python -m pip install build - - - name: Build sdist - run: | - # We DO want to force Cythoning, at least until 6.0. - export PYYAML_FORCE_CYTHON=1 - # We don't actually want to build the lib. - export PYYAML_FORCE_LIBYAML=0 - - python -m build . - - # Ensure exactly one artifact was produced. - [[ $(shopt -s nullglob; ls dist/*.tar.gz | wc -w) == 1 ]] || { - echo "Unexpected content in dist dir: '$(ls dist/*.tar.gz)'." - exit 1 - } - - - name: Test sdist - run: | - # Install some libyaml headers. - # TODO Should we smoke test the sdist against the libyaml we built? - sudo apt update - sudo apt install libyaml-dev -y - - # Ensure Cython is not present so we use only what's in the sdist. - python -m pip uninstall Cython -y || true - - # Pass no extra args. - # We should auto-install with libyaml since it's present. - python -m pip install dist/*.tar.gz -v - - python packaging/build/smoketest.py - - - name: Upload sdist artifact - uses: actions/upload-artifact@v2 - with: - name: dist - path: dist/*.tar.gz +# python_sdist: +# name: PyYAML sdist +# runs-on: ubuntu-latest +# steps: +# - name: Checkout pyyaml +# uses: actions/checkout@v2 +# +# - name: Install a python +# uses: actions/setup-python@v2 +# with: +# python-version: 3.x +# +# - name: Install build deps +# run: | +# python -V +# python -m pip install build +# +# - name: Build sdist +# run: | +# # We DO want to force Cythoning, at least until 6.0. +# export PYYAML_FORCE_CYTHON=1 +# # We don't actually want to build the lib. +# export PYYAML_FORCE_LIBYAML=0 +# +# python -m build . +# +# # Ensure exactly one artifact was produced. +# [[ $(shopt -s nullglob; ls dist/*.tar.gz | wc -w) == 1 ]] || { +# echo "Unexpected content in dist dir: '$(ls dist/*.tar.gz)'." +# exit 1 +# } +# +# - name: Test sdist +# run: | +# # Install some libyaml headers. +# # TODO Should we smoke test the sdist against the libyaml we built? +# sudo apt update +# sudo apt install libyaml-dev -y +# +# # Ensure Cython is not present so we use only what's in the sdist. +# python -m pip uninstall Cython -y || true +# +# # Pass no extra args. +# # We should auto-install with libyaml since it's present. +# python -m pip install dist/*.tar.gz -v +# +# python packaging/build/smoketest.py +# +# - name: Upload sdist artifact +# uses: actions/upload-artifact@v2 +# with: +# name: dist +# path: dist/*.tar.gz linux_libyaml: @@ -74,8 +74,8 @@ jobs: matrix: cfg: - { platform: manylinux1, arch: x86_64 } - - { platform: manylinux2014, arch: aarch64 } - - { platform: manylinux2014, arch: s390x } +# - { platform: manylinux2014, arch: aarch64 } +# - { platform: manylinux2014, arch: s390x } env: DOCKER_IMAGE: quay.io/pypa/${{matrix.cfg.platform}}_${{matrix.cfg.arch}} steps: @@ -86,9 +86,9 @@ jobs: path: libyaml key: libyamlX_${{matrix.cfg.platform}}_${{matrix.cfg.arch}}_${{env.LIBYAML_REF}} - - name: configure docker foreign arch support - uses: docker/setup-qemu-action@v1 - if: matrix.cfg.arch != 'x86_64' && steps.cached_libyaml.outputs.cache-hit != 'true' +# - name: configure docker foreign arch support +# uses: docker/setup-qemu-action@v1 +# if: matrix.cfg.arch != 'x86_64' && steps.cached_libyaml.outputs.cache-hit != 'true' - name: Checkout pyyaml uses: actions/checkout@v2 @@ -118,18 +118,18 @@ jobs: strategy: matrix: cfg: - - { platform: manylinux1, arch: x86_64, python_tag: cp36-cp36m } - - { platform: manylinux1, arch: x86_64, python_tag: cp37-cp37m } - - { platform: manylinux1, arch: x86_64, python_tag: cp38-cp38 } +# - { platform: manylinux1, arch: x86_64, python_tag: cp36-cp36m } +# - { platform: manylinux1, arch: x86_64, python_tag: cp37-cp37m } +# - { platform: manylinux1, arch: x86_64, python_tag: cp38-cp38 } - { platform: manylinux1, arch: x86_64, python_tag: cp39-cp39 } - - { platform: manylinux2014, arch: aarch64, python_tag: cp36-cp36m } - - { platform: manylinux2014, arch: aarch64, python_tag: cp37-cp37m } - - { platform: manylinux2014, arch: aarch64, python_tag: cp38-cp38 } - - { platform: manylinux2014, arch: aarch64, python_tag: cp39-cp39 } - - { platform: manylinux2014, arch: s390x, python_tag: cp36-cp36m } - - { platform: manylinux2014, arch: s390x, python_tag: cp37-cp37m } - - { platform: manylinux2014, arch: s390x, python_tag: cp38-cp38 } - - { platform: manylinux2014, arch: s390x, python_tag: cp39-cp39 } +# - { platform: manylinux2014, arch: aarch64, python_tag: cp36-cp36m } +# - { platform: manylinux2014, arch: aarch64, python_tag: cp37-cp37m } +# - { platform: manylinux2014, arch: aarch64, python_tag: cp38-cp38 } +# - { platform: manylinux2014, arch: aarch64, python_tag: cp39-cp39 } +# - { platform: manylinux2014, arch: s390x, python_tag: cp36-cp36m } +# - { platform: manylinux2014, arch: s390x, python_tag: cp37-cp37m } +# - { platform: manylinux2014, arch: s390x, python_tag: cp38-cp38 } +# - { platform: manylinux2014, arch: s390x, python_tag: cp39-cp39 } env: AW_PLAT: ${{matrix.cfg.platform}}_${{matrix.cfg.arch}} DOCKER_IMAGE: quay.io/pypa/${{matrix.cfg.platform}}_${{matrix.cfg.arch}} @@ -176,78 +176,78 @@ jobs: name: dist path: dist/*.whl - macos_libyaml: - name: libyaml ${{matrix.arch}} ${{matrix.platform}} - runs-on: ${{matrix.platform}} - strategy: - matrix: - platform: - - macos-10.15 - arch: - - x86_64 - steps: - - name: Check cached libyaml state - id: cached_libyaml - uses: actions/cache@v2 - with: - path: libyaml - key: libyaml_${{matrix.platform}}_${{matrix.arch}}_${{env.LIBYAML_REF}} - - - name: Checkout pyyaml - uses: actions/checkout@v2 - if: steps.cached_libyaml.outputs.cache-hit != 'true' - - - name: Build libyaml - env: - MACOSX_DEPLOYMENT_TARGET: '10.9' - run: | - brew install automake coreutils - bash ./packaging/build/libyaml.sh - if: steps.cached_libyaml.outputs.cache-hit != 'true' - - macos_pyyaml: - needs: macos_libyaml - name: pyyaml ${{matrix.arch}} ${{matrix.platform}} ${{matrix.python_tag}} - runs-on: ${{matrix.platform}} - strategy: - matrix: - platform: - - macos-10.15 - arch: - - x86_64 - python_tag: - - cp36* - - cp37* - - cp38* - - cp39* - steps: - - name: Checkout pyyaml - uses: actions/checkout@v2 - - - name: Get cached libyaml state - id: cached_libyaml - uses: actions/cache@v2 - with: - path: libyaml - key: libyaml_${{matrix.platform}}_${{matrix.arch}}_${{env.LIBYAML_REF}} - - - name: Ensure libyaml fetched - run: exit 1 - if: steps.cached_libyaml.outputs.cache-hit != 'true' - - - name: Install a python - uses: actions/setup-python@v2 - with: - python-version: 3.x - - - name: Build/Test/Package - env: - CIBW_BUILD: ${{matrix.python_tag}} - CIBW_BUILD_VERBOSITY: 1 - run: bash ./packaging/build/macos.sh - - - uses: actions/upload-artifact@v2 - with: - name: dist - path: dist/*.whl +# macos_libyaml: +# name: libyaml ${{matrix.arch}} ${{matrix.platform}} +# runs-on: ${{matrix.platform}} +# strategy: +# matrix: +# platform: +# - macos-10.15 +# arch: +# - x86_64 +# steps: +# - name: Check cached libyaml state +# id: cached_libyaml +# uses: actions/cache@v2 +# with: +# path: libyaml +# key: libyaml_${{matrix.platform}}_${{matrix.arch}}_${{env.LIBYAML_REF}} +# +# - name: Checkout pyyaml +# uses: actions/checkout@v2 +# if: steps.cached_libyaml.outputs.cache-hit != 'true' +# +# - name: Build libyaml +# env: +# MACOSX_DEPLOYMENT_TARGET: '10.9' +# run: | +# brew install automake coreutils +# bash ./packaging/build/libyaml.sh +# if: steps.cached_libyaml.outputs.cache-hit != 'true' +# +# macos_pyyaml: +# needs: macos_libyaml +# name: pyyaml ${{matrix.arch}} ${{matrix.platform}} ${{matrix.python_tag}} +# runs-on: ${{matrix.platform}} +# strategy: +# matrix: +# platform: +# - macos-10.15 +# arch: +# - x86_64 +# python_tag: +# - cp36* +# - cp37* +# - cp38* +# - cp39* +# steps: +# - name: Checkout pyyaml +# uses: actions/checkout@v2 +# +# - name: Get cached libyaml state +# id: cached_libyaml +# uses: actions/cache@v2 +# with: +# path: libyaml +# key: libyaml_${{matrix.platform}}_${{matrix.arch}}_${{env.LIBYAML_REF}} +# +# - name: Ensure libyaml fetched +# run: exit 1 +# if: steps.cached_libyaml.outputs.cache-hit != 'true' +# +# - name: Install a python +# uses: actions/setup-python@v2 +# with: +# python-version: 3.x +# +# - name: Build/Test/Package +# env: +# CIBW_BUILD: ${{matrix.python_tag}} +# CIBW_BUILD_VERBOSITY: 1 +# run: bash ./packaging/build/macos.sh +# +# - uses: actions/upload-artifact@v2 +# with: +# name: dist +# path: dist/*.whl ... diff --git a/lib/yaml/__init__.py b/lib/yaml/__init__.py index 86d07b55..856caf52 100644 --- a/lib/yaml/__init__.py +++ b/lib/yaml/__init__.py @@ -202,6 +202,7 @@ def emit(events, stream=None, Dumper=Dumper, if stream is None: stream = io.StringIO() getvalue = stream.getvalue + # FIXME: redefine these defaults with sentinels to allow preservation of wrapped config defaults dumper = Dumper(stream, canonical=canonical, indent=indent, width=width, allow_unicode=allow_unicode, line_break=line_break) try: @@ -228,6 +229,7 @@ def serialize_all(nodes, stream=None, Dumper=Dumper, else: stream = io.BytesIO() getvalue = stream.getvalue + # FIXME: redefine these defaults with sentinels to allow preservation of wrapped config defaults dumper = Dumper(stream, canonical=canonical, indent=indent, width=width, allow_unicode=allow_unicode, line_break=line_break, encoding=encoding, version=version, tags=tags, @@ -250,28 +252,35 @@ def serialize(node, stream=None, Dumper=Dumper, **kwds): return serialize_all([node], stream, Dumper=Dumper, **kwds) def dump_all(documents, stream=None, Dumper=Dumper, - default_style=None, default_flow_style=False, - canonical=None, indent=None, width=None, - allow_unicode=None, line_break=None, - encoding=None, explicit_start=None, explicit_end=None, - version=None, tags=None, sort_keys=True): + default_style=..., default_flow_style=..., + canonical=..., indent=..., width=..., + allow_unicode=..., line_break=..., + encoding=..., explicit_start=..., explicit_end=..., + version=..., tags=..., sort_keys=..., **kwargs): """ Serialize a sequence of Python objects into a YAML stream. If stream is None, return the produced string instead. """ getvalue = None if stream is None: - if encoding is None: + if encoding is None or encoding is ...: stream = io.StringIO() else: stream = io.BytesIO() getvalue = stream.getvalue - dumper = Dumper(stream, default_style=default_style, - default_flow_style=default_flow_style, - canonical=canonical, indent=indent, width=width, - allow_unicode=allow_unicode, line_break=line_break, - encoding=encoding, version=version, tags=tags, - explicit_start=explicit_start, explicit_end=explicit_end, sort_keys=sort_keys) + + # preserve wrapped config defaults for values where we didn't get a default + # FIXME: share this code with the one in config mixin + dumper_init_kwargs = dict( + default_style=default_style, + default_flow_style=default_flow_style, + canonical=canonical, indent=indent, width=width, + allow_unicode=allow_unicode, line_break=line_break, + encoding=encoding, version=version, tags=tags, + explicit_start=explicit_start, explicit_end=explicit_end, sort_keys=sort_keys, **kwargs) + + dumper_init_kwargs = {k: v for k, v in dumper_init_kwargs.items() if v is not ...} + dumper = Dumper(stream, **dumper_init_kwargs) try: dumper.open() for data in documents: @@ -381,7 +390,17 @@ def add_multi_representer(data_type, multi_representer, Dumper=Dumper): """ Dumper.add_multi_representer(data_type, multi_representer) +def experimental_12_Core_loader(): + return loader._12_CoreLoader +def experimental_12_JSON_loader(): + return loader._12_JSONLoader + +def experimental_12_Core_dumper(): + return dumper._12_CoreDumper +def experimental_12_JSON_dumper(): + return dumper._12_JSONDumper class YAMLObjectMetaclass(type): + """ The metaclass for YAMLObject. """ diff --git a/lib/yaml/config.py b/lib/yaml/config.py new file mode 100644 index 00000000..df8aeee8 --- /dev/null +++ b/lib/yaml/config.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +import typing as t + +from functools import partialmethod +from .tagset import TagSet + +T = t.TypeVar('T') + + +class LoaderConfigMixin: + @classmethod + # FIXME: fix tagset type to use DataClasses, at least externally? + def config(cls: type[T], type_name: str | None = None, tagset: TagSet | ... = ..., **kwargs) -> type[T]: + if not type_name: + # FIXME: hash the inputs for a dynamic type name and cache it? + type_name = f'abcd_from_{cls.__name__}' + + new_type = t.cast(cls, type(type_name, (cls, ), {})) + + # FIXME: add support for arbitrary kwargs passthru ala dumper? + + if tagset is not ...: + # FIXME: provide a base class hook/method for this reset + new_type.yaml_implicit_resolvers = {} + new_type.init_resolvers(tagset.resolvers) + new_type.yaml_constructors = {} + new_type.init_constructors(tagset.constructors) + + return new_type + + +class DumperConfigMixin: + @classmethod + def config(cls: type[T], type_name: str | None = None, + tagset: TagSet | ... = ..., + # FIXME: make some of the more obscure style things "nicer" (eg enums?) or just pass through existing values? + default_style: str | ... = ..., default_flow_style: bool | ... = ..., + # FIXME: properly type-annotate the rest of these + canonical=..., indent=..., width=..., + allow_unicode=..., line_break=..., + encoding=..., explicit_start=..., explicit_end=..., + version=..., tags=..., sort_keys=..., + **kwargs) -> type[T]: + + if not type_name: + # FIXME: hash the inputs for a dynamic type name and cache it? + type_name = f'abcd_from_{cls.__name__}' + + # preserve wrapped config defaults for values where we didn't get a default + # FIXME: share this code with the one in __init__.dump_all (and implement on others) + dumper_init_kwargs = dict( + default_style=default_style, + default_flow_style=default_flow_style, + canonical=canonical, indent=indent, width=width, + allow_unicode=allow_unicode, line_break=line_break, + encoding=encoding, version=version, tags=tags, + explicit_start=explicit_start, explicit_end=explicit_end, sort_keys=sort_keys, **kwargs) + + dumper_init_kwargs = {k: v for k, v in dumper_init_kwargs.items() if v is not ...} + + patched_init = partialmethod(cls.__init__, + **dumper_init_kwargs) + + new_type = t.cast(cls, type(type_name, (cls, ), {'__init__': patched_init})) + + # FIXME: support all the dynamic dispatch types (multi*, etc) + if tagset is not ...: + # FIXME: provide a base class hook/method for this reset + new_type.yaml_implicit_resolvers = {} + new_type.init_resolvers(tagset.resolvers) + new_type.yaml_representers = {} + new_type.init_representers(tagset.representers) + + return new_type diff --git a/lib/yaml/constructor.py b/lib/yaml/constructor.py index 619acd30..73f421d4 100644 --- a/lib/yaml/constructor.py +++ b/lib/yaml/constructor.py @@ -5,19 +5,26 @@ 'FullConstructor', 'UnsafeConstructor', 'Constructor', - 'ConstructorError' + 'ConstructorError', ] from .error import * from .nodes import * import collections.abc, datetime, base64, binascii, re, sys, types +import typing as t class ConstructorError(MarkedYAMLError): pass class BaseConstructor: + inf_value = 1e300 + while inf_value != inf_value*inf_value: + inf_value *= inf_value + nan_value = -inf_value/inf_value # Trying to make a quiet NaN (like C99). + + yaml_constructors = {} yaml_multi_constructors = {} @@ -156,67 +163,6 @@ def construct_pairs(self, node, deep=False): pairs.append((key, value)) return pairs - @classmethod - def add_constructor(cls, tag, constructor): - if not 'yaml_constructors' in cls.__dict__: - cls.yaml_constructors = cls.yaml_constructors.copy() - cls.yaml_constructors[tag] = constructor - - @classmethod - def add_multi_constructor(cls, tag_prefix, multi_constructor): - if not 'yaml_multi_constructors' in cls.__dict__: - cls.yaml_multi_constructors = cls.yaml_multi_constructors.copy() - cls.yaml_multi_constructors[tag_prefix] = multi_constructor - -class SafeConstructor(BaseConstructor): - - def construct_scalar(self, node): - if isinstance(node, MappingNode): - for key_node, value_node in node.value: - if key_node.tag == 'tag:yaml.org,2002:value': - return self.construct_scalar(value_node) - return super().construct_scalar(node) - - def flatten_mapping(self, node): - merge = [] - index = 0 - while index < len(node.value): - key_node, value_node = node.value[index] - if key_node.tag == 'tag:yaml.org,2002:merge': - del node.value[index] - if isinstance(value_node, MappingNode): - self.flatten_mapping(value_node) - merge.extend(value_node.value) - elif isinstance(value_node, SequenceNode): - submerge = [] - for subnode in value_node.value: - if not isinstance(subnode, MappingNode): - raise ConstructorError("while constructing a mapping", - node.start_mark, - "expected a mapping for merging, but found %s" - % subnode.id, subnode.start_mark) - self.flatten_mapping(subnode) - submerge.append(subnode.value) - submerge.reverse() - for value in submerge: - merge.extend(value) - else: - raise ConstructorError("while constructing a mapping", node.start_mark, - "expected a mapping or list of mappings for merging, but found %s" - % value_node.id, value_node.start_mark) - elif key_node.tag == 'tag:yaml.org,2002:value': - key_node.tag = 'tag:yaml.org,2002:str' - index += 1 - else: - index += 1 - if merge: - node.value = merge + node.value - - def construct_mapping(self, node, deep=False): - if isinstance(node, MappingNode): - self.flatten_mapping(node) - return super().construct_mapping(node, deep=deep) - def construct_yaml_null(self, node): self.construct_scalar(node) return None @@ -234,42 +180,45 @@ def construct_yaml_bool(self, node): value = self.construct_scalar(node) return self.bool_values[value.lower()] - def construct_yaml_int(self, node): + def construct_yaml_str(self, node): + return self.construct_scalar(node) + + def construct_yaml_seq(self, node): + data = [] + yield data + data.extend(self.construct_sequence(node)) + + def construct_yaml_map(self, node): + data = {} + yield data + value = self.construct_mapping(node) + data.update(value) + + def construct_undefined(self, node): + raise ConstructorError(None, None, + "could not determine a constructor for the tag %r" % node.tag, + node.start_mark) + + def construct_yaml_int_core(self, node): value = self.construct_scalar(node) - value = value.replace('_', '') sign = +1 if value[0] == '-': sign = -1 if value[0] in '+-': value = value[1:] + if value == '0': return 0 - elif value.startswith('0b'): - return sign*int(value[2:], 2) + elif value.startswith('0o'): + return sign*int(value[2:], 8) elif value.startswith('0x'): return sign*int(value[2:], 16) - elif value[0] == '0': - return sign*int(value, 8) - elif ':' in value: - digits = [int(part) for part in value.split(':')] - digits.reverse() - base = 1 - value = 0 - for digit in digits: - value += digit*base - base *= 60 - return sign*value else: return sign*int(value) - inf_value = 1e300 - while inf_value != inf_value*inf_value: - inf_value *= inf_value - nan_value = -inf_value/inf_value # Trying to make a quiet NaN (like C99). - - def construct_yaml_float(self, node): + def construct_yaml_float_core(self, node): value = self.construct_scalar(node) - value = value.replace('_', '').lower() + value = value.lower() sign = +1 if value[0] == '-': sign = -1 @@ -279,18 +228,56 @@ def construct_yaml_float(self, node): return sign*self.inf_value elif value == '.nan': return self.nan_value - elif ':' in value: - digits = [float(part) for part in value.split(':')] - digits.reverse() - base = 1 - value = 0.0 - for digit in digits: - value += digit*base - base *= 60 - return sign*value else: return sign*float(value) + def construct_yaml_int_json(self, node): + value = self.construct_scalar(node) + sign = +1 + if value[0] == '-': + sign = -1 + if value[0] in '+-': + value = value[1:] + + if value == '0': + return 0 + else: + return sign*int(value) + + def construct_yaml_float_json(self, node): + value = self.construct_scalar(node) + value = value.lower() + sign = +1 + if value[0] == '-': + sign = -1 + if value[0] in '+-': + value = value[1:] + return sign*float(value) + + @classmethod + def add_constructor(cls, tag, constructor): + if not 'yaml_constructors' in cls.__dict__: + cls.yaml_constructors = cls.yaml_constructors.copy() + cls.yaml_constructors[tag] = constructor + + @classmethod + def add_multi_constructor(cls, tag_prefix, multi_constructor): + if not 'yaml_multi_constructors' in cls.__dict__: + cls.yaml_multi_constructors = cls.yaml_multi_constructors.copy() + cls.yaml_multi_constructors[tag_prefix] = multi_constructor + + + @classmethod + def init_constructors(cls, tagset: dict[str, t.Callable]): + for type_name, constructor in tagset.items(): + # FIXME: encode full tag names in TagSets to avoid this logic and make user-definable types easier + tag_name = f'tag:yaml.org,2002:{type_name}' if type_name else None + cls.add_constructor(tag_name, constructor) + + +# SafeConstructor implements YAML 1.1 +class SafeConstructor(BaseConstructor): + def construct_yaml_binary(self, node): try: value = self.construct_scalar(node).encode('ascii') @@ -399,19 +386,104 @@ def construct_yaml_set(self, node): value = self.construct_mapping(node) data.update(value) - def construct_yaml_str(self, node): - return self.construct_scalar(node) + def construct_scalar(self, node): + if isinstance(node, MappingNode): + for key_node, value_node in node.value: + if key_node.tag == 'tag:yaml.org,2002:value': + return self.construct_scalar(value_node) + return super().construct_scalar(node) - def construct_yaml_seq(self, node): - data = [] - yield data - data.extend(self.construct_sequence(node)) + def flatten_mapping(self, node): + merge = [] + index = 0 + while index < len(node.value): + key_node, value_node = node.value[index] + if key_node.tag == 'tag:yaml.org,2002:merge': + del node.value[index] + if isinstance(value_node, MappingNode): + self.flatten_mapping(value_node) + merge.extend(value_node.value) + elif isinstance(value_node, SequenceNode): + submerge = [] + for subnode in value_node.value: + if not isinstance(subnode, MappingNode): + raise ConstructorError("while constructing a mapping", + node.start_mark, + "expected a mapping for merging, but found %s" + % subnode.id, subnode.start_mark) + self.flatten_mapping(subnode) + submerge.append(subnode.value) + submerge.reverse() + for value in submerge: + merge.extend(value) + else: + raise ConstructorError("while constructing a mapping", node.start_mark, + "expected a mapping or list of mappings for merging, but found %s" + % value_node.id, value_node.start_mark) + elif key_node.tag == 'tag:yaml.org,2002:value': + key_node.tag = 'tag:yaml.org,2002:str' + index += 1 + else: + index += 1 + if merge: + node.value = merge + node.value - def construct_yaml_map(self, node): - data = {} - yield data - value = self.construct_mapping(node) - data.update(value) + def construct_yaml_int(self, node): + value = self.construct_scalar(node) + value = value.replace('_', '') + sign = +1 + if value[0] == '-': + sign = -1 + if value[0] in '+-': + value = value[1:] + if value == '0': + return 0 + elif value.startswith('0b'): + return sign*int(value[2:], 2) + elif value.startswith('0x'): + return sign*int(value[2:], 16) + elif value[0] == '0': + return sign*int(value, 8) + elif ':' in value: + digits = [int(part) for part in value.split(':')] + digits.reverse() + base = 1 + value = 0 + for digit in digits: + value += digit*base + base *= 60 + return sign*value + else: + return sign*int(value) + + def construct_yaml_float(self, node): + value = self.construct_scalar(node) + value = value.replace('_', '').lower() + sign = +1 + if value[0] == '-': + sign = -1 + if value[0] in '+-': + value = value[1:] + if value == '.inf': + return sign*self.inf_value + elif value == '.nan': + return self.nan_value + elif ':' in value: + digits = [float(part) for part in value.split(':')] + digits.reverse() + base = 1 + value = 0.0 + for digit in digits: + value += digit*base + base *= 60 + return sign*value + else: + return sign*float(value) + + def construct_mapping(self, node, deep=False): + if isinstance(node, MappingNode): + self.flatten_mapping(node) + return super().construct_mapping(node, deep=deep) def construct_yaml_object(self, node, cls): data = cls.__new__(cls) @@ -423,61 +495,46 @@ def construct_yaml_object(self, node, cls): state = self.construct_mapping(node) data.__dict__.update(state) - def construct_undefined(self, node): - raise ConstructorError(None, None, - "could not determine a constructor for the tag %r" % node.tag, - node.start_mark) - -SafeConstructor.add_constructor( - 'tag:yaml.org,2002:null', - SafeConstructor.construct_yaml_null) - -SafeConstructor.add_constructor( - 'tag:yaml.org,2002:bool', - SafeConstructor.construct_yaml_bool) - -SafeConstructor.add_constructor( - 'tag:yaml.org,2002:int', - SafeConstructor.construct_yaml_int) - -SafeConstructor.add_constructor( - 'tag:yaml.org,2002:float', - SafeConstructor.construct_yaml_float) - -SafeConstructor.add_constructor( - 'tag:yaml.org,2002:binary', - SafeConstructor.construct_yaml_binary) - -SafeConstructor.add_constructor( - 'tag:yaml.org,2002:timestamp', - SafeConstructor.construct_yaml_timestamp) - -SafeConstructor.add_constructor( - 'tag:yaml.org,2002:omap', - SafeConstructor.construct_yaml_omap) - -SafeConstructor.add_constructor( - 'tag:yaml.org,2002:pairs', - SafeConstructor.construct_yaml_pairs) - -SafeConstructor.add_constructor( - 'tag:yaml.org,2002:set', - SafeConstructor.construct_yaml_set) - -SafeConstructor.add_constructor( - 'tag:yaml.org,2002:str', - SafeConstructor.construct_yaml_str) - -SafeConstructor.add_constructor( - 'tag:yaml.org,2002:seq', - SafeConstructor.construct_yaml_seq) - -SafeConstructor.add_constructor( - 'tag:yaml.org,2002:map', - SafeConstructor.construct_yaml_map) -SafeConstructor.add_constructor(None, - SafeConstructor.construct_undefined) +_yaml11_constructors = { + 'str': BaseConstructor.construct_yaml_str, + 'seq': BaseConstructor.construct_yaml_seq, + 'map': BaseConstructor.construct_yaml_map, + None: BaseConstructor.construct_undefined, + 'int': SafeConstructor.construct_yaml_int, + 'float': SafeConstructor.construct_yaml_float, + 'null': BaseConstructor.construct_yaml_null, + 'bool': BaseConstructor.construct_yaml_bool, + 'binary': SafeConstructor.construct_yaml_binary, + 'timestamp': SafeConstructor.construct_yaml_timestamp, + 'omap': SafeConstructor.construct_yaml_omap, + 'pairs': SafeConstructor.construct_yaml_pairs, + 'set': SafeConstructor.construct_yaml_set, + } + +_core_constructors = { + 'str': BaseConstructor.construct_yaml_str, + 'seq': BaseConstructor.construct_yaml_seq, + 'map': BaseConstructor.construct_yaml_map, + None: BaseConstructor.construct_undefined, + 'int': BaseConstructor.construct_yaml_int_core, + 'float': BaseConstructor.construct_yaml_float_core, + 'null': BaseConstructor.construct_yaml_null, + 'bool': BaseConstructor.construct_yaml_bool, + } + +_json_constructors = { + 'str': BaseConstructor.construct_yaml_str, + 'seq': BaseConstructor.construct_yaml_seq, + 'map': BaseConstructor.construct_yaml_map, + None: BaseConstructor.construct_undefined, + 'int': BaseConstructor.construct_yaml_int_json, + 'float': BaseConstructor.construct_yaml_float_json, + 'null': BaseConstructor.construct_yaml_null, + 'bool': BaseConstructor.construct_yaml_bool, + } + +SafeConstructor.init_constructors(_yaml11_constructors) class FullConstructor(SafeConstructor): # 'extend' is blacklisted because it is used by diff --git a/lib/yaml/cyaml.py b/lib/yaml/cyaml.py index 0c213458..9019209e 100644 --- a/lib/yaml/cyaml.py +++ b/lib/yaml/cyaml.py @@ -6,6 +6,7 @@ from yaml._yaml import CParser, CEmitter +from .config import LoaderConfigMixin, DumperConfigMixin from .constructor import * from .serializer import * @@ -13,7 +14,7 @@ from .resolver import * -class CBaseLoader(CParser, BaseConstructor, BaseResolver): +class CBaseLoader(CParser, BaseConstructor, BaseResolver, LoaderConfigMixin): def __init__(self, stream): CParser.__init__(self, stream) @@ -41,7 +42,7 @@ def __init__(self, stream): UnsafeConstructor.__init__(self) Resolver.__init__(self) -class CLoader(CParser, Constructor, Resolver): +class CLoader(CParser, Constructor, Resolver, LoaderConfigMixin): def __init__(self, stream): CParser.__init__(self, stream) @@ -82,7 +83,7 @@ def __init__(self, stream, default_flow_style=default_flow_style, sort_keys=sort_keys) Resolver.__init__(self) -class CDumper(CEmitter, Serializer, Representer, Resolver): +class CDumper(CEmitter, Serializer, Representer, Resolver, DumperConfigMixin): def __init__(self, stream, default_style=None, default_flow_style=False, diff --git a/lib/yaml/dumper.py b/lib/yaml/dumper.py index 6aadba55..1c5c236e 100644 --- a/lib/yaml/dumper.py +++ b/lib/yaml/dumper.py @@ -1,12 +1,21 @@ -__all__ = ['BaseDumper', 'SafeDumper', 'Dumper'] +__all__ = ['BaseDumper', 'SafeDumper', 'Dumper', 'CommonDumper', 'FastestBaseDumper'] +from . import tagset + +from .config import DumperConfigMixin from .emitter import * from .serializer import * from .representer import * from .resolver import * -class BaseDumper(Emitter, Serializer, BaseRepresenter, BaseResolver): +try: + from .cyaml import CDumper as FastestBaseDumper +except ImportError as ie: + FastestBaseDumper = None + + +class BaseDumper(Emitter, Serializer, BaseRepresenter, BaseResolver, DumperConfigMixin): def __init__(self, stream, default_style=None, default_flow_style=False, @@ -24,7 +33,17 @@ def __init__(self, stream, default_flow_style=default_flow_style, sort_keys=sort_keys) Resolver.__init__(self) -class SafeDumper(Emitter, Serializer, SafeRepresenter, Resolver): + +if not FastestBaseDumper: + # fall back to pure-Python if CBaseDumper is unavailable + FastestBaseDumper = BaseDumper + +# FIXME: reimplement all these as config calls, eg: +# SafeDumper = FastestBaseDumper.config(type_name='SafeDumper', tagset=tagset.yaml11) + + + +class SafeDumper(Emitter, Serializer, SafeRepresenter, Resolver, DumperConfigMixin): def __init__(self, stream, default_style=None, default_flow_style=False, @@ -42,7 +61,31 @@ def __init__(self, stream, default_flow_style=default_flow_style, sort_keys=sort_keys) Resolver.__init__(self) -class Dumper(Emitter, Serializer, Representer, Resolver): +class CommonDumper(Emitter, Serializer, CommonRepresenter, BaseResolver, DumperConfigMixin): + + def __init__(self, stream, + default_style=None, default_flow_style=False, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None, + encoding=None, explicit_start=None, explicit_end=None, + version=None, tags=None, sort_keys=True): + Emitter.__init__(self, stream, canonical=canonical, + indent=indent, width=width, + allow_unicode=allow_unicode, line_break=line_break) + Serializer.__init__(self, encoding=encoding, + explicit_start=explicit_start, explicit_end=explicit_end, + version=version, tags=tags) + CommonRepresenter.__init__(self, default_style=default_style, + default_flow_style=default_flow_style, sort_keys=sort_keys) + BaseResolver.__init__(self) + + @classmethod + def init_tags(cls, tagset: tagset.TagSet): + cls.init_representers(tagset.representers) + cls.init_resolvers(tagset.resolvers) + + +class Dumper(Emitter, Serializer, Representer, Resolver, DumperConfigMixin): def __init__(self, stream, default_style=None, default_flow_style=False, @@ -60,3 +103,8 @@ def __init__(self, stream, default_flow_style=default_flow_style, sort_keys=sort_keys) Resolver.__init__(self) + +_12_CoreDumper = CommonDumper.config(type_name='_12_CoreDumper', tagset=tagset.core) +_12_JSONDumper = CommonDumper.config(type_name='_12_JSONDumper', tagset=tagset.json) + + diff --git a/lib/yaml/loader.py b/lib/yaml/loader.py index e90c1122..61ab4c94 100644 --- a/lib/yaml/loader.py +++ b/lib/yaml/loader.py @@ -1,15 +1,24 @@ +from __future__ import annotations -__all__ = ['BaseLoader', 'FullLoader', 'SafeLoader', 'Loader', 'UnsafeLoader'] +__all__ = ['BaseLoader', 'FullLoader', 'SafeLoader', 'Loader', 'UnsafeLoader', 'FastestBaseLoader'] +from .config import LoaderConfigMixin from .reader import * from .scanner import * from .parser import * from .composer import * from .constructor import * from .resolver import * +from . import tagset -class BaseLoader(Reader, Scanner, Parser, Composer, BaseConstructor, BaseResolver): +# FIXME: defer setup to fix circular import +try: + from .cyaml import CBaseLoader as FastestBaseLoader +except ImportError as ie: + FastestBaseLoader = None + +class BaseLoader(Reader, Scanner, Parser, Composer, BaseConstructor, BaseResolver, LoaderConfigMixin): def __init__(self, stream): Reader.__init__(self, stream) Scanner.__init__(self) @@ -18,7 +27,24 @@ def __init__(self, stream): BaseConstructor.__init__(self) BaseResolver.__init__(self) -class FullLoader(Reader, Scanner, Parser, Composer, FullConstructor, Resolver): + @classmethod + def init_tags(cls, tagset: tagset.TagSet): + cls.init_constructors(tagset.constructors) + cls.init_resolvers(tagset.resolvers) + + +if not FastestBaseLoader: + # fall back to pure Python if CBaseLoader is unavailable + FastestBaseLoader = BaseLoader + + +# FIXME: once we fully implement the config stuff, these can all be collapsed to a config call, eg: +# FullLoader = FastestBaseLoader.config(type_name='FullLoader', tagset=tagset.yaml11 | tagset.python_full) +# SafeLoader = FastestBaseLoader.config(type_name='SafeLoader', tagset=tagset.yaml11) +# UnsafeLoader = FastestBaseLoader.config(type_name='UnsafeLoader', tagset=tagset.yaml11 | tagset.python_unsafe) +# this pattern will also allow a much easier path for users to bolt on default behavior to any old loader + +class FullLoader(Reader, Scanner, Parser, Composer, FullConstructor, Resolver, LoaderConfigMixin): def __init__(self, stream): Reader.__init__(self, stream) @@ -28,7 +54,7 @@ def __init__(self, stream): FullConstructor.__init__(self) Resolver.__init__(self) -class SafeLoader(Reader, Scanner, Parser, Composer, SafeConstructor, Resolver): +class SafeLoader(Reader, Scanner, Parser, Composer, SafeConstructor, Resolver, LoaderConfigMixin): def __init__(self, stream): Reader.__init__(self, stream) @@ -38,7 +64,7 @@ def __init__(self, stream): SafeConstructor.__init__(self) Resolver.__init__(self) -class Loader(Reader, Scanner, Parser, Composer, Constructor, Resolver): +class Loader(Reader, Scanner, Parser, Composer, Constructor, Resolver, LoaderConfigMixin): def __init__(self, stream): Reader.__init__(self, stream) @@ -52,7 +78,7 @@ def __init__(self, stream): # untrusted input). Use of either Loader or UnsafeLoader should be rare, since # FullLoad should be able to load almost all YAML safely. Loader is left intact # to ensure backwards compatibility. -class UnsafeLoader(Reader, Scanner, Parser, Composer, Constructor, Resolver): +class UnsafeLoader(Reader, Scanner, Parser, Composer, Constructor, Resolver, LoaderConfigMixin): def __init__(self, stream): Reader.__init__(self, stream) @@ -61,3 +87,8 @@ def __init__(self, stream): Composer.__init__(self) Constructor.__init__(self) Resolver.__init__(self) + + +_12_CoreLoader = BaseLoader.config(type_name='_12_CoreLoader', tagset=tagset.core) +_12_JSONLoader = BaseLoader.config(type_name='_12_JSONLoader', tagset=tagset.json) + diff --git a/lib/yaml/representer.py b/lib/yaml/representer.py index 3b0b192e..70905a3c 100644 --- a/lib/yaml/representer.py +++ b/lib/yaml/representer.py @@ -1,11 +1,12 @@ __all__ = ['BaseRepresenter', 'SafeRepresenter', 'Representer', - 'RepresenterError'] + 'RepresenterError', 'CommonRepresenter'] from .error import * from .nodes import * import datetime, copyreg, types, base64, collections +import typing as t class RepresenterError(YAMLError): pass @@ -131,29 +132,30 @@ def represent_mapping(self, tag, mapping, flow_style=None): def ignore_aliases(self, data): return False -class SafeRepresenter(BaseRepresenter): + def represent_str(self, data): + return self.represent_scalar('tag:yaml.org,2002:str', data) - def ignore_aliases(self, data): - if data is None: - return True - if isinstance(data, tuple) and data == (): - return True - if isinstance(data, (str, bytes, bool, int, float)): - return True + def represent_list(self, data): + #pairs = (len(data) > 0 and isinstance(data, list)) + #if pairs: + # for item in data: + # if not isinstance(item, tuple) or len(item) != 2: + # pairs = False + # break + #if not pairs: + return self.represent_sequence('tag:yaml.org,2002:seq', data) + #value = [] + #for item_key, item_value in data: + # value.append(self.represent_mapping(u'tag:yaml.org,2002:map', + # [(item_key, item_value)])) + #return SequenceNode(u'tag:yaml.org,2002:pairs', value) + + def represent_dict(self, data): + return self.represent_mapping('tag:yaml.org,2002:map', data) def represent_none(self, data): return self.represent_scalar('tag:yaml.org,2002:null', 'null') - def represent_str(self, data): - return self.represent_scalar('tag:yaml.org,2002:str', data) - - def represent_binary(self, data): - if hasattr(base64, 'encodebytes'): - data = base64.encodebytes(data).decode('ascii') - else: - data = base64.encodestring(data).decode('ascii') - return self.represent_scalar('tag:yaml.org,2002:binary', data, style='|') - def represent_bool(self, data): if data: value = 'true' @@ -184,27 +186,17 @@ def represent_float(self, data): # Unfortunately, this is not a valid float representation according # to the definition of the `!!float` tag. We fix this by adding # '.0' before the 'e' symbol. + # TODO (In YAML 1.2 Core, 1e17 would be a valid float though) if '.' not in value and 'e' in value: value = value.replace('e', '.0e', 1) return self.represent_scalar('tag:yaml.org,2002:float', value) - def represent_list(self, data): - #pairs = (len(data) > 0 and isinstance(data, list)) - #if pairs: - # for item in data: - # if not isinstance(item, tuple) or len(item) != 2: - # pairs = False - # break - #if not pairs: - return self.represent_sequence('tag:yaml.org,2002:seq', data) - #value = [] - #for item_key, item_value in data: - # value.append(self.represent_mapping(u'tag:yaml.org,2002:map', - # [(item_key, item_value)])) - #return SequenceNode(u'tag:yaml.org,2002:pairs', value) - - def represent_dict(self, data): - return self.represent_mapping('tag:yaml.org,2002:map', data) + def represent_binary(self, data): + if hasattr(base64, 'encodebytes'): + data = base64.encodebytes(data).decode('ascii') + else: + data = base64.encodestring(data).decode('ascii') + return self.represent_scalar('tag:yaml.org,2002:binary', data, style='|') def represent_set(self, data): value = {} @@ -220,54 +212,90 @@ def represent_datetime(self, data): value = data.isoformat(' ') return self.represent_scalar('tag:yaml.org,2002:timestamp', value) - def represent_yaml_object(self, tag, data, cls, flow_style=None): - if hasattr(data, '__getstate__'): - state = data.__getstate__() - else: - state = data.__dict__.copy() - return self.represent_mapping(tag, state, flow_style=flow_style) - def represent_undefined(self, data): raise RepresenterError("cannot represent an object", data) -SafeRepresenter.add_representer(type(None), - SafeRepresenter.represent_none) - -SafeRepresenter.add_representer(str, - SafeRepresenter.represent_str) - -SafeRepresenter.add_representer(bytes, - SafeRepresenter.represent_binary) - -SafeRepresenter.add_representer(bool, - SafeRepresenter.represent_bool) - -SafeRepresenter.add_representer(int, - SafeRepresenter.represent_int) - -SafeRepresenter.add_representer(float, - SafeRepresenter.represent_float) + @classmethod + def init_representers(cls, tagset_representers: dict[str, t.Callable]): + for type_name, representer in tagset_representers.items(): + cls.add_representer(type_name, representer) -SafeRepresenter.add_representer(list, - SafeRepresenter.represent_list) +class CommonRepresenter(BaseRepresenter): -SafeRepresenter.add_representer(tuple, - SafeRepresenter.represent_list) + def ignore_aliases(self, data): + if data is None: + return True + if isinstance(data, tuple) and data == (): + return True + if isinstance(data, (str, bytes, bool, int, float)): + return True -SafeRepresenter.add_representer(dict, - SafeRepresenter.represent_dict) + def represent_yaml_object(self, tag, data, cls, flow_style=None): + if hasattr(data, '__getstate__'): + state = data.__getstate__() + else: + state = data.__dict__.copy() + return self.represent_mapping(tag, state, flow_style=flow_style) -SafeRepresenter.add_representer(set, - SafeRepresenter.represent_set) +class SafeRepresenter(BaseRepresenter): -SafeRepresenter.add_representer(datetime.date, - SafeRepresenter.represent_date) + def ignore_aliases(self, data): + if data is None: + return True + if isinstance(data, tuple) and data == (): + return True + if isinstance(data, (str, bytes, bool, int, float)): + return True -SafeRepresenter.add_representer(datetime.datetime, - SafeRepresenter.represent_datetime) + def represent_yaml_object(self, tag, data, cls, flow_style=None): + if hasattr(data, '__getstate__'): + state = data.__getstate__() + else: + state = data.__dict__.copy() + return self.represent_mapping(tag, state, flow_style=flow_style) -SafeRepresenter.add_representer(None, - SafeRepresenter.represent_undefined) +_representers = { + 'yaml11': { + bool: BaseRepresenter.represent_bool, + dict: BaseRepresenter.represent_dict, + list: BaseRepresenter.represent_list, + tuple: BaseRepresenter.represent_list, + str: BaseRepresenter.represent_str, + float: BaseRepresenter.represent_float, + int: BaseRepresenter.represent_int, + type(None): BaseRepresenter.represent_none, + bytes: BaseRepresenter.represent_binary, + set: BaseRepresenter.represent_set, + datetime.date: BaseRepresenter.represent_date, + datetime.datetime: BaseRepresenter.represent_datetime, + None: BaseRepresenter.represent_undefined, + }, + 'core': { + bool: BaseRepresenter.represent_bool, + dict: BaseRepresenter.represent_dict, + list: BaseRepresenter.represent_list, + tuple: BaseRepresenter.represent_list, + str: BaseRepresenter.represent_str, + float: BaseRepresenter.represent_float, + int: BaseRepresenter.represent_int, + type(None): BaseRepresenter.represent_none, + None: BaseRepresenter.represent_undefined, + }, + 'json': { + bool: BaseRepresenter.represent_bool, + dict: BaseRepresenter.represent_dict, + list: BaseRepresenter.represent_list, + tuple: BaseRepresenter.represent_list, + str: BaseRepresenter.represent_str, + float: BaseRepresenter.represent_float, + int: BaseRepresenter.represent_int, + type(None): BaseRepresenter.represent_none, + None: BaseRepresenter.represent_undefined, + }, +} + + +SafeRepresenter.init_representers(_representers['yaml11']) class Representer(SafeRepresenter): @@ -369,7 +397,7 @@ def represent_ordered_dict(self, data): Representer.add_representer(tuple, Representer.represent_tuple) -Representer.add_representer(type, +Representer.add_multi_representer(type, Representer.represent_name) Representer.add_representer(collections.OrderedDict, diff --git a/lib/yaml/resolver.py b/lib/yaml/resolver.py index 013896d2..e85500d7 100644 --- a/lib/yaml/resolver.py +++ b/lib/yaml/resolver.py @@ -1,10 +1,11 @@ __all__ = ['BaseResolver', 'Resolver'] +import re + from .error import * from .nodes import * -import re class ResolverError(YAMLError): pass @@ -164,64 +165,102 @@ def resolve(self, kind, value, implicit): elif kind is MappingNode: return self.DEFAULT_MAPPING_TAG -class Resolver(BaseResolver): - pass + @classmethod + def init_resolvers(cls, implicit_resolvers: list[list]): + # FIXME: factor this out as a config mixin with dataclasses + for args in implicit_resolvers: + cls.add_implicit_resolver( + 'tag:yaml.org,2002:' + args[0], + args[1], args[2] + ) + + + +# FIXME: come up with a better way to lazy init these so we can define all the well-known stuff in one place +# avoid circular import by defining here for now +_yaml11_resolvers = [ + ['bool', + re.compile(r'''^(?:yes|Yes|YES|no|No|NO + |true|True|TRUE|false|False|FALSE + |on|On|ON|off|Off|OFF)$''', re.X), + list('yYnNtTfFoO')], + ['float', + re.compile(r'''^(?:[-+]?(?:[0-9][0-9_]*)\.[0-9_]*(?:[eE][-+][0-9]+)? + |\.[0-9_]+(?:[eE][-+][0-9]+)? + |[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+\.[0-9_]* + |[-+]?\.(?:inf|Inf|INF) + |\.(?:nan|NaN|NAN))$''', re.X), + list('-+0123456789.')], + ['int', + re.compile(r'''^(?:[-+]?0b[0-1_]+ + |[-+]?0[0-7_]+ + |[-+]?(?:0|[1-9][0-9_]*) + |[-+]?0x[0-9a-fA-F_]+ + |[-+]?[1-9][0-9_]*(?::[0-5]?[0-9])+)$''', re.X), + list('-+0123456789')], + ['merge', + re.compile(r'^(?:<<)$'), + ['<']], + ['null', + re.compile(r'''^(?: ~ + |null|Null|NULL + | )$''', re.X), + ['~', 'n', 'N', '']], + ['timestamp', + re.compile(r'''^(?:[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] + |[0-9][0-9][0-9][0-9] -[0-9][0-9]? -[0-9][0-9]? + (?:[Tt]|[ \t]+)[0-9][0-9]? + :[0-9][0-9] :[0-9][0-9] (?:\.[0-9]*)? + (?:[ \t]*(?:Z|[-+][0-9][0-9]?(?::[0-9][0-9])?))?)$''', re.X), + list('0123456789')], + ['value', + re.compile(r'^(?:=)$'), + ['=']], + # The following resolver is only for documentation purposes. It cannot work + # because plain scalars cannot start with '!', '&', or '*'. + ['yaml', + re.compile(r'^(?:!|&|\*)$'), + list('!&*')], +] + +_core_resolvers = [ + ['bool', + re.compile(r'''^(?:|true|True|TRUE|false|False|FALSE)$''', re.X), + list('tTfF')], + ['int', + re.compile(r'''^(?: + |0o[0-7]+ + |[-+]?(?:[0-9]+) + |0x[0-9a-fA-F]+ + )$''', re.X), + list('-+0123456789')], + ['float', + re.compile(r'''^(?:[-+]?(?:\.[0-9]+|[0-9]+(\.[0-9]*)?)(?:[eE][-+]?[0-9]+)? + |[-+]?\.(?:inf|Inf|INF) + |\.(?:nan|NaN|NAN))$''', re.X), + list('-+0123456789.')], + ['null', + re.compile(r'''^(?:~||null|Null|NULL)$''', re.X), + ['~', 'n', 'N', '']], + ] + +_json_resolvers = [ + ['bool', + re.compile(r'''^(?:true|false)$''', re.X), + list('tf')], + ['int', + re.compile(r'''^-?(?:0|[1-9][0-9]*)$''', re.X), + list('-0123456789')], + ['float', + re.compile(r'''^-?(?:0|[1-9][0-9]*)(\.[0-9]*)?(?:[eE][-+]?[0-9]+)?$''', re.X), + list('-0123456789.')], + ['null', + re.compile(r'''^null$''', re.X), + ['n']], + ] + +class Resolver(BaseResolver): pass -Resolver.add_implicit_resolver( - 'tag:yaml.org,2002:bool', - re.compile(r'''^(?:yes|Yes|YES|no|No|NO - |true|True|TRUE|false|False|FALSE - |on|On|ON|off|Off|OFF)$''', re.X), - list('yYnNtTfFoO')) - -Resolver.add_implicit_resolver( - 'tag:yaml.org,2002:float', - re.compile(r'''^(?:[-+]?(?:[0-9][0-9_]*)\.[0-9_]*(?:[eE][-+][0-9]+)? - |\.[0-9_]+(?:[eE][-+][0-9]+)? - |[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+\.[0-9_]* - |[-+]?\.(?:inf|Inf|INF) - |\.(?:nan|NaN|NAN))$''', re.X), - list('-+0123456789.')) - -Resolver.add_implicit_resolver( - 'tag:yaml.org,2002:int', - re.compile(r'''^(?:[-+]?0b[0-1_]+ - |[-+]?0[0-7_]+ - |[-+]?(?:0|[1-9][0-9_]*) - |[-+]?0x[0-9a-fA-F_]+ - |[-+]?[1-9][0-9_]*(?::[0-5]?[0-9])+)$''', re.X), - list('-+0123456789')) - -Resolver.add_implicit_resolver( - 'tag:yaml.org,2002:merge', - re.compile(r'^(?:<<)$'), - ['<']) - -Resolver.add_implicit_resolver( - 'tag:yaml.org,2002:null', - re.compile(r'''^(?: ~ - |null|Null|NULL - | )$''', re.X), - ['~', 'n', 'N', '']) - -Resolver.add_implicit_resolver( - 'tag:yaml.org,2002:timestamp', - re.compile(r'''^(?:[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] - |[0-9][0-9][0-9][0-9] -[0-9][0-9]? -[0-9][0-9]? - (?:[Tt]|[ \t]+)[0-9][0-9]? - :[0-9][0-9] :[0-9][0-9] (?:\.[0-9]*)? - (?:[ \t]*(?:Z|[-+][0-9][0-9]?(?::[0-9][0-9])?))?)$''', re.X), - list('0123456789')) - -Resolver.add_implicit_resolver( - 'tag:yaml.org,2002:value', - re.compile(r'^(?:=)$'), - ['=']) - -# The following resolver is only for documentation purposes. It cannot work -# because plain scalars cannot start with '!', '&', or '*'. -Resolver.add_implicit_resolver( - 'tag:yaml.org,2002:yaml', - re.compile(r'^(?:!|&|\*)$'), - list('!&*')) +# FIXME: config mixin or defer this to loader setup via config? +Resolver.init_resolvers(_yaml11_resolvers) diff --git a/lib/yaml/tagset.py b/lib/yaml/tagset.py new file mode 100644 index 00000000..f04d78aa --- /dev/null +++ b/lib/yaml/tagset.py @@ -0,0 +1,37 @@ +import re +import typing as t + +from dataclasses import dataclass +# FIXME: disentangle various class inits so we can define these all together +from .constructor import _core_constructors, _json_constructors, _yaml11_constructors +from .representer import _representers # FIXME make consistent +from .resolver import _core_resolvers, _json_resolvers, _yaml11_resolvers + +# FIXME: restructure these as dataclasses for each type that are individually accessible to make it easier to build up a custom tag set with well-known types + +# FIXME: add Python tags here as well + +@dataclass +class TagSet: + name: str + constructors: dict[str, t.Callable] # FIXME: implement constructor dataclass? + representers: dict[str, t.Callable] # FIXME: implement representer dataclass? + resolvers: list[list] # FIXME: implement ImplicitResolver dataclass + # FIXME: add support for multi/implicit constructors, representers, resolvers, etc? + + +yaml11 = TagSet(name='yaml11', + constructors=_yaml11_constructors, + representers=_representers['yaml11'], + resolvers=_yaml11_resolvers) + +core = TagSet(name='core', + constructors=_core_constructors, + representers=_representers['core'], + resolvers=_core_resolvers) + +json = TagSet(name='json', + constructors=_json_constructors, + representers=_representers['json'], + resolvers=_json_resolvers) + diff --git a/tests/data/construct-python-name-module.code b/tests/data/construct-python-name-module.code index 6f391488..b8a4b6f8 100644 --- a/tests/data/construct-python-name-module.code +++ b/tests/data/construct-python-name-module.code @@ -1 +1 @@ -[str, yaml.Loader, yaml.dump, abs, yaml.tokens] +[str, yaml.Loader, yaml.dump, abs, yaml.tokens, signal.Handlers] diff --git a/tests/data/construct-python-name-module.data b/tests/data/construct-python-name-module.data index f0c9712b..f1a2c24a 100644 --- a/tests/data/construct-python-name-module.data +++ b/tests/data/construct-python-name-module.data @@ -3,3 +3,4 @@ - !!python/name:yaml.dump - !!python/name:abs - !!python/module:yaml.tokens +- !!python/name:signal.Handlers diff --git a/tests/data/core.schema b/tests/data/core.schema new file mode 100644 index 00000000..f830a277 --- /dev/null +++ b/tests/data/core.schema @@ -0,0 +1,235 @@ +# https://github.com/perlpunk/yaml-test-schema/blob/master/data/schema-core.yaml +--- +'!!bool FALSE': ['bool', 'false()', 'false'] +'!!bool False': ['bool', 'false()', 'false'] +'!!bool TRUE': ['bool', 'true()', 'true'] +'!!bool True': ['bool', 'true()', 'true'] +'!!bool false': ['bool', 'false()', 'false'] +'!!bool true': ['bool', 'true()', 'true'] +'!!float +.INF': ['inf', 'inf()', '.inf'] +'!!float +.Inf': ['inf', 'inf()', '.inf'] +'!!float +.inf': ['inf', 'inf()', '.inf'] +'!!float +0.3e+3': ['float', '300.0', '300.0'] +'!!float +0.3e3': ['float', '300.0', '300.0'] +'!!float -.INF': ['inf', 'inf-neg()', '-.inf'] +'!!float -.Inf': ['inf', 'inf-neg()', '-.inf'] +'!!float -.inf': ['inf', 'inf-neg()', '-.inf'] +'!!float -3.14': ['float', '-3.14', '-3.14'] +'!!float .0': ['float', '0.0', '0.0'] +'!!float .14': ['float', '0.14', '0.14'] +'!!float .3E-1': ['float', '0.03', '0.03'] +'!!float .3e+3': ['float', '300.0', '300.0'] +'!!float .3e3': ['float', '300.0', '300.0'] +'!!float .INF': ['inf', 'inf()', '.inf'] +'!!float .Inf': ['inf', 'inf()', '.inf'] +'!!float .NAN': ['nan', 'nan()', '.nan'] +'!!float .NaN': ['nan', 'nan()', '.nan'] +'!!float .inf': ['inf', 'inf()', '.inf'] +'!!float .nan': ['nan', 'nan()', '.nan'] +'!!float 0.0': ['float', '0.0', '0.0'] +'!!float 0.3e3': ['float', '300.0', '300.0'] +'!!float 001.23': ['float', '1.23', '1.23'] +'!!float 3.': ['float', '3.0', '3.0'] +'!!float 3.14': ['float', '3.14', '3.14'] +'!!float 3.3e+3': ['float', '3300.0', '3300.0'] +'!!int +0': ['int', '0', '0'] +'!!int +23': ['int', '23', '23'] +'!!int -0': ['int', '0', '0'] +'!!int -23': ['int', '-23', '-23'] +'!!int 0': ['int', '0', '0'] +'!!int 0011': ['int', '11', '11'] +'!!int 07': ['int', '7', '7'] +'!!int 0o0': ['int', '0', '0'] +'!!int 0o10': ['int', '8', '8'] +'!!int 0o7': ['int', '7', '7'] +'!!int 0x0': ['int', '0', '0'] +'!!int 0x10': ['int', '16', '16'] +'!!int 0x42': ['int', '66', '66'] +'!!int 0xa': ['int', '10', '10'] +'!!int 23': ['int', '23', '23'] +'!!null #empty': ['null', 'null()', "null"] +'!!null NULL': ['null', 'null()', "null"] +'!!null Null': ['null', 'null()', "null"] +'!!null null': ['null', 'null()', 'null'] +'!!null ~': ['null', 'null()', 'null'] +'!!str #empty': ['str', '', "''"] +'!!str +.INF': ['str', '+.INF', "'+.INF'"] +'!!str +.Inf': ['str', '+.Inf', "'+.Inf'"] +'!!str +.inf': ['str', '+.inf', "'+.inf'"] +'!!str +0': ['str', '+0', "'+0'"] +'!!str +0.3e+3': ['str', '+0.3e+3', "'+0.3e+3'"] +'!!str +0.3e3': ['str', '+0.3e3', "'+0.3e3'"] +'!!str +0100_200': ['str', '+0100_200', "+0100_200"] +'!!str +0b100': ['str', '+0b100', "+0b100"] +'!!str +190:20:30': ['str', '+190:20:30', "+190:20:30"] +'!!str +23': ['str', '+23', "'+23'"] +'!!str -.INF': ['str', '-.INF', "'-.INF'"] +'!!str -.Inf': ['str', '-.Inf', "'-.Inf'"] +'!!str -.inf': ['str', '-.inf', "'-.inf'"] +'!!str -0': ['str', '-0', "'-0'"] +'!!str -0100_200': ['str', '-0100_200', "-0100_200"] +'!!str -0b101': ['str', '-0b101', "-0b101"] +'!!str -0x30': ['str', '-0x30', "-0x30"] +'!!str -190:20:30': ['str', '-190:20:30', "-190:20:30"] +'!!str -23': ['str', '-23', "'-23'"] +'!!str -3.14': ['str', '-3.14', "'-3.14'"] +'!!str .': ['str', '.', '.'] +'!!str .0': ['str', '.0', "'.0'"] +'!!str .14': ['str', '.14', "'.14'"] +'!!str .1_4': ['str', '.1_4', '.1_4'] +'!!str .3E-1': ['str', '.3E-1', "'.3E-1'"] +'!!str .3e+3': ['str', '.3e+3', "'.3e+3'"] +'!!str .3e3': ['str', '.3e3', "'.3e3'"] +'!!str .INF': ['str', '.INF', "'.INF'"] +'!!str .Inf': ['str', '.Inf', "'.Inf'"] +'!!str .NAN': ['str', '.NAN', "'.NAN'"] +'!!str .NaN': ['str', '.NaN', "'.NaN'"] +'!!str ._': ['str', '._', '._'] +'!!str ._14': ['str', '._14', '._14'] +'!!str .inf': ['str', '.inf', "'.inf'"] +'!!str .nan': ['str', '.nan', "'.nan'"] +'!!str 0': ['str', '0', "'0'"] +'!!str 0.0': ['str', '0.0', "'0.0'"] +'!!str 0.3e3': ['str', '0.3e3', "'0.3e3'"] +'!!str 00': ['str', '00', "'00'"] +'!!str 001.23': ['str', '001.23', "'001.23'"] +'!!str 0011': ['str', '0011', "'0011'"] +'!!str 010': ['str', '010', "'010'"] +'!!str 02_0': ['str', '02_0', "02_0"] +'!!str 07': ['str', '07', "'07'"] +'!!str 0b0': ['str', '0b0', "0b0"] +'!!str 0b100_101': ['str', '0b100_101', "0b100_101"] +'!!str 0o0': ['str', '0o0', "'0o0'"] +'!!str 0o10': ['str', '0o10', "'0o10'"] +'!!str 0o7': ['str', '0o7', "'0o7'"] +'!!str 0x0': ['str', '0x0', "'0x0'"] +'!!str 0x2_0': ['str', '0x2_0', "0x2_0"] +'!!str 0xa': ['str', '0xa', "'0xa'"] +'!!str 100_000': ['str', '100_000', "100_000"] +'!!str 190:20:30': ['str', '190:20:30', "190:20:30"] +'!!str 190:20:30.15': ['str', '190:20:30.15', "190:20:30.15"] +'!!str 23': ['str', '23', "'23'"] +'!!str 3.': ['str', '3.', "'3.'"] +'!!str 3.14': ['str', '3.14', "'3.14'"] +'!!str 3.3e+3': ['str', '3.3e+3', "'3.3e+3'"] +'!!str 85.230_15e+03': ['str', '85.230_15e+03', "85.230_15e+03"] +'!!str 85_230.15': ['str', '85_230.15', "85_230.15"] +'!!str FALSE': ['str', 'FALSE', "'FALSE'"] +'!!str False': ['str', 'False', "'False'"] +'!!str N': ['str', 'N', "N"] +'!!str NO': ['str', 'NO', "NO"] +'!!str NULL': ['str', 'NULL', "'NULL'"] +'!!str Null': ['str', 'Null', "'Null'"] +'!!str OFF': ['str', 'OFF', "OFF"] +'!!str ON': ['str', 'ON', "ON"] +'!!str Off': ['str', 'Off', "Off"] +'!!str On': ['str', 'On', "On"] +'!!str TRUE': ['str', 'TRUE', "'TRUE'"] +'!!str True': ['str', 'True', "'True'"] +'!!str Y': ['str', 'Y', "Y"] +'!!str YES': ['str', 'YES', "YES"] +'!!str Yes': ['str', 'Yes', "Yes"] +'!!str _._': ['str', '_._', '_._'] +'!!str false': ['str', 'false', "'false'"] +'!!str n': ['str', 'n', "n"] +'!!str no': ['str', 'no', "no"] +'!!str null': ['str', 'null', "'null'"] +'!!str off': ['str', 'off', "off"] +'!!str on': ['str', 'on', "on"] +'!!str true': ['str', 'true', "'true'"] +'!!str y': ['str', 'y', "y"] +'!!str yes': ['str', 'yes', "yes"] +'!!str ~': ['str', '~', "'~'"] +'#empty': ['null', 'null()', "null"] +'+.INF': ['inf', 'inf()', '.inf'] +'+.Inf': ['inf', 'inf()', '.inf'] +'+.inf': ['inf', 'inf()', '.inf'] +'+0': ['int', '0', '0'] +'+0.3e+3': ['float', '300.0', '300.0'] +'+0.3e3': ['float', '300.0', '300.0'] +'+0100_200': ['str', '+0100_200', '+0100_200'] +'+0b100': ['str', '+0b100', '+0b100'] +'+190:20:30': ['str', '+190:20:30', '+190:20:30'] +'+23': ['int', '23', '23'] +'+3.14': ['float', '3.14', '3.14'] +'-.INF': ['inf', 'inf-neg()', '-.inf'] +'-.Inf': ['inf', 'inf-neg()', '-.inf'] +'-.inf': ['inf', 'inf-neg()', '-.inf'] +'-0': ['int', '0', '0'] +'-0100_200': ['str', '-0100_200', '-0100_200'] +'-0b101': ['str', '-0b101', '-0b101'] +'-0x30': ['str', '-0x30', '-0x30'] +'-190:20:30': ['str', '-190:20:30', '-190:20:30'] +'-23': ['int', '-23', '-23'] +'-3.14': ['float', '-3.14', '-3.14'] +'.': ['str', '.', '.'] +'.0': ['float', '0.0', '0.0'] +'.14': ['float', '0.14', '0.14'] +'.1_4': ['str', '.1_4', '.1_4'] +'.3E-1': ['float', '0.03', '0.03'] +'.3e+3': ['float', '300.0', '300.0'] +'.3e3': ['float', '300.0', '300.0'] +'.INF': ['inf', 'inf()', '.inf'] +'.Inf': ['inf', 'inf()', '.inf'] +'.NAN': ['nan', 'nan()', '.nan'] +'.NaN': ['nan', 'nan()', '.nan'] +'._': ['str', '._', '._'] +'._14': ['str', '._14', '._14'] +'.inf': ['inf', 'inf()', '.inf'] +'.nan': ['nan', 'nan()', '.nan'] +'0': ['int', '0', '0'] +'0.0': ['float', '0.0', '0.0'] +'0.3e3': ['float', '300.0', '300.0'] +'00': ['int', '0', '0'] +'001.23': ['float', '1.23', '1.23'] +'0011': ['int', '11', '11'] +'010': ['int', '10', '10'] +'02_0': ['str', '02_0', '02_0'] +'07': ['int', '7', '7'] +'08': ['int', '8', '8'] +'0b0': ['str', '0b0', '0b0'] +'0b100_101': ['str', '0b100_101', '0b100_101'] +'0o0': ['int', '0', '0'] +'0o10': ['int', '8', '8'] +'0o7': ['int', '7', '7'] +'0x0': ['int', '0', '0'] +'0x10': ['int', '16', '16'] +'0x2_0': ['str', '0x2_0', '0x2_0'] +'0x42': ['int', '66', '66'] +'0xa': ['int', '10', '10'] +'100_000': ['str', '100_000', '100_000'] +'190:20:30': ['str', '190:20:30', '190:20:30'] +'190:20:30.15': ['str', '190:20:30.15', '190:20:30.15'] +'23': ['int', '23', '23'] +'3.': ['float', '3.0', '3.0'] +'3.14': ['float', '3.14', '3.14'] +'3.3e+3': ['float', '3300', '3300.0'] +'3e3': ['float', '3000', '3000.0'] +'85.230_15e+03': ['str', '85.230_15e+03', '85.230_15e+03'] +'85_230.15': ['str', '85_230.15', '85_230.15'] +'FALSE': ['bool', 'false()', 'false'] +'False': ['bool', 'false()', 'false'] +'N': ['str', 'N', "N"] +'NO': ['str', 'NO', "NO"] +'NULL': ['null', 'null()', "null"] +'Null': ['null', 'null()', "null"] +'OFF': ['str', 'OFF', "OFF"] +'ON': ['str', 'ON', "ON"] +'Off': ['str', 'Off', "Off"] +'On': ['str', 'On', "On"] +'TRUE': ['bool', 'true()', 'true'] +'True': ['bool', 'true()', 'true'] +'Y': ['str', 'Y', "Y"] +'YES': ['str', 'YES', "YES"] +'Yes': ['str', 'Yes', "Yes"] +'_._': ['str', '_._', '_._'] +'false': ['bool', 'false()', 'false'] +'n': ['str', 'n', "n"] +'no': ['str', 'no', "no"] +'null': ['null', 'null()', "null"] +'off': ['str', 'off', "off"] +'on': ['str', 'on', "on"] +'true': ['bool', 'true()', 'true'] +'y': ['str', 'y', "y"] +'yes': ['str', 'yes', "yes"] +'~': ['null', 'null()', "null"] diff --git a/tests/data/core.schema-skip b/tests/data/core.schema-skip new file mode 100644 index 00000000..d412d083 --- /dev/null +++ b/tests/data/core.schema-skip @@ -0,0 +1,4 @@ +load: { + } +dump: { + } diff --git a/tests/data/json.schema b/tests/data/json.schema new file mode 100644 index 00000000..676f2909 --- /dev/null +++ b/tests/data/json.schema @@ -0,0 +1,195 @@ +# https://github.com/perlpunk/yaml-test-schema/blob/master/data/schema-json.yaml +--- +'!!bool false': ['bool', 'false()', 'false'] +'!!bool true': ['bool', 'true()', 'true'] +'!!float -3.14': ['float', '-3.14', '-3.14'] +'!!float 0.0': ['float', '0.0', '0.0'] +'!!float 0.3e3': ['float', '300.0', '300.0'] +'!!float 3.': ['float', '3.0', '3.0'] +'!!float 3.14': ['float', '3.14', '3.14'] +'!!int -0': ['int', '0', '0'] +'!!int -23': ['int', '-23', '-23'] +'!!int 0': ['int', '0', '0'] +'!!int 23': ['int', '23', '23'] +'!!null null': ['null', 'null()', 'null'] +'!!str #empty': ['str', '', "''"] +'!!str +.INF': ['str', '+.INF', "+.INF"] +'!!str +.Inf': ['str', '+.Inf', "+.Inf"] +'!!str +.inf': ['str', '+.inf', "+.inf"] +'!!str +0': ['str', '+0', "+0"] +'!!str +0.3e+3': ['str', '+0.3e+3', "+0.3e+3"] +'!!str +0.3e3': ['str', '+0.3e3', "+0.3e3"] +'!!str +0100_200': ['str', '+0100_200', "+0100_200"] +'!!str +0b100': ['str', '+0b100', "+0b100"] +'!!str +190:20:30': ['str', '+190:20:30', "+190:20:30"] +'!!str +23': ['str', '+23', "+23"] +'!!str -.INF': ['str', '-.INF', "-.INF"] +'!!str -.Inf': ['str', '-.Inf', "-.Inf"] +'!!str -.inf': ['str', '-.inf', "-.inf"] +'!!str -0': ['str', '-0', "'-0'"] +'!!str -0100_200': ['str', '-0100_200', "-0100_200"] +'!!str -0b101': ['str', '-0b101', "-0b101"] +'!!str -0x30': ['str', '-0x30', "-0x30"] +'!!str -190:20:30': ['str', '-190:20:30', "-190:20:30"] +'!!str -23': ['str', '-23', "'-23'"] +'!!str -3.14': ['str', '-3.14', "'-3.14'"] +'!!str .': ['str', '.', '.'] +'!!str .0': ['str', '.0', ".0"] +'!!str .14': ['str', '.14', '.14'] +'!!str .1_4': ['str', '.1_4', '.1_4'] +'!!str .3E-1': ['str', '.3E-1', ".3E-1"] +'!!str .3e+3': ['str', '.3e+3', ".3e+3"] +'!!str .3e3': ['str', '.3e3', ".3e3"] +'!!str .INF': ['str', '.INF', ".INF"] +'!!str .Inf': ['str', '.Inf', ".Inf"] +'!!str .NAN': ['str', '.NAN', ".NAN"] +'!!str .NaN': ['str', '.NaN', ".NaN"] +'!!str ._': ['str', '._', '._'] +'!!str ._14': ['str', '._14', '._14'] +'!!str .inf': ['str', '.inf', ".inf"] +'!!str .nan': ['str', '.nan', ".nan"] +'!!str 0': ['str', '0', "'0'"] +'!!str 0.0': ['str', '0.0', "'0.0'"] +'!!str 0.3e3': ['str', '0.3e3', "'0.3e3'"] +'!!str 00': ['str', '00', "00"] +'!!str 001.23': ['str', '001.23', "001.23"] +'!!str 0011': ['str', '0011', "0011"] +'!!str 010': ['str', '010', "010"] +'!!str 02_0': ['str', '02_0', "02_0"] +'!!str 07': ['str', '07', "07"] +'!!str 0b0': ['str', '0b0', "0b0"] +'!!str 0b100_101': ['str', '0b100_101', "0b100_101"] +'!!str 0o0': ['str', '0o0', "0o0"] +'!!str 0o10': ['str', '0o10', "0o10"] +'!!str 0o7': ['str', '0o7', "0o7"] +'!!str 0x0': ['str', '0x0', "0x0"] +'!!str 0x2_0': ['str', '0x2_0', "0x2_0"] +'!!str 0xa': ['str', '0xa', "0xa"] +'!!str 100_000': ['str', '100_000', "100_000"] +'!!str 190:20:30': ['str', '190:20:30', "190:20:30"] +'!!str 190:20:30.15': ['str', '190:20:30.15', "190:20:30.15"] +'!!str 23': ['str', '23', "'23'"] +'!!str 3.': ['str', '3.', "'3.'"] +'!!str 3.14': ['str', '3.14', "'3.14'"] +'!!str 3.3e+3': ['str', '3.3e+3', "'3.3e+3'"] +'!!str 85.230_15e+03': ['str', '85.230_15e+03', "85.230_15e+03"] +'!!str 85_230.15': ['str', '85_230.15', "85_230.15"] +'!!str FALSE': ['str', 'FALSE', "FALSE"] +'!!str False': ['str', 'False', "False"] +'!!str N': ['str', 'N', "N"] +'!!str NO': ['str', 'NO', "NO"] +'!!str NULL': ['str', 'NULL', "NULL"] +'!!str Null': ['str', 'Null', "Null"] +'!!str OFF': ['str', 'OFF', "OFF"] +'!!str ON': ['str', 'ON', "ON"] +'!!str Off': ['str', 'Off', "Off"] +'!!str On': ['str', 'On', "On"] +'!!str TRUE': ['str', 'TRUE', 'TRUE'] +'!!str True': ['str', 'True', 'True'] +'!!str Y': ['str', 'Y', "Y"] +'!!str YES': ['str', 'YES', "YES"] +'!!str Yes': ['str', 'Yes', "Yes"] +'!!str _._': ['str', '_._', '_._'] +'!!str false': ['str', 'false', "'false'"] +'!!str n': ['str', 'n', "n"] +'!!str no': ['str', 'no', "no"] +'!!str null': ['str', 'null', "'null'"] +'!!str off': ['str', 'off', "off"] +'!!str on': ['str', 'on', "on"] +'!!str true': ['str', 'true', "'true'"] +'!!str y': ['str', 'y', "y"] +'!!str yes': ['str', 'yes', "yes"] +'!!str ~': ['str', '~', '~'] +'#empty': ['str', '', "''"] +'+.INF': ['str', '+.INF', '+.INF'] +'+.Inf': ['str', '+.Inf', '+.Inf'] +'+.inf': ['str', '+.inf', '+.inf'] +'+0': ['str', '+0', '+0'] +'+0.3e+3': ['str', '+0.3e+3', '+0.3e+3'] +'+0.3e3': ['str', '+0.3e3', '+0.3e3'] +'+0100_200': ['str', '+0100_200', '+0100_200'] +'+0b100': ['str', '+0b100', '+0b100'] +'+190:20:30': ['str', '+190:20:30', '+190:20:30'] +'+23': ['str', '+23', '+23'] +'+3.14': ['str', '+3.14', '+3.14'] +'-.INF': ['str', '-.INF', '-.INF'] +'-.Inf': ['str', '-.Inf', '-.Inf'] +'-.inf': ['str', '-.inf', '-.inf'] +'-0': ['int', '0', '0'] +'-0100_200': ['str', '-0100_200', '-0100_200'] +'-0b101': ['str', '-0b101', '-0b101'] +'-0x30': ['str', '-0x30', '-0x30'] +'-190:20:30': ['str', '-190:20:30', '-190:20:30'] +'-23': ['int', '-23', '-23'] +'-3.14': ['float', '-3.14', '-3.14'] +'.': ['str', '.', '.'] +'.0': ['str', '.0', '.0'] +'.14': ['str', '.14', '.14'] +'.1_4': ['str', '.1_4', '.1_4'] +'.3E-1': ['str', '.3E-1', '.3E-1'] +'.3e+3': ['str', '.3e+3', '.3e+3'] +'.3e3': ['str', '.3e3', '.3e3'] +'.INF': ['str', '.INF', '.INF'] +'.Inf': ['str', '.Inf', '.Inf'] +'.NAN': ['str', '.NAN', '.NAN'] +'.NaN': ['str', '.NaN', '.NaN'] +'._': ['str', '._', '._'] +'._14': ['str', '._14', '._14'] +'.inf': ['str', '.inf', '.inf'] +'.nan': ['str', '.nan', '.nan'] +'0': ['int', '0', '0'] +'0.0': ['float', '0.0', '0.0'] +'0.3e3': ['float', '300.0', '300.0'] +'00': ['str', '00', '00'] +'001.23': ['str', '001.23', '001.23'] +'0011': ['str', '0011', '0011'] +'010': ['str', '010', '010'] +'02_0': ['str', '02_0', '02_0'] +'07': ['str', '07', '07'] +'08': ['str', '08', '08'] +'0b0': ['str', '0b0', '0b0'] +'0b100_101': ['str', '0b100_101', '0b100_101'] +'0o0': ['str', '0o0', '0o0'] +'0o10': ['str', '0o10', '0o10'] +'0o7': ['str', '0o7', '0o7'] +'0x0': ['str', '0x0', '0x0'] +'0x10': ['str', '0x10', '0x10'] +'0x2_0': ['str', '0x2_0', '0x2_0'] +'0x42': ['str', '0x42', '0x42'] +'0xa': ['str', '0xa', '0xa'] +'100_000': ['str', '100_000', '100_000'] +'190:20:30': ['str', '190:20:30', '190:20:30'] +'190:20:30.15': ['str', '190:20:30.15', '190:20:30.15'] +'23': ['int', '23', '23'] +'3.': ['float', '3.0', '3.0'] +'3.14': ['float', '3.14', '3.14'] +'3.3e+3': ['float', '3300', '3300.0'] +'3e3': ['float', '3000', '3000.0'] +'85.230_15e+03': ['str', '85.230_15e+03', '85.230_15e+03'] +'85_230.15': ['str', '85_230.15', '85_230.15'] +'FALSE': ['str', 'FALSE', 'FALSE'] +'False': ['str', 'False', 'False'] +'N': ['str', 'N', "N"] +'NO': ['str', 'NO', "NO"] +'NULL': ['str', 'NULL', 'NULL'] +'Null': ['str', 'Null', 'Null'] +'OFF': ['str', 'OFF', "OFF"] +'ON': ['str', 'ON', "ON"] +'Off': ['str', 'Off', "Off"] +'On': ['str', 'On', "On"] +'TRUE': ['str', 'TRUE', 'TRUE'] +'True': ['str', 'True', 'True'] +'Y': ['str', 'Y', "Y"] +'YES': ['str', 'YES', "YES"] +'Yes': ['str', 'Yes', "Yes"] +'_._': ['str', '_._', '_._'] +'false': ['bool', 'false()', 'false'] +'n': ['str', 'n', "n"] +'no': ['str', 'no', "no"] +'null': ['null', 'null()', "null"] +'off': ['str', 'off', "off"] +'on': ['str', 'on', "on"] +'true': ['bool', 'true()', 'true'] +'y': ['str', 'y', "y"] +'yes': ['str', 'yes', "yes"] +'~': ['str', '~', '~'] diff --git a/tests/data/json.schema-skip b/tests/data/json.schema-skip new file mode 100644 index 00000000..d2fe12e4 --- /dev/null +++ b/tests/data/json.schema-skip @@ -0,0 +1,6 @@ +load: { + } +dump: { + '#empty': 1, + '!!str #empty': 1, + } diff --git a/tests/data/yaml11.schema b/tests/data/yaml11.schema new file mode 100644 index 00000000..e2791aa7 --- /dev/null +++ b/tests/data/yaml11.schema @@ -0,0 +1,264 @@ +# https://github.com/perlpunk/yaml-test-schema/blob/master/data/schema-yaml11.yaml +--- +'!!bool FALSE': ['bool', 'false()', 'false'] +'!!bool False': ['bool', 'false()', 'false'] +'!!bool N': ['bool', 'false()', "false"] +'!!bool NO': ['bool', 'false()', "false"] +'!!bool No': ['bool', 'false()', "false"] +'!!bool OFF': ['bool', 'false()', "false"] +'!!bool ON': ['bool', 'true()', "true"] +'!!bool Off': ['bool', 'false()', "false"] +'!!bool On': ['bool', 'true()', "true"] +'!!bool TRUE': ['bool', 'true()', 'true'] +'!!bool True': ['bool', 'true()', 'true'] +'!!bool Y': ['bool', 'true()', "true"] +'!!bool YES': ['bool', 'true()', "true"] +'!!bool Yes': ['bool', 'true()', "true"] +'!!bool false': ['bool', 'false()', 'false'] +'!!bool n': ['bool', 'false()', "false"] +'!!bool no': ['bool', 'false()', "false"] +'!!bool off': ['bool', 'false()', "false"] +'!!bool on': ['bool', 'true()', "true"] +'!!bool true': ['bool', 'true()', 'true'] +'!!bool y': ['bool', 'true()', "true"] +'!!bool yes': ['bool', 'true()', "true"] +'!!float +.INF': ['inf', 'inf()', '.inf'] +'!!float +.Inf': ['inf', 'inf()', '.inf'] +'!!float +.inf': ['inf', 'inf()', '.inf'] +'!!float +0.3e+3': ['float', '300.0', '300.0'] +'!!float -.INF': ['inf', 'inf-neg()', '-.inf'] +'!!float -.Inf': ['inf', 'inf-neg()', '-.inf'] +'!!float -.inf': ['inf', 'inf-neg()', '-.inf'] +'!!float -3.14': ['float', '-3.14', '-3.14'] +'!!float .0': ['float', '0.0', '0.0'] +'!!float .14': ['float', '0.14', '0.14'] +'!!float .1_4': ['float', '0.14', '0.14'] +'!!float .3E-1': ['float', '0.03', '0.03'] +'!!float .3e+3': ['float', '300.0', '300.0'] +'!!float .INF': ['inf', 'inf()', '.inf'] +'!!float .Inf': ['inf', 'inf()', '.inf'] +'!!float .NAN': ['nan', 'nan()', '.nan'] +'!!float .NaN': ['nan', 'nan()', '.nan'] +'!!float .inf': ['inf', 'inf()', '.inf'] +'!!float .nan': ['nan', 'nan()', '.nan'] +'!!float 0.0': ['float', '0.0', '0.0'] +'!!float 001.23': ['float', '1.23', '1.23'] +'!!float 190:20:30.15': ['float', '685230.15', '685230.15'] +'!!float 3.': ['float', '3.0', '3.0'] +'!!float 3.14': ['float', '3.14', '3.14'] +'!!float 3.3e+3': ['float', '3300.0', '3300.0'] +'!!float 85.230_15e+03': ['float', '85230.15', '85230.15'] +'!!float 85_230.15': ['float', '85230.15', '85230.15'] +'!!int +0': ['int', '0', '0'] +'!!int +0100_200': ['int', '32896', '32896'] +'!!int +0b100': ['int', '4', '4'] +'!!int +190:20:30': ['int', '685230', '685230'] +'!!int +23': ['int', '23', '23'] +'!!int -0': ['int', '0', '0'] +'!!int -0100_200': ['int', '-32896', '-32896'] +'!!int -0b101': ['int', '-5', '-5'] +'!!int -0x30': ['int', '-48', '-48'] +'!!int -190:20:30': ['int', '-685230', '-685230'] +'!!int -23': ['int', '-23', '-23'] +'!!int 0': ['int', '0', '0'] +'!!int 00': ['int', '0', '0'] +'!!int 0011': ['int', '9', '9'] +'!!int 010': ['int', '8', '8'] +'!!int 02_0': ['int', '16', '16'] +'!!int 07': ['int', '7', '7'] +'!!int 0b0': ['int', '0', '0'] +'!!int 0b100_101': ['int', '37', '37'] +'!!int 0x0': ['int', '0', '0'] +'!!int 0x10': ['int', '16', '16'] +'!!int 0x2_0': ['int', '32', '32'] +'!!int 0x42': ['int', '66', '66'] +'!!int 0xa': ['int', '10', '10'] +'!!int 100_000': ['int', '100000', '100000'] +'!!int 190:20:30': ['int', '685230', '685230'] +'!!int 23': ['int', '23', '23'] +'!!null #empty': ['null', 'null()', "null"] +'!!null NULL': ['null', 'null()', "null"] +'!!null Null': ['null', 'null()', "null"] +'!!null null': ['null', 'null()', 'null'] +'!!null ~': ['null', 'null()', 'null'] +'!!str #empty': ['str', '', "''"] +'!!str +.INF': ['str', '+.INF', "'+.INF'"] +'!!str +.Inf': ['str', '+.Inf', "'+.Inf'"] +'!!str +.inf': ['str', '+.inf', "'+.inf'"] +'!!str +0': ['str', '+0', "'+0'"] +'!!str +0.3e+3': ['str', '+0.3e+3', "'+0.3e+3'"] +'!!str +0.3e3': ['str', '+0.3e3', "+0.3e3"] +'!!str +0100_200': ['str', '+0100_200', "'+0100_200'"] +'!!str +0b100': ['str', '+0b100', "'+0b100'"] +'!!str +190:20:30': ['str', '+190:20:30', "'+190:20:30'"] +'!!str +23': ['str', '+23', "'+23'"] +'!!str -.INF': ['str', '-.INF', "'-.INF'"] +'!!str -.Inf': ['str', '-.Inf', "'-.Inf'"] +'!!str -.inf': ['str', '-.inf', "'-.inf'"] +'!!str -0': ['str', '-0', "'-0'"] +'!!str -0100_200': ['str', '-0100_200', "'-0100_200'"] +'!!str -0b101': ['str', '-0b101', "'-0b101'"] +'!!str -0x30': ['str', '-0x30', "'-0x30'"] +'!!str -190:20:30': ['str', '-190:20:30', "'-190:20:30'"] +'!!str -23': ['str', '-23', "'-23'"] +'!!str -3.14': ['str', '-3.14', "'-3.14'"] +'!!str .': ['str', '.', '.'] +'!!str .0': ['str', '.0', "'.0'"] +'!!str .14': ['str', '.14', "'.14'"] +'!!str .1_4': ['str', '.1_4', "'.1_4'"] +'!!str .3E-1': ['str', '.3E-1', "'.3E-1'"] +'!!str .3e+3': ['str', '.3e+3', "'.3e+3'"] +'!!str .3e3': ['str', '.3e3', ".3e3"] +'!!str .INF': ['str', '.INF', "'.INF'"] +'!!str .Inf': ['str', '.Inf', "'.Inf'"] +'!!str .NAN': ['str', '.NAN', "'.NAN'"] +'!!str .NaN': ['str', '.NaN', "'.NaN'"] +'!!str ._': ['str', '._', '._'] +'!!str ._14': ['str', '._14', '._14'] +'!!str .inf': ['str', '.inf', "'.inf'"] +'!!str .nan': ['str', '.nan', "'.nan'"] +'!!str 0': ['str', '0', "'0'"] +'!!str 0.0': ['str', '0.0', "'0.0'"] +'!!str 0.3e3': ['str', '0.3e3', "0.3e3"] +'!!str 00': ['str', '00', "'00'"] +'!!str 001.23': ['str', '001.23', "'001.23'"] +'!!str 0011': ['str', '0011', "'0011'"] +'!!str 010': ['str', '010', "'010'"] +'!!str 02_0': ['str', '02_0', "'02_0'"] +'!!str 07': ['str', '07', "'07'"] +'!!str 0b0': ['str', '0b0', "'0b0'"] +'!!str 0b100_101': ['str', '0b100_101', "'0b100_101'"] +'!!str 0o0': ['str', '0o0', "0o0"] +'!!str 0o10': ['str', '0o10', "0o10"] +'!!str 0o7': ['str', '0o7', "0o7"] +'!!str 0x0': ['str', '0x0', "'0x0'"] +'!!str 0x2_0': ['str', '0x2_0', "'0x2_0'"] +'!!str 0xa': ['str', '0xa', "'0xa'"] +'!!str 100_000': ['str', '100_000', "'100_000'"] +'!!str 190:20:30': ['str', '190:20:30', "'190:20:30'"] +'!!str 190:20:30.15': ['str', '190:20:30.15', "'190:20:30.15'"] +'!!str 23': ['str', '23', "'23'"] +'!!str 3.': ['str', '3.', "'3.'"] +'!!str 3.14': ['str', '3.14', "'3.14'"] +'!!str 3.3e+3': ['str', '3.3e+3', "'3.3e+3'"] +'!!str 85.230_15e+03': ['str', '85.230_15e+03', "'85.230_15e+03'"] +'!!str 85_230.15': ['str', '85_230.15', "'85_230.15'"] +'!!str FALSE': ['str', 'FALSE', "'FALSE'"] +'!!str False': ['str', 'False', "'False'"] +'!!str N': ['str', 'N', "'N'"] +'!!str NO': ['str', 'NO', "'NO'"] +'!!str NULL': ['str', 'NULL', "'NULL'"] +'!!str Null': ['str', 'Null', "'Null'"] +'!!str OFF': ['str', 'OFF', "'OFF'"] +'!!str ON': ['str', 'ON', "'ON'"] +'!!str Off': ['str', 'Off', "'Off'"] +'!!str On': ['str', 'On', "'On'"] +'!!str TRUE': ['str', 'TRUE', "'TRUE'"] +'!!str True': ['str', 'True', "'True'"] +'!!str Y': ['str', 'Y', "'Y'"] +'!!str YES': ['str', 'YES', "'YES'"] +'!!str Yes': ['str', 'Yes', "'Yes'"] +'!!str _._': ['str', '_._', '_._'] +'!!str false': ['str', 'false', "'false'"] +'!!str n': ['str', 'n', "'n'"] +'!!str no': ['str', 'no', "'no'"] +'!!str null': ['str', 'null', "'null'"] +'!!str off': ['str', 'off', "'off'"] +'!!str on': ['str', 'on', "'on'"] +'!!str true': ['str', 'true', "'true'"] +'!!str y': ['str', 'y', "'y'"] +'!!str yes': ['str', 'yes', "'yes'"] +'!!str ~': ['str', '~', "'~'"] +'#empty': ['null', 'null()', "null"] +'+.INF': ['inf', 'inf()', '.inf'] +'+.Inf': ['inf', 'inf()', '.inf'] +'+.inf': ['inf', 'inf()', '.inf'] +'+0': ['int', '0', '0'] +'+0.3e+3': ['float', '300.0', '300.0'] +'+0.3e3': ['str', '+0.3e3', '+0.3e3'] +'+0100_200': ['int', '32896', '32896'] +'+0b100': ['int', '4', '4'] +'+190:20:30': ['int', '685230', '685230'] +'+23': ['int', '23', '23'] +'+3.14': ['float', '3.14', '3.14'] +'-.INF': ['inf', 'inf-neg()', '-.inf'] +'-.Inf': ['inf', 'inf-neg()', '-.inf'] +'-.inf': ['inf', 'inf-neg()', '-.inf'] +'-0': ['int', '0', '0'] +'-0100_200': ['int', '-32896', '-32896'] +'-0b101': ['int', '-5', '-5'] +'-0x30': ['int', '-48', '-48'] +'-190:20:30': ['int', '-685230', '-685230'] +'-23': ['int', '-23', '-23'] +'-3.14': ['float', '-3.14', '-3.14'] +'.': ['str', '.', '.'] +'.0': ['float', '0.0', '0.0'] +'.14': ['float', '0.14', '0.14'] +'.1_4': ['float', '0.14', '0.14'] +'.3E-1': ['float', '0.03', '0.03'] +'.3e+3': ['float', '300.0', '300.0'] +'.3e3': ['str', '.3e3', '.3e3'] +'.INF': ['inf', 'inf()', '.inf'] +'.Inf': ['inf', 'inf()', '.inf'] +'.NAN': ['nan', 'nan()', '.nan'] +'.NaN': ['nan', 'nan()', '.nan'] +'._': ['str', '._', '._'] +'._14': ['str', '._14', '._14'] +'.inf': ['inf', 'inf()', '.inf'] +'.nan': ['nan', 'nan()', '.nan'] +'0': ['int', '0', '0'] +'0.0': ['float', '0.0', '0.0'] +'0.3e3': ['str', '0.3e3', '0.3e3'] +'00': ['int', '0', '0'] +'001.23': ['float', '1.23', '1.23'] +'0011': ['int', '9', '9'] +'010': ['int', '8', '8'] +'02_0': ['int', '16', '16'] +'07': ['int', '7', '7'] +'08': ['str', '08', '08'] +'0b0': ['int', '0', '0'] +'0b100_101': ['int', '37', '37'] +'0o0': ['str', '0o0', '0o0'] +'0o10': ['str', '0o10', '0o10'] +'0o7': ['str', '0o7', '0o7'] +'0x0': ['int', '0', '0'] +'0x10': ['int', '16', '16'] +'0x2_0': ['int', '32', '32'] +'0x42': ['int', '66', '66'] +'0xa': ['int', '10', '10'] +'100_000': ['int', '100000', '100000'] +'190:20:30': ['int', '685230', '685230'] +'190:20:30.15': ['float', '685230.15', '685230.15'] +'23': ['int', '23', '23'] +'3.': ['float', '3.0', '3.0'] +'3.14': ['float', '3.14', '3.14'] +'3.3e+3': ['float', '3300', '3300.0'] +'3e3': ['str', '3e3', '3e3'] +'85.230_15e+03': ['float', '85230.15', '85230.15'] +'85_230.15': ['float', '85230.15', '85230.15'] +'FALSE': ['bool', 'false()', 'false'] +'False': ['bool', 'false()', 'false'] +'N': ['bool', 'false()', "false"] +'NO': ['bool', 'false()', "false"] +'NULL': ['null', 'null()', "null"] +'Null': ['null', 'null()', "null"] +'OFF': ['bool', 'false()', "false"] +'ON': ['bool', 'true()', "true"] +'Off': ['bool', 'false()', "false"] +'On': ['bool', 'true()', "true"] +'TRUE': ['bool', 'true()', 'true'] +'True': ['bool', 'true()', 'true'] +'Y': ['bool', 'true()', "true"] +'YES': ['bool', 'true()', "true"] +'Yes': ['bool', 'true()', "true"] +'_._': ['str', '_._', '_._'] +'false': ['bool', 'false()', 'false'] +'n': ['bool', 'false()', "false"] +'no': ['bool', 'false()', "false"] +'null': ['null', 'null()', "null"] +'off': ['bool', 'false()', "false"] +'on': ['bool', 'true()', "true"] +'true': ['bool', 'true()', 'true'] +'y': ['bool', 'true()', "true"] +'yes': ['bool', 'true()', "true"] +'~': ['null', 'null()', "null"] diff --git a/tests/data/yaml11.schema-skip b/tests/data/yaml11.schema-skip new file mode 100644 index 00000000..4fe0f0bf --- /dev/null +++ b/tests/data/yaml11.schema-skip @@ -0,0 +1,9 @@ +load: { + 'Y': 1, 'y': 1, 'N': 1, 'n': 1, + '!!bool Y': 1, '!!bool N': 1, '!!bool n': 1, '!!bool y': 1, + '._', '!!str ._', + '._14', '!!str ._14' + } +dump: { + '!!str N': 1, '!!str Y': 1, '!!str n': 1, '!!str y': 1, + } diff --git a/tests/lib/test_constructor.py b/tests/lib/test_constructor.py index f9a50770..28987d29 100644 --- a/tests/lib/test_constructor.py +++ b/tests/lib/test_constructor.py @@ -5,6 +5,9 @@ import datetime import yaml.tokens +# Import any packages here that need to be referenced in .code files. +import signal + def execute(code): global value exec(code) diff --git a/tests/lib/test_schema.py b/tests/lib/test_schema.py new file mode 100644 index 00000000..7bc84b7f --- /dev/null +++ b/tests/lib/test_schema.py @@ -0,0 +1,188 @@ +import yaml +import os +import sys +import pprint +import math + +def check_bool(value, expected): + if expected == 'false()' and value is False: + return 1 + if expected == 'true()' and value is True: + return 1 + print(value) + print(expected) + return 0 + +def check_int(value, expected): + if (int(expected) == value): + return 1 + print(value) + print(expected) + return 0 + +def check_float(value, expected): + if expected == 'inf()': + if value == math.inf: + return 1 + elif expected == 'inf-neg()': + if value == -math.inf: + return 1 + elif expected == 'nan()': + if math.isnan(value): + return 1 + elif (float(expected) == value): + return 1 + else: + print(value) + print(expected) + return 0 + +def check_str(value, expected): + if value == expected: + return 1 + print(value) + print(expected) + return 0 + + +def _fail(input, test): + print("Input: >>" + input + "<<") + print(test) + +class MyCoreLoader(yaml.BaseLoader): pass +class MyJSONLoader(yaml.BaseLoader): pass +class MyCoreDumper(yaml.CommonDumper): pass +class MyJSONDumper(yaml.CommonDumper): pass + +MyCoreLoader.init_tags('core') +MyJSONLoader.init_tags('json') + +MyCoreDumper.init_tags('core') +MyJSONDumper.init_tags('json') + +# The tests/data/yaml11.schema file is copied from +# https://github.com/perlpunk/yaml-test-schema/blob/master/data/schema-yaml11.yaml +def test_implicit_resolver(data_filename, skip_filename, verbose=False): + types = { + 'str': [str, check_str], + 'int': [int, check_int], + 'float': [float, check_float], + 'inf': [float, check_float], + 'nan': [float, check_float], + 'bool': [bool, check_bool], + } + loaders = { + 'yaml11': yaml.SafeLoader, + 'core': MyCoreLoader, + 'json': MyJSONLoader, + } + dumpers = { + 'yaml11': yaml.SafeDumper, + 'core': MyCoreDumper, + 'json': MyJSONDumper, + } + loadername = os.path.splitext(os.path.basename(data_filename))[0] + print('==================') + print(loadername) + with open(skip_filename, 'rb') as file: + skipdata = yaml.load(file, Loader=yaml.SafeLoader) + skip_load = skipdata['load'] + skip_dump = skipdata['dump'] + if verbose: + print(skip_load) + with open(data_filename, 'rb') as file: + tests = yaml.load(file, Loader=yaml.SafeLoader) + + i = 0 + fail = 0 + for i, (input, test) in enumerate(sorted(tests.items())): + if verbose: + print('-------------------- ' + str(i)) + + # Skip known loader bugs + if input in skip_load: + continue + + exp_type = test[0] + data = test[1] + exp_dump = test[2] + + # Test loading + try: + doc_input = """---\n""" + input + loaded = yaml.load(doc_input, Loader=loaders[loadername]) + except: + print("Error:", sys.exc_info()[0], '(', sys.exc_info()[1], ')') + fail+=1 + _fail(input, test) + continue + + if verbose: + print(input) + print(test) + print(loaded) + print(type(loaded)) + + if exp_type == 'null': + if loaded is None: + pass + else: + fail+=1 + _fail(input, test) + else: + t = types[exp_type][0] + code = types[exp_type][1] + + if isinstance(loaded, t): + if code(loaded, data): + pass + else: + fail+=1 + print("Expected data: >>" + str(data) + "<< Got: >>" + str(loaded) + "<<") + _fail(input, test) + else: + fail+=1 + print("Expected type: >>" + exp_type + "<< Got: >>" + str(loaded) + "<<") + _fail(input, test) + + # Skip known dumper bugs + if input in skip_dump: + continue + + dump = yaml.dump(loaded, explicit_end=False, Dumper=dumpers[loadername]) + # strip trailing newlines and footers + if (dump == '...\n'): + dump = '' + if dump.endswith('\n...\n'): + dump = dump[:-5] + if dump.endswith('\n'): + dump = dump[:-1] + if dump == exp_dump: + pass + else: + print("Compare: >>" + dump + "<< >>" + exp_dump + "<<") + print(skip_dump) + print(input) + print(test) + print(loaded) + print(type(loaded)) + fail+=1 + _fail(input, test) + +# if i >= 80: +# break + + if fail > 0: + print("Failed " + str(fail) + " / " + str(i) + " tests") + assert(False) + else: + print("Passed " + str(i) + " tests") + print("Skipped " + str(len(skip_load)) + " load tests") + print("Skipped " + str(len(skip_dump)) + " dump tests") + +test_implicit_resolver.unittest = ['.schema', '.schema-skip'] + +if __name__ == '__main__': + import test_appliance + test_appliance.run(globals()) + diff --git a/tests/lib/test_yaml.py b/tests/lib/test_yaml.py index 352cd8d1..7b3d8f9d 100644 --- a/tests/lib/test_yaml.py +++ b/tests/lib/test_yaml.py @@ -14,6 +14,8 @@ from test_sort_keys import * from test_multi_constructor import * +from test_schema import * + if __name__ == '__main__': import test_appliance test_appliance.run(globals()) diff --git a/yaml/_yaml.pyx b/yaml/_yaml.pyx index ff4efe80..47dedab5 100644 --- a/yaml/_yaml.pyx +++ b/yaml/_yaml.pyx @@ -1,6 +1,14 @@ import yaml +# FIXME: change all the below stuff to from X import Y to avoid this +# import submodules explicitly since we're getting loaded earlier +import yaml.emitter +import yaml.parser +import yaml.reader +import yaml.scanner +import yaml.serializer + def get_version_string(): cdef char *value value = yaml_get_version_string()