From bff366fd21e988667979d688c7223ff275e045e9 Mon Sep 17 00:00:00 2001 From: yukinarit Date: Sat, 26 Oct 2024 21:39:10 +0900 Subject: [PATCH 1/2] Implement deny_unknown_fields class attribute Added a `deny_unknown_fields` feature for stricter deserialization, rejecting any unexpected or unrecognized fields during deserialization. --- examples/deny_unknown_fields.py | 20 +++++++++++ examples/runner.py | 4 ++- serde/__init__.py | 4 +++ serde/de.py | 26 +++++++++++++- tests/test_de.py | 61 +++++++++++++++++++++++++++++++++ 5 files changed, 113 insertions(+), 2 deletions(-) create mode 100644 examples/deny_unknown_fields.py diff --git a/examples/deny_unknown_fields.py b/examples/deny_unknown_fields.py new file mode 100644 index 00000000..a15322e8 --- /dev/null +++ b/examples/deny_unknown_fields.py @@ -0,0 +1,20 @@ +from serde import serde, SerdeError +from serde.json import from_json + + +@serde(deny_unknown_fields=True) +class Foo: + a: int + b: str + + +def main() -> None: + try: + s = '{"a": 10, "b": "foo", "c": 100.0, "d": true}' + print(f"From Json: {from_json(Foo, s)}") + except SerdeError: + pass + + +if __name__ == "__main__": + main() diff --git a/examples/runner.py b/examples/runner.py index a8398078..b56daa1c 100644 --- a/examples/runner.py +++ b/examples/runner.py @@ -35,6 +35,7 @@ def run_all() -> None: import pep681 import plain_dataclass import plain_dataclass_class_attribute + import deny_unknown_fields import python_pickle import recursive import recursive_list @@ -107,6 +108,7 @@ def run_all() -> None: run(class_var) run(plain_dataclass) run(plain_dataclass_class_attribute) + run(deny_unknown_fields) run(msg_pack) run(primitive_subclass) run(kw_only) @@ -133,6 +135,6 @@ def run(module: typing.Any) -> None: try: run_all() print("-----------------") - print("all tests passed successfully!") + print("all examples completed successfully!") except Exception: sys.exit(1) diff --git a/serde/__init__.py b/serde/__init__.py index 02dfa00f..d6f4c7e0 100644 --- a/serde/__init__.py +++ b/serde/__init__.py @@ -124,6 +124,7 @@ def serde( serialize_class_var: bool = False, class_serializer: Optional[ClassSerializer] = None, class_deserializer: Optional[ClassDeserializer] = None, + deny_unknown_fields: bool = False, ) -> Type[T]: ... @@ -140,6 +141,7 @@ def serde( serialize_class_var: bool = False, class_serializer: Optional[ClassSerializer] = None, class_deserializer: Optional[ClassDeserializer] = None, + deny_unknown_fields: bool = False, ) -> Callable[[type[T]], type[T]]: ... @@ -156,6 +158,7 @@ def serde( serialize_class_var: bool = False, class_serializer: Optional[ClassSerializer] = None, class_deserializer: Optional[ClassDeserializer] = None, + deny_unknown_fields: bool = False, ) -> Any: """ serde decorator. Keyword arguments are passed in `serialize` and `deserialize`. @@ -187,6 +190,7 @@ def wrap(cls: Any) -> Any: type_check=type_check, serialize_class_var=serialize_class_var, class_deserializer=class_deserializer, + deny_unknown_fields=deny_unknown_fields, ) return cls diff --git a/serde/de.py b/serde/de.py index 1e88e121..0ba01e9c 100644 --- a/serde/de.py +++ b/serde/de.py @@ -192,6 +192,7 @@ def deserialize( tagging: Tagging = DefaultTagging, type_check: TypeCheck = strict, class_deserializer: Optional[ClassDeserializer] = None, + deny_unknown_fields: bool = False, **kwargs: Any, ) -> type[T]: """ @@ -329,7 +330,12 @@ def wrap(cls: type[T]) -> type[T]: scope, FROM_DICT, render_from_dict( - cls, rename_all, deserializer, type_check, class_deserializer=class_deserializer + cls, + rename_all, + deserializer, + type_check, + class_deserializer=class_deserializer, + deny_unknown_fields=deny_unknown_fields, ), g, ) @@ -1041,6 +1047,13 @@ def {{func}}(cls=cls, maybe_generic=None, maybe_generic_type_vars=None, data=Non if reuse_instances is None: reuse_instances = {{serde_scope.reuse_instances_default}} + {% if deny_unknown_fields %} + known_fields = {{ known_fields }} + unknown_fields = set((data or {}).keys()) - known_fields + if unknown_fields: + raise SerdeError(f'unknown fields: {unknown_fields}, expected one of {known_fields}') + {% endif %} + maybe_generic_type_vars = maybe_generic_type_vars or {{cls_type_vars}} {% for f in fields %} @@ -1143,12 +1156,18 @@ def render_from_iter( return res +def get_known_fields(f: DeField[Any], rename_all: Optional[str]) -> list[str]: + names: list[str] = [f.conv_name(rename_all)] + return names + f.alias + + def render_from_dict( cls: type[Any], rename_all: Optional[str] = None, legacy_class_deserializer: Optional[DeserializeFunc] = None, type_check: TypeCheck = strict, class_deserializer: Optional[ClassDeserializer] = None, + deny_unknown_fields: bool = False, ) -> str: renderer = Renderer( FROM_DICT, @@ -1159,6 +1178,9 @@ def render_from_dict( class_name=typename(cls), ) fields = list(filter(renderable, defields(cls))) + known_fields = set( + itertools.chain.from_iterable([get_known_fields(f, rename_all) for f in fields]) + ) res = jinja2_env.get_template("dict").render( func=FROM_DICT, serde_scope=getattr(cls, SERDE_SCOPE), @@ -1167,6 +1189,8 @@ def render_from_dict( cls_type_vars=get_type_var_names(cls), rvalue=renderer.render, arg=functools.partial(to_arg, rename_all=rename_all), + deny_unknown_fields=deny_unknown_fields, + known_fields=known_fields, ) if renderer.import_numpy: diff --git a/tests/test_de.py b/tests/test_de.py index d6e2d10b..6bb64dde 100644 --- a/tests/test_de.py +++ b/tests/test_de.py @@ -1,5 +1,8 @@ +import pytest from decimal import Decimal from typing import Union, Optional +from serde import serde, SerdeError, field +from serde.json import from_json from serde.de import deserialize, from_obj, Renderer, DeField @@ -125,3 +128,61 @@ class Foo: rendered_foo = f"Foo.__serde__.funcs['foo'](data=data[\"f\"], {kwargs})" rendered_opt = f'({rendered_foo}) if data.get("f") is not None else None' assert rendered == rendered_opt + + +def test_deny_unknown_fields() -> None: + @serde(deny_unknown_fields=True) + class Foo: + a: int + b: str + + with pytest.raises(SerdeError): + from_json(Foo, '{"a": 10, "b": "foo", "c": 100.0, "d": true}') + + f = from_json(Foo, '{"a": 10, "b": "foo"}') + assert f.a == 10 + assert f.b == "foo" + + +def test_deny_renamed_unknown_fields() -> None: + @serde(deny_unknown_fields=True) + class Foo: + a: int + b: str = field(rename="B") + + with pytest.raises(SerdeError): + from_json(Foo, '{"a": 10, "b": "foo"}') + + f = from_json(Foo, '{"a": 10, "B": "foo"}') + assert f.a == 10 + assert f.b == "foo" + + @serde(rename_all="constcase", deny_unknown_fields=True) + class Bar: + a: int + b: str + + with pytest.raises(SerdeError): + from_json(Bar, '{"a": 10, "b": "foo"}') + + b = from_json(Bar, '{"A": 10, "B": "foo"}') + assert b.a == 10 + assert b.b == "foo" + + +def test_deny_aliased_unknown_fields() -> None: + @serde(deny_unknown_fields=True) + class Foo: + a: int + b: str = field(alias=["B"]) # type: ignore + + with pytest.raises(SerdeError): + from_json(Foo, '{"a": 10, "b": "foo", "c": 100.0, "d": true}') + + f = from_json(Foo, '{"a": 10, "b": "foo"}') + assert f.a == 10 + assert f.b == "foo" + + f = from_json(Foo, '{"a": 10, "B": "foo"}') + assert f.a == 10 + assert f.b == "foo" From dac6c33044e221dcca12c25a55bd6619de8ca40a Mon Sep 17 00:00:00 2001 From: yukinarit Date: Mon, 28 Oct 2024 16:36:25 +0900 Subject: [PATCH 2/2] Update docs for deny_unknown_fields --- docs/en/class-attributes.md | 20 ++++++++++++++++++++ docs/ja/class-attributes.md | 22 +++++++++++++++++++++- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/docs/en/class-attributes.md b/docs/en/class-attributes.md index cff16a50..1f395bcc 100644 --- a/docs/en/class-attributes.md +++ b/docs/en/class-attributes.md @@ -163,3 +163,23 @@ class Foo: See [examples/class_var.py](https://github.com/yukinarit/pyserde/blob/main/examples/class_var.py) for complete example. [^1]: [dataclasses.fields](https://docs.python.org/3/library/dataclasses.html#dataclasses.fields) + +### **`deny_unknown_fields`** + +New in v0.22.0, the `deny_unknown_fields` option in the pyserde decorator allows you to enforce strict field validation during deserialization. When this option is enabled, any fields in the input data that are not defined in the target class will cause deserialization to fail with a `SerdeError`. + +Consider the following example: +```python +@serde(deny_unknown_fields=True) +class Foo: + a: int + b: str +``` + +With `deny_unknown_fields=True`, attempting to deserialize data containing fields beyond those defined (a and b in this case) will raise an error. For instance: +``` +from_json(Foo, '{"a": 10, "b": "foo", "c": 100.0, "d": true}') +``` +This will raise a `SerdeError` since fields c and d are not recognized members of Foo. + +See [examples/deny_unknown_fields.py](https://github.com/yukinarit/pyserde/blob/main/examples/deny_unknown_fields.py) for complete example. diff --git a/docs/ja/class-attributes.md b/docs/ja/class-attributes.md index 284afb15..95c30500 100644 --- a/docs/ja/class-attributes.md +++ b/docs/ja/class-attributes.md @@ -178,6 +178,26 @@ class Foo: a: ClassVar[int] = 10 ``` -完全な例については、[examples/class_var.py](https://github.com/yukinarit/pyserde/blob/main/examples/class_var.py) を参照してください。 +完全な例については、[examples/class_var.py](https://github.com/yukinarit/pyserde/blob/main/examples/class_var.py)を参照してください。 [^1]: [dataclasses.fields](https://docs.python.org/3/library/dataclasses.html#dataclasses.fields) + +### **`deny_unknown_fields`** + +バージョン0.22.0で新規追加。 pyserdeデコレータの`deny_unknown_fields`オプションはデシリアライズ時のより厳格なフィールドチェックを制御できます。このオプションをTrueにするとデシリアライズ時に宣言されていないフィールドが見つかると`SerdeError`を投げることができます。 + +以下の例を考えてください。 +```python +@serde(deny_unknown_fields=True) +class Foo: + a: int + b: str +``` + +`deny_unknown_fields=True`が指定されていると、 宣言されているフィールド(この場合aとb)以外がインプットにあると例外を投げます。例えば、 +``` +from_json(Foo, '{"a": 10, "b": "foo", "c": 100.0, "d": true}') +``` +上記のコードはフィールドcとdという宣言されていないフィールドがあるためエラーとなります。 + +完全な例については、[examples/deny_unknown_fields.py](https://github.com/yukinarit/pyserde/blob/main/examples/deny_unknown_fields.py)を参照してください。