From b6c55dcbde2c106989414935ddbf7f0960cb88f2 Mon Sep 17 00:00:00 2001 From: Daniel Bluhm Date: Thu, 4 Aug 2022 12:19:50 -0400 Subject: [PATCH 1/4] fix: schema class can set Meta.unknown Signed-off-by: Daniel Bluhm --- aries_cloudagent/messaging/models/base.py | 119 ++++++++++++++++++---- aries_cloudagent/utils/classloader.py | 7 +- 2 files changed, 102 insertions(+), 24 deletions(-) diff --git a/aries_cloudagent/messaging/models/base.py b/aries_cloudagent/messaging/models/base.py index fd00c7d68d..9ce1f98485 100644 --- a/aries_cloudagent/messaging/models/base.py +++ b/aries_cloudagent/messaging/models/base.py @@ -5,7 +5,7 @@ from abc import ABC from collections import namedtuple -from typing import Mapping, Union +from typing import Literal, Mapping, Optional, Type, TypeVar, Union, cast, overload from marshmallow import Schema, post_dump, pre_load, post_load, ValidationError, EXCLUDE @@ -17,7 +17,7 @@ SerDe = namedtuple("SerDe", "ser de") -def resolve_class(the_cls, relative_cls: type = None): +def resolve_class(the_cls, relative_cls: Optional[type] = None) -> type: """ Resolve a class. @@ -38,6 +38,10 @@ def resolve_class(the_cls, relative_cls: type = None): elif isinstance(the_cls, str): default_module = relative_cls and relative_cls.__module__ resolved = ClassLoader.load_class(the_cls, default_module) + else: + raise TypeError( + f"Could not resolve class from {the_cls}; incorrect type {type(the_cls)}" + ) return resolved @@ -70,6 +74,9 @@ class BaseModelError(BaseError): """Base exception class for base model errors.""" +ModelType = TypeVar("ModelType", bound="BaseModel") + + class BaseModel(ABC): """Base model that provides convenience methods.""" @@ -94,7 +101,7 @@ def __init__(self): ) @classmethod - def _get_schema_class(cls): + def _get_schema_class(cls) -> Type["BaseModelSchema"]: """ Get the schema class. @@ -102,10 +109,16 @@ def _get_schema_class(cls): The resolved schema class """ - return resolve_class(cls.Meta.schema_class, cls) + resolved = resolve_class(cls.Meta.schema_class, cls) + if issubclass(resolved, BaseModelSchema): + return resolved + + raise TypeError( + f"Resolved class is not a subclass of BaseModelSchema: {resolved}" + ) @property - def Schema(self) -> type: + def Schema(self) -> Type["BaseModelSchema"]: """ Accessor for the model's schema class. @@ -115,8 +128,46 @@ def Schema(self) -> type: """ return self._get_schema_class() + @overload @classmethod - def deserialize(cls, obj, unknown: str = None, none2none: str = False): + def deserialize( + cls: Type[ModelType], + obj, + *, + unknown: Optional[str] = None, + ) -> ModelType: + ... + + @overload + @classmethod + def deserialize( + cls: Type[ModelType], + obj, + *, + none2none: Literal[False], + unknown: Optional[str] = None, + ) -> ModelType: + ... + + @overload + @classmethod + def deserialize( + cls: Type[ModelType], + obj, + *, + none2none: Literal[True], + unknown: Optional[str] = None, + ) -> Optional[ModelType]: + ... + + @classmethod + def deserialize( + cls: Type[ModelType], + obj, + *, + unknown: Optional[str] = None, + none2none: bool = False, + ) -> Optional[ModelType]: """ Convert from JSON representation to a model instance. @@ -132,18 +183,41 @@ def deserialize(cls, obj, unknown: str = None, none2none: str = False): if obj is None and none2none: return None - schema = cls._get_schema_class()(unknown=unknown or EXCLUDE) + schema_cls = cls._get_schema_class() + schema = schema_cls(unknown=unknown or schema_cls.Meta.unknown) + try: - return schema.loads(obj) if isinstance(obj, str) else schema.load(obj) + return cast( + ModelType, + schema.loads(obj) if isinstance(obj, str) else schema.load(obj), + ) except (AttributeError, ValidationError) as err: LOGGER.exception(f"{cls.__name__} message validation error:") raise BaseModelError(f"{cls.__name__} schema validation failed") from err + @overload def serialize( self, - as_string=False, - unknown: str = None, + *, + as_string: Literal[True], + unknown: Optional[str] = None, + ) -> str: + ... + + @overload + def serialize( + self, + *, + unknown: Optional[str] = None, ) -> dict: + ... + + def serialize( + self, + *, + as_string: bool = False, + unknown: Optional[str] = None, + ) -> Union[str, dict]: """ Create a JSON-compatible dict representation of the model instance. @@ -154,7 +228,8 @@ def serialize( A dict representation of this model, or a JSON string if as_string is True """ - schema = self.Schema(unknown=unknown or EXCLUDE) + schema_cls = self._get_schema_class() + schema = schema_cls(unknown=unknown or schema_cls.Meta.unknown) try: return ( schema.dumps(self, separators=(",", ":")) @@ -168,18 +243,17 @@ def serialize( ) from err @classmethod - def serde(cls, obj: Union["BaseModel", Mapping]) -> SerDe: + def serde(cls, obj: Union["BaseModel", Mapping]) -> Optional[SerDe]: """Return serialized, deserialized representations of input object.""" + if obj is None: + return None - return ( - SerDe(obj.serialize(), obj) - if isinstance(obj, BaseModel) - else None - if obj is None - else SerDe(obj, cls.deserialize(obj)) - ) + if isinstance(obj, BaseModel): + return SerDe(obj.serialize(), obj) + + return SerDe(obj, cls.deserialize(obj)) - def validate(self, unknown: str = None): + def validate(self, unknown: Optional[str] = None): """Validate a constructed model.""" schema = self.Schema(unknown=unknown) errors = schema.validate(self.serialize()) @@ -191,7 +265,7 @@ def validate(self, unknown: str = None): def from_json( cls, json_repr: Union[str, bytes], - unknown: str = None, + unknown: Optional[str] = None, ): """ Parse a JSON string into a model instance. @@ -218,7 +292,7 @@ def to_json(self, unknown: str = None) -> str: A JSON representation of this message """ - return json.dumps(self.serialize(unknown=unknown or EXCLUDE)) + return json.dumps(self.serialize(unknown=unknown)) def __repr__(self) -> str: """ @@ -246,6 +320,7 @@ class Meta: model_class = None skip_values = [None] ordered = True + unknown = EXCLUDE def __init__(self, *args, **kwargs): """ diff --git a/aries_cloudagent/utils/classloader.py b/aries_cloudagent/utils/classloader.py index 7c429b0ea9..2b4de2a207 100644 --- a/aries_cloudagent/utils/classloader.py +++ b/aries_cloudagent/utils/classloader.py @@ -7,7 +7,7 @@ from importlib import import_module from importlib.util import find_spec, resolve_name from types import ModuleType -from typing import Sequence, Type +from typing import Optional, Sequence, Type from ..core.error import BaseError @@ -75,7 +75,10 @@ def load_module(cls, mod_path: str, package: str = None) -> ModuleType: @classmethod def load_class( - cls, class_name: str, default_module: str = None, package: str = None + cls, + class_name: str, + default_module: Optional[str] = None, + package: Optional[str] = None, ): """ Resolve a complete class path (ie. typing.Dict) to the class itself. From 0de5904c356d05a8f7ffb4ee0d14859dd552f1b7 Mon Sep 17 00:00:00 2001 From: Daniel Bluhm Date: Thu, 4 Aug 2022 12:53:18 -0400 Subject: [PATCH 2/4] test: schema meta unknown respected Signed-off-by: Daniel Bluhm --- aries_cloudagent/messaging/models/base.py | 17 +++-- .../messaging/models/tests/test_base.py | 74 ++++++++++++++++--- 2 files changed, 76 insertions(+), 15 deletions(-) diff --git a/aries_cloudagent/messaging/models/base.py b/aries_cloudagent/messaging/models/base.py index 9ce1f98485..77f1c8ce40 100644 --- a/aries_cloudagent/messaging/models/base.py +++ b/aries_cloudagent/messaging/models/base.py @@ -5,7 +5,8 @@ from abc import ABC from collections import namedtuple -from typing import Literal, Mapping, Optional, Type, TypeVar, Union, cast, overload +from typing import Mapping, Optional, Type, TypeVar, Union, cast, overload +from typing_extensions import Literal from marshmallow import Schema, post_dump, pre_load, post_load, ValidationError, EXCLUDE @@ -57,7 +58,10 @@ def resolve_meta_property(obj, prop_name: str, defval=None): The meta property """ - cls = obj.__class__ + if isinstance(obj, type): + cls = obj + else: + cls = obj.__class__ found = defval while cls: Meta = getattr(cls, "Meta", None) @@ -184,7 +188,9 @@ def deserialize( return None schema_cls = cls._get_schema_class() - schema = schema_cls(unknown=unknown or schema_cls.Meta.unknown) + schema = schema_cls( + unknown=unknown or resolve_meta_property(schema_cls, "unknown", EXCLUDE) + ) try: return cast( @@ -229,7 +235,9 @@ def serialize( """ schema_cls = self._get_schema_class() - schema = schema_cls(unknown=unknown or schema_cls.Meta.unknown) + schema = schema_cls( + unknown=unknown or resolve_meta_property(schema_cls, "unknown", EXCLUDE) + ) try: return ( schema.dumps(self, separators=(",", ":")) @@ -320,7 +328,6 @@ class Meta: model_class = None skip_values = [None] ordered = True - unknown = EXCLUDE def __init__(self, *args, **kwargs): """ diff --git a/aries_cloudagent/messaging/models/tests/test_base.py b/aries_cloudagent/messaging/models/tests/test_base.py index 62c327b7a1..9cc6eaf868 100644 --- a/aries_cloudagent/messaging/models/tests/test_base.py +++ b/aries_cloudagent/messaging/models/tests/test_base.py @@ -1,15 +1,6 @@ -import json - from asynctest import TestCase as AsyncTestCase, mock as async_mock -from marshmallow import EXCLUDE, fields, validates_schema, ValidationError - -from ....cache.base import BaseCache -from ....config.injection_context import InjectionContext -from ....storage.base import BaseStorage, StorageRecord - -from ...responder import BaseResponder, MockResponder -from ...util import time_now +from marshmallow import EXCLUDE, INCLUDE, fields, validates_schema, ValidationError from ..base import BaseModel, BaseModelError, BaseModelSchema @@ -35,6 +26,48 @@ def validate_fields(self, data, **kwargs): raise ValidationError("") +class ModelImplWithUnknown(BaseModel): + class Meta: + schema_class = "SchemaImplWithUnknown" + + def __init__(self, *, attr=None, **kwargs): + self.attr = attr + self.extra = kwargs + + +class SchemaImplWithUnknown(BaseModelSchema): + class Meta: + model_class = ModelImplWithUnknown + unknown = INCLUDE + + attr = fields.String(required=True) + + @validates_schema + def validate_fields(self, data, **kwargs): + if data["attr"] != "succeeds": + raise ValidationError("") + + +class ModelImplWithoutUnknown(BaseModel): + class Meta: + schema_class = "SchemaImplWithoutUnknown" + + def __init__(self, *, attr=None): + self.attr = attr + + +class SchemaImplWithoutUnknown(BaseModelSchema): + class Meta: + model_class = ModelImplWithoutUnknown + + attr = fields.String(required=True) + + @validates_schema + def validate_fields(self, data, **kwargs): + if data["attr"] != "succeeds": + raise ValidationError("") + + class TestBase(AsyncTestCase): def test_model_validate_fails(self): model = ModelImpl(attr="string") @@ -63,3 +96,24 @@ def test_from_json_x(self): data = "{}{}" with self.assertRaises(BaseModelError): ModelImpl.from_json(data) + + def test_model_with_unknown(self): + model = ModelImplWithUnknown(attr="succeeds") + model = model.validate() + assert model.attr == "succeeds" + + model = ModelImplWithUnknown.deserialize( + {"attr": "succeeds", "another": "value"} + ) + assert model.extra + assert model.extra["another"] == "value" + assert model.attr == "succeeds" + + def test_model_without_unknown_default_exclude(self): + model = ModelImplWithoutUnknown(attr="succeeds") + model = model.validate() + assert model.attr == "succeeds" + + assert ModelImplWithoutUnknown.deserialize( + {"attr": "succeeds", "another": "value"} + ) From 87a7c205e76dd078789e2a60631c8d9f0eef5856 Mon Sep 17 00:00:00 2001 From: Daniel Bluhm Date: Thu, 4 Aug 2022 21:15:40 -0400 Subject: [PATCH 3/4] style: appease flake8 docstring requirements Signed-off-by: Daniel Bluhm --- aries_cloudagent/messaging/models/base.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/aries_cloudagent/messaging/models/base.py b/aries_cloudagent/messaging/models/base.py index 77f1c8ce40..6a85e9f993 100644 --- a/aries_cloudagent/messaging/models/base.py +++ b/aries_cloudagent/messaging/models/base.py @@ -140,6 +140,7 @@ def deserialize( *, unknown: Optional[str] = None, ) -> ModelType: + """Convert from JSON representation to a model instance.""" ... @overload @@ -151,6 +152,7 @@ def deserialize( none2none: Literal[False], unknown: Optional[str] = None, ) -> ModelType: + """Convert from JSON representation to a model instance.""" ... @overload @@ -162,6 +164,7 @@ def deserialize( none2none: Literal[True], unknown: Optional[str] = None, ) -> Optional[ModelType]: + """Convert from JSON representation to a model instance.""" ... @classmethod @@ -208,6 +211,7 @@ def serialize( as_string: Literal[True], unknown: Optional[str] = None, ) -> str: + """Create a JSON-compatible dict representation of the model instance.""" ... @overload @@ -216,6 +220,7 @@ def serialize( *, unknown: Optional[str] = None, ) -> dict: + """Create a JSON-compatible dict representation of the model instance.""" ... def serialize( From be53f255847c9470899ae027a3da979c47882a3e Mon Sep 17 00:00:00 2001 From: Moriarty Date: Tue, 18 Oct 2022 15:28:07 +0200 Subject: [PATCH 4/4] feat: update pynacl version from 1.4.0 to 1.50 this fixes install for m1 mac arm64 arch Signed-off-by: Moriarty --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 70bc72e54e..fe33a1021c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,7 +13,7 @@ markupsafe==2.0.1 marshmallow==3.5.1 msgpack~=1.0 prompt_toolkit~=2.0.9 -pynacl~=1.4.0 +pynacl~=1.5.0 requests~=2.25.0 packaging~=20.4 pyld~=2.0.3