Discriminator support in DRAFT_7?

[thomascobb]

We are using apischema to make a JSON schema for a YAML file, that is then deserialized into a sequence of objects to act as configuration for a program. We edit the YAML file using VSCode YAML editor that has support for Draft 7

We are currently using a Union with literal fields (with some converters to automate). The strategy boils down to this:

import json
from dataclasses import dataclass
from typing import Sequence, Union

from apischema import deserialize, serialize
from apischema.json_schema import JsonSchemaVersion, deserialization_schema
from typing_extensions import Literal

@dataclass
class Cat:
    type: Literal["Cat"] = "Cat"

@dataclass
class Dog:
    type: Literal["Dog"] = "Dog"

Pets = Sequence[Union[Cat, Dog]]
union_schema = deserialization_schema(
    Pets, all_refs=True, version=JsonSchemaVersion.DRAFT_7
)

def test_union():
    with open("pets_union.schema.json", "w") as f:
        f.write(json.dumps(union_schema, indent=2))
    assert deserialize(Pets, [{"type": "Dog"}]) == [Dog()]
    assert serialize(Pets, [Dog()]) == [{"type": "Dog"}]
    assert union_schema == {
        "type": "array",
        "items": {
            "anyOf": [{"$ref": "#/definitions/Cat"}, {"$ref": "#/definitions/Dog"}]
        },
        "definitions": {
            "Cat": {
                "type": "object",
                "properties": {
                    "type": {"type": "string", "const": "Cat", "default": "Cat"}
                },
                "additionalProperties": False,
            },
            "Dog": {
                "type": "object",
                "properties": {
                    "type": {"type": "string", "const": "Dog", "default": "Dog"}
                },
                "additionalProperties": False,
            },
        },
        "$schema": "http://json-schema.org/draft-07/schema#",
    }

I tried to convert this to using discriminators:

import json
from dataclasses import dataclass
from typing import Sequence, Union

from apischema import deserialize, discriminator, serialize
from apischema.json_schema import JsonSchemaVersion, deserialization_schema
from typing_extensions import Annotated

@dataclass
class Cat:
    pass


@dataclass
class Dog:
    pass

Pets = Sequence[Annotated[Union[Cat, Dog], discriminator("type")]]
discriminator_schema = deserialization_schema(
    Pets, all_refs=True, version=JsonSchemaVersion.DRAFT_7
)
def test_discriminator():
    with open("pets_discriminator.schema.json", "w") as f:
        f.write(json.dumps(discriminator_schema, indent=2))
    assert deserialize(Pets, [{"type": "Dog"}]) == [Dog()]
    assert serialize(Pets, [Dog()]) == [{"type": "Dog"}]
    assert discriminator_schema == {
        "type": "array",
        "items": {
            "oneOf": [{"$ref": "#/definitions/Cat"}, {"$ref": "#/definitions/Dog"}],
            "discriminator": {"propertyName": "type"},
        },
        "definitions": {
            "Cat": {"type": "object", "additionalProperties": False},
            "Dog": {"type": "object", "additionalProperties": False},
        },
        "$schema": "http://json-schema.org/draft-07/schema#",
    }

Unfortunately this doesn't play nicely with the editor, as Draft 7 doesn't support discriminator, it is left with a union of two objects with no "type" field.

I was wondering what would be the best way to produce the schema from the union example from a nested type structure with discriminators? Is it something that could be done in deserialization_schema, or in some custom converters, or should I process the (nested) class structure before passing to deserialization_schema?

Happy to make a PR to apischema if that is the best place for code that does this.

Thanks,
Tom

[wyfo]

Hopefully, maybe there is no need to make a PR. Or just a little one to update the documentation.
In fact, I realize that I've forget to document an important discriminator feature: you can indeed have the discriminator attribute in your class and that will still work. And it will even override the mapping value if the field has a literal type.

Let me update your example:

import json
from dataclasses import dataclass
from typing import Literal, Sequence, Union

from apischema import deserialize, discriminator, serialize
from apischema.json_schema import JsonSchemaVersion, deserialization_schema
from typing_extensions import Annotated


@dataclass
class Cat:
    type: Literal["cat"] = "cat"
    pass


@dataclass
class Dog:
    type: Literal["dog"] = "dog"


Pets = Sequence[Annotated[Union[Cat, Dog], discriminator("type")]]
discriminator_schema = deserialization_schema(
    Pets, all_refs=True, version=JsonSchemaVersion.DRAFT_7
)
assert deserialize(Pets, [{"type": "dog"}]) == [Dog()]
assert serialize(Pets, [Dog()]) == [{"type": "dog"}]
assert discriminator_schema == {
    "type": "array",
    "items": {
        "oneOf": [{"$ref": "#/definitions/Cat"}, {"$ref": "#/definitions/Dog"}],
        "discriminator": {
            "propertyName": "type",
            "mapping": {"cat": "#/definitions/Cat", "dog": "#/definitions/Dog"},
        },
    },
    "definitions": {
        "Cat": {
            "type": "object",
            "properties": {
                "type": {"type": "string", "const": "cat", "default": "cat"}
            },
            "additionalProperties": False,
        },
        "Dog": {
            "type": "object",
            "properties": {
                "type": {"type": "string", "const": "dog", "default": "dog"}
            },
            "additionalProperties": False,
        },
    },
    "$schema": "http://json-schema.org/draft-07/schema#",
}

When the discriminator field is not declared, I've judged that it should not be put in the schema, as the type could be used outside of a discriminated union, so one should not expect an undeclared field to be here. Actually, I'm not sure that it's really compatible with OpenAPI standard (to have a discriminator not present in fields).

Maybe I could do something like this :

{
    "type": "array",
    "items": {
        "oneOf": [
            {
                "allOf": [
                    {"$ref": "#/definitions/Cat"},
                    {"properties": {"type": {"type": "string"}}},
                ]
            },
            {
                "allOf": [
                    {"$ref": "#/definitions/Dog"},
                    {"properties": {"type": {"type": "string"}}},
                ]
            },
        ],
        "discriminator": {"propertyName": "type"},
    },
    "definitions": {
        "Cat": {"type": "object", "additionalProperties": False},
        "Dog": {"type": "object", "additionalProperties": False},
    },
    "$schema": "http://json-schema.org/draft-07/schema#",
}

but there would be the issue of "additionalProperties": true, and draft7 doesn't support unevaluatedProperties.
JSON schema is a big mess because of additionalProperties and its handling in allOf, and even unevaluatedProperties doesn't fix completely the issue. That makes the JSON schema generation quite complex, and sometime quite impossible, especially with old drafts.

If you encounter some issues, you can still use schema overriding to patch them, but again, additionalProperties/unevaluatedProperties/allOf make me crazy.

For now, I hope this trick about discriminator will do the job for you, I will have to update the documentation to include it.

[thomascobb]

Thanks, that example seems to work with the editor, although I had to change oneOf to anyOf so it recognized the union.

Would you mind having a look at my attempt at adding the literal fields into the schema without putting them in the dataclass? Not sure if there's a better way to do this...

from dataclasses import dataclass, field, make_dataclass
from functools import lru_cache
from typing import List, Sequence, TypeVar, Union

from apischema import (
    deserialize,
    deserializer,
    discriminator,
    order,
    serialize,
    serialized,
)
from apischema.conversions import Conversion
from apischema.conversions.converters import serializer
from apischema.json_schema import JsonSchemaVersion, deserialization_schema
from apischema.utils import identity
from typing_extensions import Annotated, Literal

Cls = TypeVar("Cls", bound=type)


# Permanently cache so we don't include deserialization subclasses defined below
@lru_cache(maxsize=None)
def rec_subclasses(cls: Cls) -> List[Cls]:
    """Recursive implementation of type.__subclasses__"""
    subclasses = []
    for sub_cls in cls.__subclasses__():
        subclasses += [sub_cls] + rec_subclasses(sub_cls)
    return subclasses


def as_discriminated_union(cls: Cls) -> Cls:
    def deserialization() -> Conversion:
        def typed_subs():
            for sub in rec_subclasses(cls):
                # Make typed_sub derived from sub with additional type entry
                type_field = (
                    "type",
                    Literal[sub.__name__],  # type: ignore
                    field(default=sub.__name__, metadata=order(-1)),
                )
                yield make_dataclass(sub.__name__, [type_field], bases=(sub,))

        def convert(sub_inst):
            """Make instance of sub from instance of typed_sub"""
            d = {k: v for k, v in sub_inst.__dict__.items() if k != "type"}
            return type(sub_inst).__bases__[0](**d)

        return Conversion(
            convert,
            # source=Union[tuple(typed_subs())],  # type: ignore
            source=Annotated[Union[tuple(typed_subs())], discriminator("type")],  # type: ignore
            target=cls,
        )

    def serialization() -> Conversion:
        @serialized("type", owner=cls, order=order(-1))
        def add_type(obj) -> str:
            """Add type entry to serialized dictionary"""
            return type(obj).__name__

        return Conversion(
            identity,
            source=cls,
            target=Union[tuple(rec_subclasses(cls))],  # type: ignore
            inherited=False,
        )

    serializer(lazy=serialization, source=cls)
    deserializer(lazy=deserialization, target=cls)
    return cls


@as_discriminated_union
class Pet:
    pass


@dataclass
class Cat(Pet):
    pass


@dataclass
class Dog(Pet):
    pass


Pets = Sequence[Pet]
discriminator_schema = deserialization_schema(
    Pets, all_refs=True, version=JsonSchemaVersion.DRAFT_7
)


def test_pets():
    assert deserialize(Pets, [{"type": "Dog"}]) == [Dog()]
    assert serialize(Pets, [Dog()]) == [{"type": "Dog"}]
    assert discriminator_schema == {
        "$schema": "http://json-schema.org/draft-07/schema#",
        "type": "array",
        "items": {"$ref": "#/definitions/Pet"},
        "definitions": {
            "Cat": {
                "additionalProperties": False,
                "properties": {
                    "type": {"const": "Cat", "default": "Cat", "type": "string"}
                },
                "type": "object",
            },
            "Dog": {
                "additionalProperties": False,
                "properties": {
                    "type": {"const": "Dog", "default": "Dog", "type": "string"}
                },
                "type": "object",
            },
            "Pet": {
                "discriminator": {"propertyName": "type"},
                "oneOf": [{"$ref": "#/definitions/Cat"}, {"$ref": "#/definitions/Dog"}],
            },
        },
    }