Generate jsonschema from pydantic v2

.gitignore

@@ -1,4 +1,6 @@
 
 .DS_Store
 node_modules/
-.venv
+.venv
+env
+__pycache__

README.md

@@ -112,4 +112,9 @@ _Do you use a different IDE which also supports JSON Schema? Please open a PR wi
 ## Contributing 
 PRs that improve these schemata are welcome! 
 
-Please ensure that JSON keys are sorted by [vscode-sort-json](https://marketplace.visualstudio.com/items?itemName=richie5um2.vscode-sort-json) according to the rules in `.vscode/settings`. 
+These schemas are generated from [pydantic models](https://docs.pydantic.dev/latest/concepts/json_schema/). To make updates, the process is as follows:
+
+1. Create a virtual environment and install the dependencies: `pip install -r requirements.txt`
+2. Make changes to the corresponding pydantic models in `src/latest.py`
+3. Run the generation script: `python3 src/generate.py`
+4. Add tests for valid and invalid JSON files in `tests/latest`

generate.py

@@ -0,0 +1,56 @@
+import json
+from pathlib import Path
+from typing import Any, Dict
+
+from src.latest.dbt_cloud import DbtCloud
+from src.latest.dbt_yml_files import DbtYmlFiles
+from src.latest.dependencies import Dependencies, Packages
+from src.latest.selectors import Selectors
+from pydantic.json_schema import GenerateJsonSchema
+
+
+class RemoveNullsGenerateJsonSchema(GenerateJsonSchema):
+    """A GenerateJsonSchema which removes nullability from types.
+
+    We do not want to include optional values in the json schema because
+    that would inhibit code completion and validation.
+
+    Certain properties (such as freshness overrides) need to be nullable,
+    which can be achieved by setting the $comment value below.
+    """
+
+    def _remove_null(self, json_schema: Dict[str, Any]):
+        if "$comment" in json_schema and json_schema["$comment"] == "truly_nullable":
+            return
+        if "anyOf" in json_schema:
+            json_schema["anyOf"] = [
+                item for item in json_schema["anyOf"] if item != {"type": "null"}
+            ]
+        for v in json_schema.values():
+            if isinstance(v, dict):
+                self._remove_null(v)
+
+    def generate(self, schema, mode="validation"):
+        json_schema = super().generate(schema, mode=mode)
+        json_schema["$schema"] = "http://json-schema.org/draft-07/schema#"
+        self._remove_null(json_schema)
+        return json_schema
+
+
+if __name__ == "__main__":
+    files = {
+        "dbt_yml_files": DbtYmlFiles,
+        "dependencies": Dependencies,
+        "packages": Packages,
+        "selectors": Selectors,
+        "dbt_cloud": DbtCloud,
+    }
+    output_directory = Path("schemas/latest")
+    for file_name, model in files.items():
+        schema_file = output_directory / f"{file_name}-latest.json"
+        schema_file.parent.mkdir(parents=True, exist_ok=True)
+        schema = model.model_json_schema(
+            mode="validation", schema_generator=RemoveNullsGenerateJsonSchema
+        )
+        print("Generating", schema_file)
+        schema_file.write_text(json.dumps(schema, indent=2))

requirements.txt

@@ -0,0 +1 @@
+datamodel-code-generator

schemas/latest-originals/dbt_cloud-latest.json

@@ -0,0 +1,14 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "type": "object",
+  "properties": {
+    "project-id": {
+      "type": "string"
+    },
+    "defer-env-id": {
+      "type": "string"
+    }
+  },
+  "required": ["project-id"],
+  "additionalProperties": false
+}