Extend json-schema model parser

Summary

For https://gitlab.indiscale.com/caosdb/customers/3dmmto/management/-/issues/9.

This implements

• pattern properties
• type null properties
• json schema references
• and json schemas without top-level recordtype specification

for the json schema datamodel parser.

Focus

Mainly additions to the JsonSchemaParser class. We also defined a treatment for type: array properties that lack a strict items specification (i.e., could be lists of elements with mixed types, so a manual specification has to be provided -- or they are ignored altogether).

Test Environment

New unit tests. You can also test it with the chemotion json schema in nextcloud/Customers/3dmmto/example_files/testdaten_chemotion/schema.json.

Check List for the Author

Please, prepare your MR for a review. Be sure to write a summary and a focus and create gitlab comments for the reviewer. They should guide the reviewer through the changes, explain your changes and also point out open questions. For further good practices have a look at our review guidelines

• All automated tests pass
• Reference related issues
• Up-to-date CHANGELOG.md (or not necessary)
• Up-to-date JSON schema (or not necessary)
• Appropriate user and developer documentation (or not necessary)
  • How do I use the software? Assume "stupid" users.
  • How do I develop or debug the software? Assume novice developers.
• Annotations in code (Gitlab comments)
  • Intent of new code
  • Problems with old code
  • Why this implementation?

Check List for the Reviewer

• I understand the intent of this MR
• All automated tests pass
• Up-to-date CHANGELOG.md (or not necessary)
• Appropriate user and developer documentation (or not necessary)
• The test environment setup works and the intended behavior is reproducible in the test environment
• In-code documentation and comments are up-to-date.
• Check: Are there specifications? Are they satisfied?

For further good practices have a look at our review guidelines.

src/caosadvancedtools/models/parser.py

@@ -35,8 +35,9 @@ not defined, simply the name can be supplied with no value.
 Parents can be provided under the 'inherit_from_xxxx' keywords. The value needs
 to be a list with the names. Here, NO NEW entities can be defined.
 """
-import json
 import argparse
+import json
+import jsonref
 import re
 import sys
 import yaml
@@ -152,13 +153,17 @@ def parse_model_from_string(string):
     return parser.parse_model_from_string(string)
 
 
-def parse_model_from_json_schema(filename: str):
+def parse_model_from_json_schema(filename: str, top_level_recordtype: bool = True):
     """Return a datamodel parsed from a json schema definition.
 
     Parameters
     ----------
     filename : str
         The path of the json schema file that is to be parsed
+    top_level_recordtype : bool, optional
+        Whether there is a record type defined at the top level of the
+        schema. Default is true.
+
 
     Returns
     -------
@@ -177,7 +182,7 @@ def parse_model_from_json_schema(filename: str):
     # @review Daniel Hornung 2022-02-18
     parser = JsonSchemaParser()
 
-    return parser.parse_model_from_json_schema(filename)
+    return parser.parse_model_from_json_schema(filename, top_level_recordtype)
 
 
 class Parser(object):
@@ -623,7 +628,7 @@ class JsonSchemaParser(Parser):
     # @date 2022-02-17
     # @review Timm Fitschen 2022-02-30
 
-    def parse_model_from_json_schema(self, filename: str):
+    def parse_model_from_json_schema(self, filename: str, top_level_recordtype: bool = True):
         """Return a datamodel created from the definition in the json schema in
         `filename`.
 
@@ -631,6 +636,9 @@ class JsonSchemaParser(Parser):
         ----------
         filename : str
             The path to the json-schema file containing the datamodel definition
+        top_level_recordtype : bool, optional
+            Whether there is a record type defined at the top level of the
+            schema. Default is true.
 
         Returns
         -------
@@ -641,11 +649,11 @@ class JsonSchemaParser(Parser):
         # @date 2022-02-17
         # @review Timm Fitschen 2022-02-30
         with open(filename, 'r') as schema_file:
-            model_dict = json.load(schema_file)
+            model_dict = jsonref.load(schema_file)
 
-        return self._create_model_from_dict(model_dict)
+        return self._create_model_from_dict(model_dict, top_level_recordtype=top_level_recordtype)
 
-    def _create_model_from_dict(self, model_dict: [dict, List[dict]]):
+    def _create_model_from_dict(self, model_dict: [dict, List[dict]], top_level_recordtype: bool = True):
         """Parse a dictionary and return the Datamodel created from it.
 
         The dictionary was typically created from the model definition in a json schema file.
@@ -654,6 +662,9 @@ class JsonSchemaParser(Parser):
         ----------
         model_dict : dict or list[dict]
             One or several dictionaries read in from a json-schema file
+        top_level_recordtype : bool, optional
+            Whether there is a record type defined at the top level of the
+            schema. Default is true.
 
         Returns
         -------
@@ -665,23 +676,43 @@ class JsonSchemaParser(Parser):
             model_dict = [model_dict]
 
         for ii, elt in enumerate(model_dict):
-            if "title" not in elt:
-                raise JsonSchemaDefinitionError(
-                    f"Object {ii+1} is lacking the `title` key word")
-            if "type" not in elt:
-                raise JsonSchemaDefinitionError(
-                    f"Object {ii+1} is lacking the `type` key word")
-            # Check if this is a valid Json Schema
             try:
                 jsonschema.Draft202012Validator.check_schema(elt)
             except jsonschema.SchemaError as err:
+                key = elt["title"] if "title" in elt else f"element {ii}"
                 raise JsonSchemaDefinitionError(
-                    f"Json Schema error in {elt['title']}:\n{str(err)}") from err
-            name = self._stringify(elt["title"], context=elt)
-            self._treat_element(elt, name)
+                    f"Json Schema error in {key}:\n{str(err)}") from err
+
+            if top_level_recordtype:
+                if "title" not in elt:
+                    raise JsonSchemaDefinitionError(
+                        f"Object {ii+1} is lacking the `title` key word")
+                if "type" not in elt:
+                    raise JsonSchemaDefinitionError(
+                        f"Object {ii+1} is lacking the `type` key word")
+                # Check if this is a valid Json Schema
+                name = self._stringify(elt["title"], context=elt)
+                self._treat_element(elt, name)
+            elif "properties" in elt:
+                for key, prop in elt["properties"].items():
+                    name = self._get_name_from_property(key, prop)
+                    self._treat_element(prop, name)
+            else:
+                # Neither RecordType itself, nor further properties in schema,
+                # so nothing to do here. Maybe add something in the future.
+                continue
 
         return DataModel(self.model.values())
 
+    def _get_name_from_property(self, key: str, prop: dict):
+
+        if "title" in prop:
+            name = self._stringify(prop["title"])
+        else:
+            name = self._stringify(key)
+
+        return name
+
     def _get_atomic_datatype(self, elt):
         # @review Timm Fitschen 2022-02-30
         if elt["type"] == "string":
@@ -716,6 +747,11 @@ class JsonSchemaParser(Parser):
                     "be identified with CaosDB's name property."
                 )
             return None, force_list
+        # LinkAhead suports null for all types, so in the very special case of
+        # `"type": ["null", "<other_type>"]`, only consider the other type:
+        if isinstance(elt["type"], list) and len(elt["type"]) == 2 and "null" in elt["type"]:
+            elt["type"].remove("null")
+            elt["type"] = elt["type"][0]
         if "enum" in elt:
             ent = self._treat_enum(elt, name)
         elif elt["type"] in JSON_SCHEMA_ATOMIC_TYPES:
@@ -745,10 +781,7 @@ class JsonSchemaParser(Parser):
             required = []
         if "properties" in elt:
             for key, prop in elt["properties"].items():
-                if "title" in prop:
-                    name = self._stringify(prop["title"])
-                else:
-                    name = self._stringify(key)
+                name = self._get_name_from_property(key, prop)
                 prop_ent, force_list = self._treat_element(prop, name)
                 if prop_ent is None:
                     # Nothing to be appended since the property has to be