Extend json-schema model parser

Summary

For https://gitlab.indiscale.com/caosdb/customers/3dmmto/management/-/issues/9.

This implements

• pattern properties
• type null properties
• json schema references
• and json schemas without top-level recordtype specification

for the json schema datamodel parser.

Focus

Mainly additions to the JsonSchemaParser class. We also defined a treatment for type: array properties that lack a strict items specification (i.e., could be lists of elements with mixed types, so a manual specification has to be provided -- or they are ignored altogether).

Test Environment

New unit tests. You can also test it with the chemotion json schema in nextcloud/Customers/3dmmto/example_files/testdaten_chemotion/schema.json.

Check List for the Author

Please, prepare your MR for a review. Be sure to write a summary and a focus and create gitlab comments for the reviewer. They should guide the reviewer through the changes, explain your changes and also point out open questions. For further good practices have a look at our review guidelines

• All automated tests pass
• Reference related issues
• Up-to-date CHANGELOG.md (or not necessary)
• Up-to-date JSON schema (or not necessary)
• Appropriate user and developer documentation (or not necessary)
  • How do I use the software? Assume "stupid" users.
  • How do I develop or debug the software? Assume novice developers.
• Annotations in code (Gitlab comments)
  • Intent of new code
  • Problems with old code
  • Why this implementation?

Check List for the Reviewer

• I understand the intent of this MR
• All automated tests pass
• Up-to-date CHANGELOG.md (or not necessary)
• Appropriate user and developer documentation (or not necessary)
• The test environment setup works and the intended behavior is reproducible in the test environment
• In-code documentation and comments are up-to-date.
• Check: Are there specifications? Are they satisfied?

For further good practices have a look at our review guidelines.

src/caosadvancedtools/models/parser.py

@@ -192,7 +192,7 @@ def parse_model_from_json_schema(
     """
     # @author Florian Spreckelsen
     # @date 2022-02-17
-    # @review Daniel Hornung 2022-02-18
+    # @review Timm Fitschen 2023-05-25
     parser = JsonSchemaParser(types_for_missing_array_items, ignore_unspecified_array_items)
 
     return parser.parse_model_from_json_schema(filename, top_level_recordtype)
@@ -600,7+600,7 @@
                             dtype))
                     else:
                         value.datatype = db.__getattribute__(  # pylint: disable=no-member
                             dtype)
 
                     continue
 
                 raise ValueError("Property {} has an unknown datatype: {}".format(
                     value.name, value.datatype))
 
     def _set_recordtypes(self):
         """ properties are defined in first iteration; set remaining as RTs """
 
         for key, value in self.model.items():
@@ -639,7+639,7 @@
     """
     # @author Florian Spreckelsen
     # @date 2022-02-17
-    # @review Timm Fitschen 2022-02-30
+    # @review Timm Fitschen 2023-05-25
 
     def __init__(self, types_for_missing_array_items={}, ignore_unspecified_array_items=False):
         super().__init__()
@@ -660,5+660,5 @@
 
         Returns
         -------
         out : DataModel
             The created DataModel
         """
         # @author Florian Spreckelsen
         # @date 2022-02-17
-        # @review Timm Fitschen 2022-02-30
+        # @review Timm Fitschen 2023-05-25
         with open(filename, 'r') as schema_file:
             model_dict = jsonref.load(schema_file)
 
@@ -699,7+699,7 @@
             except jsonschema.SchemaError as err:
                 key = elt["title"] if "title" in elt else f"element {ii}"
                 raise JsonSchemaDefinitionError(
                     f"Json Schema error in {key}:\n{str(err)}") from err
 
             if top_level_recordtype:
                 if "title" not in elt:
@@ -711,7+711,7 @@
                 # Check if this is a valid Json Schema
                 name = self._stringify(elt["title"], context=elt)
                 self._treat_element(elt, name)
-            elif "properties" in elt or "patternProperties":
+            elif "properties" in elt or "patternProperties" in elt:
                 # No top-level type but there are entities
                 if "properties" in elt:
                     for key, prop in elt["properties"].items():
@@ -732,7 +732,7 @@ class JsonSchemaParser(Parser):
         return DataModel(self.model.values())
 
     def _get_name_from_property(self, key: str, prop: dict):
-
+        # @review Timm Fitschen 2023-05-25
         if "title" in prop:
             name = self._stringify(prop["title"])
         else:
             name = self._stringify(key)
 
         return name
 
     def _get_atomic_datatype(self, elt):
-        # @review Timm Fitschen 2022-02-30
+        # @review Timm Fitschen 2023-05-25
         if elt["type"] == "string":
             if "format" in elt and elt["format"] in ["date", "date-time"]:
                 return db.DATETIME
@@ -792,9 +792,6 @@ class JsonSchemaParser(Parser):
             ent = self._treat_record_type(elt, name)
         elif elt["type"] == "array":
             ent, force_list = self._treat_list(elt, name)
-        elif elt["type"] == "null":
-            # null
-            return None, force_list
         else:
             raise NotImplementedError(
                 f"Cannot parse items of type '{elt['type']}' (yet).")
@@ -862,7 +859,7 @@ class JsonSchemaParser(Parser):
         return rt
 
     def _treat_list(self, elt: dict, name: str):
-        # @review Timm Fitschen 2022-02-30
+        # @review Timm Fitschen 2023-05-25
 
         if "items" not in elt and name not in self.types_for_missing_array_items:
             if self.ignore_unspecified_array_items:
@@ -893,6 +890,7 @@ class JsonSchemaParser(Parser):
             return db.Property(name=name, datatype=datatype), False
 
     def _get_pattern_prop(self):
+        # @review Timm Fitschen 2023-05-25
         if "__pattern_property_pattern_property" in self.model:
             return self.model["__pattern_property_pattern_property"]
         pp = db.Property(name="__matched_pattern", datatype=db.TEXT)
@@ -913,11 +911,12 @@ class JsonSchemaParser(Parser):
             array.
 
         """
+        # @review Timm Fitschen 2023-05-25
         num_patterns = len(pattern_elements)
         pattern_prop = self._get_pattern_prop()
         returns = []
         for ii, (key, element) in enumerate(pattern_elements.items()):
-            if not "title" in element:
+            if "title" not in element:
                 name_suffix = f"_{ii+1}" if num_patterns > 1 else ""
                 name = name_prefix + "Entry" + name_suffix
             else: