F schema export references

Summary

For https://gitlab.indiscale.com/caosdb/customers/dimr/management/-/issues/38 Extends !80 (merged) by treatment of references and list of references.

(List of) References are translated int (arrays of) "oneOf" properties: They include a subschema for the creation of new Records of the referenced types, and an enum with the existing Records. A special case are reference properties with the generic db.REFERENCE datatype. Since here, we can't create a new Record (since we don't know a type), only the enum with all Records and Files is used.

Focus

RecordType treatment has been refactored s.th. it can be used recursively to create subschemas. Apart from this and the generation of the enum values, the existing property treatment was already enough.

Test Environment

Unit and integration tests should be sufficient but feel free to test with your favorite RT from a live system, too.

Check List for the Author

Please, prepare your MR for a review. Be sure to write a summary and a focus and create gitlab comments for the reviewer. They should guide the reviewer through the changes, explain your changes and also point out open questions. For further good practices have a look at our review guidelines

• All automated tests pass
• Reference related issues
• Up-to-date CHANGELOG.md (or not necessary)
• Up-to-date JSON schema (or not necessary)
• Appropriate user and developer documentation (or not necessary)
  • How do I use the software? Assume "stupid" users.
  • How do I develop or debug the software? Assume novice developers.
• Annotations in code (Gitlab comments)
  • Intent of new code
  • Problems with old code
  • Why this implementation?

Check List for the Reviewer

• I understand the intent of this MR
• All automated tests pass
• Up-to-date CHANGELOG.md (or not necessary)
• Appropriate user and developer documentation (or not necessary)
• The test environment setup works and the intended behavior is reproducible in the test environment
• In-code documentation and comments are up-to-date.
• Check: Are there specifications? Are they satisfied?

For further good practices have a look at our review guidelines.

unittests/test_json_schema_exporter.py

@@ -24,10 +24,50 @@ import linkahead as db
 
 from jsonschema import FormatChecker, validate, ValidationError
 from pytest import raises
+from unittest.mock import Mock, patch
 
 from caosadvancedtools.json_schema_exporter import recordtype_to_json_schema as rtjs
 
 
+def _mock_execute_query(query_string, unique=False, **kwargs):
+    """Mock the response to queries for references."""
+    all_records = db.Container()
+    all_files = db.Container()
+    other_type_rt = db.RecordType(name="OtherType")
+    other_type_rt.add_property(name="IntegerProp", datatype=db.INTEGER, importance=db.OBLIGATORY)
+    other_type_records = db.Container().extend([
+        db.Record(id=100, name="otherA").add_parent(other_type_rt),
+        db.Record(id=101, name="otherB").add_parent(other_type_rt),
+        db.Record(id=102).add_parent(other_type_rt)
+    ])
+    all_records.extend(other_type_records)
+
+    referencing_type_rt = db.RecordType(name="ReferencingType")
+    referencing_type_rt.add_property(name=other_type_rt.name, datatype=db.LIST(other_type_rt.name))
+    referencing_type_records = db.Container().extend([
+        db.Record(id=103).add_parent(referencing_type_rt),
+        db.Record(id=104, name="referencing").add_parent(referencing_type_rt)
+    ])
+    all_records.extend(referencing_type_records)
+
+    all_files.append(db.File(id=105, name="GenericFile.txt"))
+
+    if query_string == "SELECT name, id FROM RECORD 'OtherType'":
+        return other_type_records
+    elif query_string == "FIND RECORDTYPE WITH name='OtherType'" and unique is True:
+        return other_type_rt
+    elif query_string == "SELECT name, id FROM RECORD 'ReferencingType'":
+        return referencing_type_records
+    elif query_string == "FIND RECORDTYPE WITH name='ReferencingType'" and unique is True:
+        return referencing_type_rt
+    elif query_string == "SELECT name, id FROM RECORD":
+        return all_records
+    elif query_string == "SELECT name, id FROM FILE":
+        return all_files
+    else:
+        return db.Container()
+
+
 def test_empty_rt():
 
     rt = db.RecordType(name="Test", description="descr")
@@ -251,39 +291,240 @@ def test_rt_with_list_props():
         validate(example, schema, format_checker=FormatChecker())
 
 
+@patch("linkahead.execute_query", new=Mock(side_effect=_mock_execute_query))
 def test_rt_with_references():
-    """References and lists of references will come later, so test if the errors
-    are thrown correctly.
+    """References and lists of references to files will come later, so test if
+    the errors are thrown correctly.
 
     """
 
     rt = db.RecordType()
     rt.add_property(name="RefProp", datatype=db.REFERENCE)
 
-    with raises(NotImplementedError):
+    schema = rtjs(rt)
+    props = schema["properties"]
+    assert "RefProp" in props
+    assert "enum" in props["RefProp"]
+    assert isinstance(props["RefProp"]["enum"], list)
+    assert len(props["RefProp"]["enum"]) == len(
+        db.execute_query("SELECT name, id FROM RECORD")) + len(
+            db.execute_query("SELECT name, id FROM FILE"))
+    assert "oneOf" not in props["RefProp"]
 
-        rtjs(rt)
+    example = {
+        "RefProp": "101, otherB"
+    }
+    validate(example, schema)
+    example = {
+        "RefProp": "23, I don't exist"
+    }
+    with raises(ValidationError):
+        # Wrong enum value
+        validate(example, schema)
+    example = {
+        "RefProp": {
+            "IntegerProp": 12
+        }
+    }
+    with raises(ValidationError):
+        # Can't have objects in generic references
+        validate(example, schema)
 
     rt = db.RecordType()
     rt.add_property(name="RefProp", datatype="OtherType")
+    rt.add_property(name="OtherTextProp", datatype=db.TEXT)
 
-    with raises(NotImplementedError):
+    schema = rtjs(rt)
+    props = schema["properties"]
+    assert "RefProp" in props
+    assert "oneOf" in props["RefProp"]
+    assert len(props["RefProp"]["oneOf"]) == 2
+    enum_index = 0
+    if "enum" not in props["RefProp"]["oneOf"][enum_index]:
+        # We can't really require the order here, so we just know that one of
+        # the two elements must be the enum, the other the object.
+        enum_index = 1 - enum_index
+    assert "enum" in props["RefProp"]["oneOf"][enum_index]
+    assert isinstance(props["RefProp"]["oneOf"][enum_index]["enum"], list)
+    assert len(props["RefProp"]["oneOf"][enum_index]["enum"]) == 3
+    assert "100, otherA" in props["RefProp"]["oneOf"][enum_index]["enum"]
+    assert "101, otherB" in props["RefProp"]["oneOf"][enum_index]["enum"]
+    assert "102" in props["RefProp"]["oneOf"][enum_index]["enum"]
+    # the other element of oneOf is the OtherType object
+    assert props["RefProp"]["oneOf"][1 - enum_index]["type"] == "object"
+    other_props = props["RefProp"]["oneOf"][1 - enum_index]["properties"]
+    assert "IntegerProp" in other_props
+    assert other_props["IntegerProp"]["type"] == "integer"
+    assert "required" in props["RefProp"]["oneOf"][1 - enum_index]
+    assert len(props["RefProp"]["oneOf"][1 - enum_index]["required"]) == 1
+    assert "IntegerProp" in props["RefProp"]["oneOf"][1 - enum_index]["required"]
+    # The other prop also works as before
+    assert "OtherTextProp" in props
+    assert props["OtherTextProp"]["type"] == "string"
 
-        rtjs(rt)
+    example = {
+        "RefProp": {
+            "IntegerProp": 12
+        }
+    }
+    validate(example, schema)
 
-    rt = db.RecordType()
-    rt.add_property(name="RefProp", datatype=db.LIST(db.REFERENCE))
+    example = {
+        "RefProp": "101, otherB",
+        "OtherTextProp": "something"
+    }
+    validate(example, schema)
 
-    with raises(NotImplementedError):
+    rt = db.RecordType(name="TestType", description="Some description")
+    rt.add_property(name="RefProp", datatype=db.LIST(db.REFERENCE),
+                    description="I'm a list of references.")
 
-        rtjs(rt)
+    schema = rtjs(rt)
+    assert schema["title"] == rt.name
+    assert schema["description"] == rt.description
+    assert "RefProp" in schema["properties"]
+    ref_prop = schema["properties"]["RefProp"]
+    assert ref_prop["type"] == "array"
+    assert "description" in ref_prop
+    assert ref_prop["description"] == "I'm a list of references."
+    assert "items" in ref_prop
+    items = ref_prop["items"]
+    assert "enum" in items
+    assert isinstance(items["enum"], list)
+    assert len(items["enum"]) == len(
+        db.execute_query("SELECT name, id FROM RECORD")) + len(
+            db.execute_query("SELECT name, id FROM FILE"))
+    assert "oneOf" not in items
+    assert "description" not in items
+
+    example = {
+        "RefProp": "101, otherB"
+    }
+    with raises(ValidationError):
+        # Should be list but isn't
+        validate(example, schema)
+    example = {
+        "RefProp": ["101, otherB"]
+    }
+    validate(example, schema)
+    example = {
+        "RefProp": ["101, otherB", "102", "104, referencing"]
+    }
+    validate(example, schema)
 
     rt = db.RecordType()
     rt.add_property(name="RefProp", datatype=db.LIST("OtherType"))
 
+    schema = rtjs(rt, additional_properties=False,
+                  name_and_description_in_properties=True)
+    assert schema["additionalProperties"] is False
+    assert "name" in schema["properties"]
+    assert schema["properties"]["name"]["type"] == "string"
+    assert "description" in schema["properties"]
+    assert schema["properties"]["description"]["type"] == "string"
+    assert "RefProp" in schema["properties"]
+    assert schema["properties"]["RefProp"]["type"] == "array"
+    assert "additionalProperties" not in schema["properties"]["RefProp"]
+    assert "items" in schema["properties"]["RefProp"]
+    items = schema["properties"]["RefProp"]["items"]
+    assert "oneOf" in items
+    assert len(items["oneOf"]) == 2
+    # same as above, we can't rely on the order
+    enum_index = 0
+    if "enum" not in items["oneOf"][enum_index]:
+        enum_index = 1 - enum_index
+    assert "enum" in items["oneOf"][enum_index]
+    assert isinstance(items["oneOf"][enum_index]["enum"], list)
+    assert len(items["oneOf"][enum_index]["enum"]) == 3
+    assert "100, otherA" in items["oneOf"][enum_index]["enum"]
+    assert "101, otherB" in items["oneOf"][enum_index]["enum"]
+    assert "102" in items["oneOf"][enum_index]["enum"]
+    other_type = items["oneOf"][1 - enum_index]
+    assert other_type["type"] == "object"
+    assert other_type["additionalProperties"] is False
+    assert "IntegerProp" in other_type["properties"]
+    assert len(other_type["required"]) == 1
+    assert "IntegerProp" in other_type["required"]
+
+    example = {
+        "RefProp": ["101, otherB", "102", "104, referencing"]
+    }
+    with raises(ValidationError):
+        # Wrong value in enum
+        validate(example, schema)
+    example = {
+        "RefProp": [{"IntegerProp": 12}]
+    }
+    validate(example, schema)
+    example = {
+        "RefProp": [{"IntegerProp": 12, "additionalProperty": "something"}]
+    }
+    with raises(ValidationError):
+        # we have additional_properties=False which propagates to subschemas
+        validate(example, schema)
+    example = {
+        "RefProp": [{"IntegerProp": 12}, "101, otherB"]
+    }
+    validate(example, schema)
+
+    rt = db.RecordType(name="ReferenceofReferencesType")
+    rt.add_property(name="RefRefProp", datatype="ReferencingType")
+
+    schema = rtjs(rt)
+
+    assert "RefRefProp" in schema["properties"]
+    ref_ref = schema["properties"]["RefRefProp"]
+    assert "oneOf" in ref_ref
+    assert len(ref_ref["oneOf"]) == 2
+    enum_index = 0
+    if "enum" not in ref_ref["oneOf"][enum_index]:
+        enum_index = 1 - enum_index
+    assert len(ref_ref["oneOf"][enum_index]["enum"]) == 2
+    assert "103" in ref_ref["oneOf"][enum_index]["enum"]
+    assert "104, referencing" in ref_ref["oneOf"][enum_index]["enum"]
+    assert ref_ref["oneOf"][1 - enum_index]["type"] == "object"
+    assert "OtherType" in ref_ref["oneOf"][1 - enum_index]["properties"]
+    assert ref_ref["oneOf"][1 - enum_index]["properties"]["OtherType"]["type"] == "array"
+    items = ref_ref["oneOf"][1 - enum_index]["properties"]["OtherType"]["items"]
+    assert "oneOf" in items
+    assert len(items["oneOf"]) == 2
+    # same as above, we can't rely on the order
+    enum_index = 0
+    if "enum" not in items["oneOf"][enum_index]:
+        enum_index = 1 - enum_index
+    assert "enum" in items["oneOf"][enum_index]
+    assert isinstance(items["oneOf"][enum_index]["enum"], list)
+    assert len(items["oneOf"][enum_index]["enum"]) == 3
+    assert "100, otherA" in items["oneOf"][enum_index]["enum"]
+    assert "101, otherB" in items["oneOf"][enum_index]["enum"]
+    assert "102" in items["oneOf"][enum_index]["enum"]
+    other_type = items["oneOf"][1 - enum_index]
+    assert other_type["type"] == "object"
+    assert "IntegerProp" in other_type["properties"]
+    assert len(other_type["required"]) == 1
+    assert "IntegerProp" in other_type["required"]
+
+    example = {
+        "RefRefProp": {
+            "OtherType": [
+                "100, otherA",
+                {"IntegerProp": 12}
+            ]
+        }
+    }
+    validate(example, schema)
+
+    rt = db.RecordType()
+    rt.add_property(name="FileProp", datatype=db.FILE)
+
     with raises(NotImplementedError):
+        schema = rtjs(rt)
 
-        rtjs(rt)
+    rt = db.RecordType()
+    rt.add_property(name="FileProp", datatype=db.LIST(db.FILE))
+
+    with raises(NotImplementedError):
+        schema = rtjs(rt)
 
 
 def test_broken():