F dict heuristic

Summary

Part of https://gitlab.indiscale.com/caosdb/customers/3dmmto/management/-/issues/30. Introduce a converter that automatically transforms dictionary entries to property values without having to specify them explicitly. It also also creates referenced Records from object-typed dictionary values. Properties can be blacklisted.

The new converter extends the existing DictElementConverter, so we can still use the existing treatment via records and subtree entries in the converter definition. This allows for the special treatment of blacklisted dictionary keys, but it is not limited to them. We do not change the original create_children method, so all dictionary entries are also available in the subtree if needed elsewhere.

Focus

Mainly the new converter and its documentation.

Test Environment

make doc and check the new documentation. Is it understandable? Is it useful?

For actual testing the new unit test should be sufficient. Can you think of corner cases that they do not cover yet?

Check List for the Author

Please, prepare your MR for a review. Be sure to write a summary and a focus and create gitlab comments for the reviewer. They should guide the reviewer through the changes, explain your changes and also point out open questions. For further good practices have a look at our review guidelines

• All automated tests pass
• Reference related issues
• Up-to-date CHANGELOG.md (or not necessary)
• Up-to-date JSON schema (or not necessary)
• Appropriate user and developer documentation (or not necessary)
  • How do I use the software? Assume "stupid" users.
  • How do I develop or debug the software? Assume novice developers.
• Annotations in code (Gitlab comments)
  • Intent of new code
  • Problems with old code
  • Why this implementation?

Check List for the Reviewer

• I understand the intent of this MR
• All automated tests pass
• Up-to-date CHANGELOG.md (or not necessary)
• Appropriate user and developer documentation (or not necessary)
• The test environment setup works and the intended behavior is reproducible in the test environment
• In-code documentation and comments are up-to-date.
• Check: Are there specifications? Are they satisfied?

For further good practices have a look at our review guidelines.

unittests/test_converters.py

@@ -742,7 +742,7 @@ def test_properties_from_dict_basic(converter_registry):
     referenced = my_rec.get_property("scalar_ref").value
     assert isinstance(referenced, db.Record)
     assert len(referenced.parents) == 1
-    assert "scalar_ref" in [par.name for par in referenced.parents]
+    assert referenced.has_parent("scalar_ref")
     assert referenced.get_property("a") is not None
     assert referenced.get_property("a").value == 23
     # blacklisted
@@ -755,7 +755,7 @@ def test_properties_from_dict_basic(converter_registry):
     for rec in my_rec.get_property("list_ref").value:
         assert isinstance(rec, db.Record)
         assert len(rec.parents) == 1
-        assert "list_ref" in [par.name for par in rec.parents]
+        assert rec.has_parent("list_ref")
         assert rec.get_property("c") is not None
         assert type(rec.get_property("c")) is bool
     assert True in [rec.get_property("c").value for rec in my_rec.get_property("list_ref").value]
@@ -766,14 +766,14 @@ def test_properties_from_dict_basic(converter_registry):
     outer_rec = my_rec.get_property("ref_with_ref").value
     assert isinstance(outer_rec, db.Record)
     assert len(outer_rec.parents) == 1
-    assert "ref_with_ref" in [par.name for par in outer_rec.parents]
+    assert outer_rec.has_parent("ref_with_ref")
     assert outer_rec.get_property("a") is not None
     assert outer_rec.get_property("a").value == 789
     assert outer_rec.get_property("ref_in_ref") is not None
     inner_rec = outer_rec.get_property("ref_in_ref").value
     assert isinstance(inner_rec, db.Record)
     assert len(inner_rec.parents) == 1
-    assert "ref_in_ref" in [par.name for par in inner_rec.parents]
+    assert inner_rec.has_parent("ref_in_ref")
     assert inner_rec.get_property("b") is not None
     assert inner_rec.get_property("b").value == "something"
 
@@ -786,6 +786,99 @@ def test_properties_from_dict_basic(converter_registry):
     author_rec = my_rec.get_property("author").value
     assert isinstance(author_rec, db.Record)
     assert len(author_rec.parents) == 1
-    assert "Person" in [par.name for par in author_rec.parents]
+    assert author_rec.has_parent("Person")
     assert author_rec.get_property("full_name") is not None
     assert author_rec.get_property("full_name").value == "Some Author"
+
+
+def test_properties_from_dict_errors(converter_registry):
+
+    with pytest.raises(ValueError) as ve:
+        pdfc = PropertiesFromDictConverter(
+            definition={
+
+            },
+            name="TestConverter",
+            converter_registry=converter_registry
+        )
+
+    assert "You need to specify the (root) record" in str(ve.value)
+
+    with pytest.raises(ValueError) as ve:
+        pdfc = PropertiesFromDictConverter(
+            definition={
+                "record_from_dict": {}
+            },
+            name="TestConverter",
+            converter_registry=converter_registry
+        )
+
+    assert "needs to have a `variable_name`" in str(ve.value)
+
+
+def test_properties_from_dict_callable(converter_registry):
+
+    def convert_some_values(rec: db.Record):
+        """Add an URL prefix to a property value if appliccable."""
+
+        if rec.get_property("url") is not None:
+
+            old_val = rec.get_property("url").value
+            if not (old_val is None or old_val.startswith("http")):
+
+                # only add if there is a value that doesn't look like an URL
+                rec.get_property("url").value = f"https://test.com/{old_val}"
+
+        return rec
+
+    pdfc = PropertiesFromDictConverter(
+        definition={
+            "record_from_dict": {
+                "variable_name": "MyRec"
+                "name": "My New Record"
+            },
+            name = "TestConverter",
+            converter_registry = converter_registry,
+            referenced_record_callback = convert_some_values
+        }
+    )
+
+    values = GeneralStore()
+    records = RecordStore()
+    test_dict_element = DictElement("TestDictElement", {
+        "url": "something",
+        "referenced1": {
+            "url": "referenced"
+        },
+        "referenced2": {
+            "nourl": "something else"
+            "url": "https://indiscale.com"
+        }
+    })
+    pdfc.create_records(values=values, records=records, element=test_dict_element)
+    assert "MyRec" in records
+    my_rec = records["MyRec"]
+    assert isinstance(my_rec, db.Record)
+    assert len(my_rec.parents) == 1
+    assert my_rec.has_parent("MyRec")
+    assert my_rec.name == "My New Record"
+
+    # simple conversion
+    assert my_rec.get_property("url") is not None
+    assert my_rec.get_property("url").value == "https://test.com/something"
+
+    # also works in referenced
+    assert my_rec.get_property("referenced1") is not None
+    referenced1 = my_rec.get_property("referenced1").value
+    assert isinstance(referenced1, db.Record)
+    assert referenced1.get_property("url") is not None
+    assert referenced1.get_property("url").value == "https://test.com/referenced"
+
+    # ... and works as expected
+    assert my_rec.get_property("referenced2") is not None
+    referenced2 = my_rec.get_property("referenced2").value
+    assert isinstance(referenced2, db.Record)
+    assert referenced2.get_property("nourl") is not None
+    assert referenced2.get_property("nourl").value == "something else"
+    assert referenced2.get_property("url") is not None
+    assert referenced2.get_property("url").value == "https://indiscale.com"