TST: Add unit test for properties from dict converter

unittests/test_converters.py

@@ -39,15 +39,16 @@ from caoscrawler.converters import (Converter, ConverterValidationError,
                                     DictIntegerElementConverter,
                                     DirectoryConverter, FloatElementConverter,
                                     IntegerElementConverter, JSONFileConverter,
-                                    ListElementConverter,
-                                    MarkdownFileConverter, YAMLFileConverter,
+                                    ListElementConverter, MarkdownFileConverter,
+                                    PropertiesFromDictConverter,
+                                    YAMLFileConverter,
                                     _AbstractScalarValueElementConverter,
                                     handle_value, replace_variables)
 from caoscrawler.crawl import Crawler
 from caoscrawler.scanner import (_load_definition_from_yaml_dict,
                                  create_converter_registry,
                                  create_transformer_registry, load_definition)
-from caoscrawler.stores import GeneralStore
+from caoscrawler.stores import GeneralStore, RecordStore
 from caoscrawler.structure_elements import (BooleanElement, DictElement,
                                             Directory, File, FloatElement,
                                             IntegerElement, ListElement,
@@ -72,8 +73,8 @@ def converter_registry():
         "DictElement": {
             "converter": "DictElementConverter",
             "package": "caoscrawler.converters"},
-        "HeuristicDictElement": {
-            "converter": "HeuristicDictConverter",
+        "PropertiesFromDictElement": {
+            "converter": "PropertiesFromDictConverter",
             "package": "caoscrawler.converters"
         },
         "TextElement": {
@@ -665,8 +666,126 @@ def test_create_path_value(converter_registry):
     assert values["Test.path"] == "/a"
 
 
-def test_dict_heuristic_basic():
+def test_properties_from_dict_basic(converter_registry):
     """Test that a record with the correct name and properties is created, and
     that the children are still created correctly.
 
     """
+    # TODO definitions with blacklist and named references
+    pfdc = PropertiesFromDictConverter(
+        definition={
+            "type": "DictElement",
+            "match": ".*",
+            "record_from_dict": {
+                "variable_name": "MyRec",
+                "parents": ["DictRT1", "DictRT2"],
+                "properties_blacklist": ["blacklisted_int", "blacklisted_ref"],
+                "references": {
+                    "authors": {
+                        "parents": ["Person"]
+                    }
+                }
+            }
+        },
+        name="Test", converter_registry=converter_registry)
+    # TODO: Tests for Dict with skalars, dict with lists, dict with reference,
+    # dict with list of references, dict with reference with reference, named
+    # reference
+    values = GeneralStore()
+    records = RecordStore()
+    test_dict_element = DictElement("TestDictElement", {
+        "a": 5,
+        "b": ["a", "b", "c"],
+        "scalar_ref": {
+            "a": 23,
+            "blacklisted_int": 42
+        },
+        "list_ref": [
+            {
+                "c": True
+            },
+            {
+                "c": False
+            }
+        ],
+        "ref_with_ref": {
+            "a": 789,
+            "ref_in_ref": {
+                "b": "something"
+            }
+        },
+        "blacklisted_int": -123,
+        "blacklisted_ref": {
+            "a": 25
+        },
+        "author": {
+            "full_name": "Some Author"
+        }
+    })
+    pfdc.create_records(values=values, records=records, element=test_dict_element)
+    assert "MyRec" in records
+    my_rec = records["MyRec"]
+    assert isinstance(my_rec, db.Record)
+
+    # scalar prop
+    assert my_rec.get_property("a") is not None
+    assert my_rec.get_property("a").value == 5
+
+    # list prop
+    assert my_rec.get_property("b") is not None
+    assert len(my_rec.get_property("b").value) == 3
+    for elt in ["a", "b", "c"]:
+        assert elt in my_rec.get_property("b").value
+
+    # scalar ref
+    assert my_rec.get_property("scalar_ref") is not None
+    referenced = my_rec.get_property("scalar_ref").value
+    assert isinstance(referenced, db.Record)
+    assert len(referenced.parents) == 1
+    assert "scalar_ref" in [par.name for par in referenced.parents]
+    assert referenced.get_property("a") is not None
+    assert referenced.get_property("a").value == 23
+    # blacklisted
+    assert referenced.get_property("blacklisted_int") is None
+
+    # list of ref
+    assert my_rec.get_property("list_ref") is not None
+    assert isinstance(my_rec.get_property("list_ref").value, list)
+    assert len(my_rec.get_property("list_ref").value) == 2
+    for rec in my_rec.get_property("list_ref").value:
+        assert isinstance(rec, db.Record)
+        assert len(rec.parents) == 1
+        assert "list_ref" in [par.name for par in rec.parents]
+        assert rec.get_property("c") is not None
+        assert type(rec.get_property("c")) is bool
+    assert True in [rec.get_property("c").value for rec in my_rec.get_property("list_ref").value]
+    assert False in [rec.get_property("c").value for rec in my_rec.get_property("list_ref").value]
+
+    # ref with ref
+    assert my_rec.get_property("ref_with_ref") is not None
+    outer_rec = my_rec.get_property("ref_with_ref").value
+    assert isinstance(outer_rec, db.Record)
+    assert len(outer_rec.parents) == 1
+    assert "ref_with_ref" in [par.name for par in outer_rec.parents]
+    assert outer_rec.get_property("a") is not None
+    assert outer_rec.get_property("a").value == 789
+    assert outer_rec.get_property("ref_in_ref") is not None
+    inner_rec = outer_rec.get_property("ref_in_ref").value
+    assert isinstance(inner_rec, db.Record)
+    assert len(inner_rec.parents) == 1
+    assert "ref_in_ref" in [par.name for par in inner_rec.parents]
+    assert inner_rec.get_property("b") is not None
+    assert inner_rec.get_property("b").value == "something"
+
+    # blacklisted
+    assert my_rec.get_property("blacklisted_int") is None
+    assert my_rec.get_property("blacklisted_ref") is None
+
+    # named reference property
+    assert my_rec.get_property("author") is not None
+    author_rec = my_rec.get_property("author").value
+    assert isinstance(author_rec, db.Record)
+    assert len(author_rec.parents) == 1
+    assert "Person" in [par.name for par in author_rec.parents]
+    assert author_rec.get_property("full_name") is not None
+    assert author_rec.get_property("full_name").value == "Some Author"