From 79cfabed242db71c19e9222b0a9a4f1aeffccd95 Mon Sep 17 00:00:00 2001
From: Florian Spreckelsen <f.spreckelsen@indiscale.com>
Date: Thu, 28 Mar 2024 10:27:12 +0100
Subject: [PATCH] TST: Add unit test for properties from dict converter
---
unittests/test_converters.py | 131 +++++++++++++++++++++++++++++++++--
1 file changed, 125 insertions(+), 6 deletions(-)
diff --git a/unittests/test_converters.py b/unittests/test_converters.py
index 8fa44b8a..0be4bf64 100644
--- a/unittests/test_converters.py
+++ b/unittests/test_converters.py
@@ -39,15 +39,16 @@ from caoscrawler.converters import (Converter, ConverterValidationError,
DictIntegerElementConverter,
DirectoryConverter, FloatElementConverter,
IntegerElementConverter, JSONFileConverter,
- ListElementConverter,
- MarkdownFileConverter, YAMLFileConverter,
+ ListElementConverter, MarkdownFileConverter,
+ PropertiesFromDictConverter,
+ YAMLFileConverter,
_AbstractScalarValueElementConverter,
handle_value, replace_variables)
from caoscrawler.crawl import Crawler
from caoscrawler.scanner import (_load_definition_from_yaml_dict,
create_converter_registry,
create_transformer_registry, load_definition)
-from caoscrawler.stores import GeneralStore
+from caoscrawler.stores import GeneralStore, RecordStore
from caoscrawler.structure_elements import (BooleanElement, DictElement,
Directory, File, FloatElement,
IntegerElement, ListElement,
@@ -72,8 +73,8 @@ def converter_registry():
"DictElement": {
"converter": "DictElementConverter",
"package": "caoscrawler.converters"},
- "HeuristicDictElement": {
- "converter": "HeuristicDictConverter",
+ "PropertiesFromDictElement": {
+ "converter": "PropertiesFromDictConverter",
"package": "caoscrawler.converters"
},
"TextElement": {
@@ -665,8 +666,126 @@ def test_create_path_value(converter_registry):
assert values["Test.path"] == "/a"
-def test_dict_heuristic_basic():
+def test_properties_from_dict_basic(converter_registry):
"""Test that a record with the correct name and properties is created, and
that the children are still created correctly.
"""
+ # TODO definitions with blacklist and named references
+ pfdc = PropertiesFromDictConverter(
+ definition={
+ "type": "DictElement",
+ "match": ".*",
+ "record_from_dict": {
+ "variable_name": "MyRec",
+ "parents": ["DictRT1", "DictRT2"],
+ "properties_blacklist": ["blacklisted_int", "blacklisted_ref"],
+ "references": {
+ "authors": {
+ "parents": ["Person"]
+ }
+ }
+ }
+ },
+ name="Test", converter_registry=converter_registry)
+ # TODO: Tests for Dict with skalars, dict with lists, dict with reference,
+ # dict with list of references, dict with reference with reference, named
+ # reference
+ values = GeneralStore()
+ records = RecordStore()
+ test_dict_element = DictElement("TestDictElement", {
+ "a": 5,
+ "b": ["a", "b", "c"],
+ "scalar_ref": {
+ "a": 23,
+ "blacklisted_int": 42
+ },
+ "list_ref": [
+ {
+ "c": True
+ },
+ {
+ "c": False
+ }
+ ],
+ "ref_with_ref": {
+ "a": 789,
+ "ref_in_ref": {
+ "b": "something"
+ }
+ },
+ "blacklisted_int": -123,
+ "blacklisted_ref": {
+ "a": 25
+ },
+ "author": {
+ "full_name": "Some Author"
+ }
+ })
+ pfdc.create_records(values=values, records=records, element=test_dict_element)
+ assert "MyRec" in records
+ my_rec = records["MyRec"]
+ assert isinstance(my_rec, db.Record)
+
+ # scalar prop
+ assert my_rec.get_property("a") is not None
+ assert my_rec.get_property("a").value == 5
+
+ # list prop
+ assert my_rec.get_property("b") is not None
+ assert len(my_rec.get_property("b").value) == 3
+ for elt in ["a", "b", "c"]:
+ assert elt in my_rec.get_property("b").value
+
+ # scalar ref
+ assert my_rec.get_property("scalar_ref") is not None
+ referenced = my_rec.get_property("scalar_ref").value
+ assert isinstance(referenced, db.Record)
+ assert len(referenced.parents) == 1
+ assert "scalar_ref" in [par.name for par in referenced.parents]
+ assert referenced.get_property("a") is not None
+ assert referenced.get_property("a").value == 23
+ # blacklisted
+ assert referenced.get_property("blacklisted_int") is None
+
+ # list of ref
+ assert my_rec.get_property("list_ref") is not None
+ assert isinstance(my_rec.get_property("list_ref").value, list)
+ assert len(my_rec.get_property("list_ref").value) == 2
+ for rec in my_rec.get_property("list_ref").value:
+ assert isinstance(rec, db.Record)
+ assert len(rec.parents) == 1
+ assert "list_ref" in [par.name for par in rec.parents]
+ assert rec.get_property("c") is not None
+ assert type(rec.get_property("c")) is bool
+ assert True in [rec.get_property("c").value for rec in my_rec.get_property("list_ref").value]
+ assert False in [rec.get_property("c").value for rec in my_rec.get_property("list_ref").value]
+
+ # ref with ref
+ assert my_rec.get_property("ref_with_ref") is not None
+ outer_rec = my_rec.get_property("ref_with_ref").value
+ assert isinstance(outer_rec, db.Record)
+ assert len(outer_rec.parents) == 1
+ assert "ref_with_ref" in [par.name for par in outer_rec.parents]
+ assert outer_rec.get_property("a") is not None
+ assert outer_rec.get_property("a").value == 789
+ assert outer_rec.get_property("ref_in_ref") is not None
+ inner_rec = outer_rec.get_property("ref_in_ref").value
+ assert isinstance(inner_rec, db.Record)
+ assert len(inner_rec.parents) == 1
+ assert "ref_in_ref" in [par.name for par in inner_rec.parents]
+ assert inner_rec.get_property("b") is not None
+ assert inner_rec.get_property("b").value == "something"
+
+ # blacklisted
+ assert my_rec.get_property("blacklisted_int") is None
+ assert my_rec.get_property("blacklisted_ref") is None
+
+ # named reference property
+ assert my_rec.get_property("author") is not None
+ author_rec = my_rec.get_property("author").value
+ assert isinstance(author_rec, db.Record)
+ assert len(author_rec.parents) == 1
+ assert "Person" in [par.name for par in author_rec.parents]
+ assert author_rec.get_property("full_name") is not None
+ assert author_rec.get_property("full_name").value == "Some Author"
--
GitLab