Merge branch 'f-more-scifolder-scanner-unittests' into 'dev'

More scifolder scanner unittests

See merge request !124

unittests/test_scanner.py

@@ -35,15 +35,15 @@ from utils import dircheckstr as dircheck_base
 
 UNITTESTDIR = Path(__file__).parent
 
-dircheckstr = partial(dircheck_base, UNITTESTDIR/"test_directories" / "examples_article")
+dircheckstr = partial(dircheck_base, UNITTESTDIR / "test_directories" / "examples_article")
 
 
 def test_scan_structure_elements():
     tmpfi = NamedTemporaryFile(delete=False)
-    with open(UNITTESTDIR/"example_datastructure.yml", "r") as f:
+    with open(UNITTESTDIR / "example_datastructure.yml", "r") as f:
         data = yaml.load(f, Loader=yaml.SafeLoader)
 
-    crawler_definition = load_definition(UNITTESTDIR/"example_cfood.yml")
+    crawler_definition = load_definition(UNITTESTDIR / "example_cfood.yml")
     converter_registry = create_converter_registry(crawler_definition)
     recs = scan_structure_elements(DictElement(name="", value=data), crawler_definition,
                                    converter_registry)
@@ -54,10 +54,10 @@ def test_provenance_debug_data():
     # TODO rewrite the test to use a smaller example setup
     tmpfi = NamedTemporaryFile(delete=False)
     debug_tree = DebugTree()
-    with open(UNITTESTDIR/"example_datastructure.yml", "r") as f:
+    with open(UNITTESTDIR / "example_datastructure.yml", "r") as f:
         data = yaml.load(f, Loader=yaml.SafeLoader)
 
-    crawler_definition = load_definition(UNITTESTDIR/"example_cfood.yml")
+    crawler_definition = load_definition(UNITTESTDIR / "example_cfood.yml")
     converter_registry = create_converter_registry(crawler_definition)
     stuff = scan_structure_elements(DictElement(name="", value=data), crawler_definition,
                                     converter_registry, debug_tree=debug_tree)
@@ -83,8 +83,8 @@ def test_record_structure_generation():
     # TODO test creation of debug information in a separate test
 
     dbt = DebugTree()
-    scan_directory(UNITTESTDIR/"test_directories" / "examples_article",
-                   UNITTESTDIR/"scifolder_cfood.yml",
+    scan_directory(UNITTESTDIR / "test_directories" / "examples_article",
+                   UNITTESTDIR / "scifolder_cfood.yml",
                    debug_tree=dbt)
     subd = dbt.debug_tree[dircheckstr("DataAnalysis")]
     subc = dbt.debug_metadata["copied"][dircheckstr("DataAnalysis")]
@@ -167,3 +167,81 @@ def test_record_structure_generation():
     assert subc[0]["Measurement"] is False
     assert subc[0]["date"] is False
     assert subc[0]["identifier"] is False
+
+
+def test_record_generation():
+    """
+    Test the correct list of returned records by the scanner using the
+    scifolder example from the article.
+    """
+
+    records = scan_directory(UNITTESTDIR / "test_directories" / "examples_article",
+                             UNITTESTDIR / "scifolder_cfood.yml")
+
+    def parent_filter(parent_name):
+        return [p for p in records if len(p.parents) == 1 and p.parents[0].name == parent_name]
+
+    def check_properties(records, check_props, check_additional=True):
+        records_found = [0 for r in check_props]
+        for rec in records:
+            rec_found = 0
+            # Try each record to check
+            for i, check_prop in enumerate(check_props):
+                matches = True
+                # breakpoint()
+                # Verify that all props are in the record and have the right value
+                for pr in check_prop:
+                    if rec.get_property(pr) is None:
+                        matches = False
+                        break
+                    if check_prop[pr] is None:
+                        if rec.get_property(pr).value is not None:
+                            matches = False
+                            break
+                    else:
+                        if rec.get_property(pr).value != check_prop[pr]:
+                            matches = False
+                            break
+                if check_additional:
+                    # Verify that there are no additional props in the record
+                    for rpr in rec.properties:
+                        if rpr.name not in check_prop:
+                            matches = False
+                            break
+                if matches:
+                    records_found[i] += 1
+        return records_found
+
+    # Check projects:
+    # Ther are two projects in mixed categories: climate_model_predict and SpeedOfLight
+    projects_found = check_properties(parent_filter("Project"), [
+        {"identifier": "climate-model-predict", "date": "2020"},
+        {"identifier": "SpeedOfLight", "date": "2020"}
+    ])
+    assert projects_found == [3, 2]
+
+    measurements = parent_filter("Measurement")
+    assert len(measurements) == 11
+    measurements_found = check_properties(measurements, [
+        {"identifier": "prediction-errors", "date": "2020-02-08"},
+        {"identifier": "average-all-exp", "date": "2020-01-04"},
+        {"identifier": "average-all-exp-corr", "date": "2020-01-05"},
+        {"date": "1980-01-01", "identifier": None},
+        {"date": "1990-01-01", "identifier": None},
+        {"date": "2000-01-01", "identifier": None},
+        {"date": "2010-01-01", "identifier": None},
+        {"date": "2020-01-01", "identifier": "TimeOfFlight"},
+        {"date": "2020-01-02", "identifier": "Cavity"},
+        {"date": "2020-01-03", "identifier": None},
+        {"date": "2020-02-01", "identifier": None},
+    ], False)
+    for f in measurements_found:
+        assert f == 1
+
+    persons = parent_filter("Person")
+    check_props = [
+        {"first_name": None, "last_name": "Author" + letter} for letter in
+        ("A", "B", "C", "D", "E")]
+    persons_found = check_properties(persons, check_props)
+    for f in persons_found:
+        assert f > 0