diff --git a/integrationtests/test_data/extroot/use_case_simple_presentation/cfood.yml b/integrationtests/test_data/extroot/use_case_simple_presentation/cfood.yml index 6495e1828dc56e99459c162f7751951f880ea55c..da3c2dfe8276325c26607239071da3ccaf41a40c 100644 --- a/integrationtests/test_data/extroot/use_case_simple_presentation/cfood.yml +++ b/integrationtests/test_data/extroot/use_case_simple_presentation/cfood.yml @@ -25,8 +25,8 @@ extroot: parents: - mdfile role: File - path: $DataFile - file: $DataFile + path: $DataFile_path + file: $DataFile_path Experiment: mdfile: $mdfile diff --git a/src/caoscrawler/converters.py b/src/caoscrawler/converters.py index 7e0603d1ae1b9f10a38e71f4d8fa883ede6877b5..e359be4f30bf971ecfe06de343cae69bb2d29ef2 100644 --- a/src/caoscrawler/converters.py +++ b/src/caoscrawler/converters.py @@ -578,11 +578,14 @@ class MarkdownFileConverter(SimpleFileConverter): header = yaml_header_tools.get_header_from_file( element.path, clean=False) except yaml_header_tools.NoValidHeader: - path = generalStore[self.name] + if generalStore is not None and self.name in generalStore: + path = generalStore[self.name] + else: + path = "<path not set>" raise ConverterValidationError( "Error during the validation (yaml header cannot be read) of the markdown file " "located at the following node in the data structure:\n" - f"{path}\n" + err.message) + f"{path}") children: List[StructureElement] = [] for name, entry in header.items(): diff --git a/unittests/test_converters.py b/unittests/test_converters.py index d0d2e23abf02e3ff8f9641744a5d300019c4626d..c1ca9ea03c8631678f5b67cad3c33685a16cfd32 100644 --- a/unittests/test_converters.py +++ b/unittests/test_converters.py @@ -130,14 +130,11 @@ def test_markdown_converter(converter_registry): ) ) - converter = MarkdownFileConverter({ - "match": "(.*)" - }, "TestMarkdownFileConverter", - converter_registry) + converter = MarkdownFileConverter({"match": "(.*)"}, "TestMarkdownFileConverter", + converter_registry) - m = converter.match(File("test_tool.py", rfp( - "test_tool.py"))) - assert m is None + with pytest.raises(ConverterValidationError) as err: + converter.create_children(None, File("test_tool.py", rfp("test_tool.py"))) m = converter.match(test_readme) assert m is not None diff --git a/unittests/test_scalars_cfood.py b/unittests/test_scalars_cfood.py index 1bf8f0b7d67f00f2018b5b68424d6b9cc17602eb..5e5bc998842e401edad94a2f1b3cdd5348d671b6 100644 --- a/unittests/test_scalars_cfood.py +++ b/unittests/test_scalars_cfood.py @@ -42,16 +42,24 @@ def test_record_structure_generation(crawler): subd = crawler.debug_tree[dircheckstr("DataAnalysis")] assert len(subd) == 2 # variables store on Data Analysis node of debug tree - assert len(subd[0]) == 3 - assert "Data" in subd[0] - assert "DataAnalysis" in subd[0] - assert "RecordThatGetsParentsLater" in subd[0] + if "Data" in subd[0]: + subddata = subd[0] + subdRTGPL = subd[1] + else: + subddata = subd[1] + subdRTGPL = subd[0] + print(subddata.keys()) + assert len(subddata) == 5 + assert "DataAnalysis" in subddata + assert "DataAnalysis_path" in subddata + assert "Data_path" in subddata + assert "RecordThatGetsParentsLater" in subddata - prop = subd[0]["RecordThatGetsParentsLater"].get_property("someId") + prop = subddata["RecordThatGetsParentsLater"].get_property("someId") assert type(prop.value) == int assert prop.value == 23 # record store on Data Analysis node of debug tree - assert len(subd[1]) == 1 - prop2 = subd[1]["RecordThatGetsParentsLater"].get_property("someId") + assert len(subdRTGPL) == 2 + prop2 = subdRTGPL["RecordThatGetsParentsLater"].get_property("someId") assert prop == prop2 diff --git a/unittests/test_tool.py b/unittests/test_tool.py index 23b35f2dc9228eeda9137945198c49c19bf5c474..4ac2b4577fbeea6f4bdf291c48ddaf0fa418b2a5 100755 --- a/unittests/test_tool.py +++ b/unittests/test_tool.py @@ -110,15 +110,17 @@ def ident(crawler): def test_record_structure_generation(crawler): + # TODO How does this test relate to the test function in test_scalars_cfood with the same name? + # There seems to be code duplication subd = crawler.debug_tree[dircheckstr("DataAnalysis")] subc = crawler.debug_metadata["copied"][dircheckstr("DataAnalysis")] assert len(subd) == 2 # variables store on Data Analysis node of debug tree - assert len(subd[0]) == 2 + assert len(subd[0]) == 4 # record store on Data Analysis node of debug tree assert len(subd[1]) == 0 assert len(subc) == 2 - assert len(subc[0]) == 2 + assert len(subc[0]) == 4 assert len(subc[1]) == 0 # The data analysis node creates one variable for the node itself: @@ -137,7 +139,7 @@ def test_record_structure_generation(crawler): assert subd[1]["Project"].get_property( "identifier").value == "climate-model-predict" - assert len(subd[0]) == 6 + assert len(subd[0]) == 9 assert subd[0]["date"] == "2020" assert subd[0]["identifier"] == "climate-model-predict" assert subd[0]["Project"].__class__ == db.Record @@ -148,7 +150,7 @@ def test_record_structure_generation(crawler): assert subc[0]["project_dir"] is False # Check the copy flags for the first level in the hierarchy: - assert len(subc[0]) == 6 + assert len(subc[0]) == 9 assert len(subc[1]) == 1 assert subc[1]["Project"] is False assert subc[0]["Project"] is False @@ -161,7 +163,7 @@ def test_record_structure_generation(crawler): subc = crawler.debug_metadata["copied"][dircheckstr("DataAnalysis", "2020_climate-model-predict", "2020-02-08_prediction-errors")] - assert len(subd[0]) == 8 + assert len(subd[0]) == 12 assert subd[0]["date"] == "2020-02-08" assert subd[0]["identifier"] == "prediction-errors" assert subd[0]["Project"].__class__ == db.Record