FIX: fixed some bugs in markdown conversion

src/newcrawler/converters.py

@@ -29,7 +29,7 @@ import caosdb as db
 from .stores import GeneralStore, RecordStore
 from .structure_elements import (StructureElement, Directory, File,
                                  TextElement, DictTextElement, DictListElement)
-from typing import Type, Union, Literal
+from typing import Type, Optional
 from abc import abstractmethod
 import yaml_header_tools
 
@@ -160,11 +160,7 @@ class Converter(object):
         m = self.match(element)
         if m is None:
             raise RuntimeError("Condition does not match.")
-        if type(m) == bool:
-            if m == False:
-                raise RuntimeError("Result of match must not be False, use None instead.")
-            return
-        values.update(m.groupdict())
+        values.update(m)
 
     @abstractmethod
     def create_children(self, values: GeneralStore,
@@ -205,7 +201,7 @@ class Converter(object):
         pass
 
     @abstractmethod
-    def match(self, element: StructureElement) -> Union[re.Match, Literal[True], None]:
+    def match(self, element: StructureElement) -> Optional[dict]:
         pass
 
 class DirectoryConverter(Converter):
@@ -229,7 +225,10 @@ class DirectoryConverter(Converter):
     def match(self, element: StructureElement):
         if not isinstance(element, Directory):
             raise RuntimeError("Element must be a directory.")
-        return re.match(self.definition["match"], element.name)
+        m = re.match(self.definition["match"], element.name)
+        if m is None:
+            return None
+        return m.groupdict()
 
     @staticmethod
     def create_children_from_directory(element: Directory):
@@ -263,7 +262,7 @@ class MarkdownFileConverter(Converter):
         if not isinstance(element, File):
             raise RuntimeError("A markdown file is needed to create children.")
 
-        header = yaml_header_tools.get_header_from_file(element.path)
+        header = yaml_header_tools.get_header_from_file(element.path, clean=False)
         children: list[StructureElement] = []
 
         for name, entry in header.items():
@@ -283,11 +282,13 @@ class MarkdownFileConverter(Converter):
         if not isinstance(element, File):
             raise RuntimeError("Element must be a file.")
         m = re.match(self.definition["match"], element.name)
+        if m is None:
+            return None
         try:
             yaml_header_tools.get_header_from_file(element.path)
         except yaml_header_tools.NoValidHeader:
             return None
-        return m
+        return m.groupdict()
 
 class DictTextElementConverter(Converter):
     def create_children(self, generalStore: GeneralStore,
@@ -298,13 +299,19 @@ class DictTextElementConverter(Converter):
     def typecheck(self, element: StructureElement):
         return isinstance(element, DictTextElement)
 
-    def match(self, element: StructureElement) -> Union[re.Match, Literal[True], None]:
+    def match(self, element: StructureElement):
         if not isinstance(element, DictTextElement):
             raise RuntimeError("Element must be a DictTextElement.")
-        if self.name != element.name:
+        m1 = re.match(self.definition["match_name"], self.name)
+        if m1 is None:
             return None
-        m = re.match(self.definition["match"], element.value)
-        return m
+        m2 = re.match(self.definition["match_value"], element.value)
+        if m2 is None:
+            return None
+        values = dict()
+        values.update(m1.groupdict())
+        values.update(m2.groupdict())
+        return values
 
 class DictListElementConverter(Converter):
     def create_children(self, generalStore: GeneralStore,
@@ -316,14 +323,14 @@ class DictListElementConverter(Converter):
     def typecheck(self, element: StructureElement):
         return isinstance(element, DictListElement)
 
-    def match(self, element: StructureElement) -> Union[re.Match, Literal[True], None]:
+    def match(self, element: StructureElement):
         if not isinstance(element, DictListElement):
             raise RuntimeError("Element must be a DictListElement.")
         if self.name != element.name:
             return None
         if "match" in self.definition:
             raise NotImplementedError("Match is not implemented for DictListElement.")
-        return True
+        return dict()
 
 class TextElementConverter(Converter):
     def create_children(self, generalStore: GeneralStore,
@@ -333,8 +340,10 @@ class TextElementConverter(Converter):
     def typecheck(self, element: StructureElement):
         return isinstance(element, TextElement)
 
-    def match(self, element: StructureElement) -> Union[re.Match, Literal[True], None]:
+    def match(self, element: StructureElement):
         if not isinstance(element, TextElement):
             raise RuntimeError("Element must be a TextElement.")
         m = re.match(self.definition["match"], element.value)
-        return m
+        if m is None:
+            return None
+        return m.groupdict()

tests/scifolder_cfood.yml

@@ -29,12 +29,25 @@ DataAnalysis:  # name of the converter
               subtree:
                 description:
                   type: DictTextElement
-                  match: (?P<description>.*)
+                  match_value: (?P<description>.*)
+                  match_name: description
                   records:
                     Measurement:
                       description: $description
-                responsible:
+                responsible_single:
+                    type: DictTextElement
+                    match_name: responsible
+                    match_value: (?P<first_name>.+) (?P<last_name>.+)
+                    records:
+                      Person:
+                        first_name: $first_name
+                        last_name: $last_name
+                      Measurement:  # this uses the reference to the above defined record
+                        responsible: +$Person
+
+                responsible_list:
                   type: DictListElement
+                  match_name: responsible
                   subtree:
                     Person:
                       type: TextElement

tests/test_tool.py

@@ -5,7 +5,7 @@
 
 from newcrawler import Crawler
 from newcrawler.converters import MarkdownFileConverter
-from newcrawler.structure_elements import File
+from newcrawler.structure_elements import File, DictTextElement, DictListElement
 from os.path import join, dirname, basename
 import caosdb as db
 
@@ -72,16 +72,54 @@ def test_crawler():
     assert subd[1]["Measurement"].get_property("project").value == subd[0]["Project"]
 
 def test_markdown_converter():
+    test_readme = File("README.md", rfp(
+        "test_directories", "examples_article", "DataAnalysis",
+        "2020_climate-model-predict", "2020-02-08_prediction-errors", "README.md"))
+    
     converter = MarkdownFileConverter({
         "match": "(.*)"
     }, "TestMarkdownFileConverter")
 
-    m = converter.match(File("README.md", rfp(
-        "test_directories", "examples_article", "DataAnalysis",
-        "2020_climate-model-predict", "2020-02-08_prediction-errors", "README.md")))
-    assert m is not None
-    assert len(m.groups()) == 1
-
     m = converter.match(File("test_tool.py", rfp(
         "test_tool.py")))
     assert m is None
+
+    m = converter.match(test_readme)
+    assert m is not None
+    assert m.__class__ == dict
+    assert len(m) == 0
+
+    converter = MarkdownFileConverter({
+        "match": "README.md"
+    }, "TestMarkdownFileConverter")
+
+    m = converter.match(test_readme)
+    assert m is not None
+    assert len(m) == 0
+
+    children = converter.create_children(None, test_readme)
+    assert len(children) == 5
+    assert children[1].__class__ == DictTextElement
+    assert children[1].name == "description"
+    assert children[1].value.__class__ == str
+
+    assert children[0].__class__ == DictTextElement
+    assert children[0].name == "responsible"
+    assert children[0].value.__class__ == str
+
+    test_readme2 = File("README.md", rfp("test_directories", "examples_article", "ExperimentalData", "2020_SpeedOfLight", "2020-01-01_TimeOfFlight", "README.md"))
+
+    m = converter.match(test_readme2)
+    assert m is not None
+    assert len(m) == 0
+
+    children = converter.create_children(None, test_readme2)
+    assert len(children) == 2
+    assert children[1].__class__ == DictTextElement
+    assert children[1].name == "description"
+    assert children[1].value.__class__ == str
+
+    assert children[0].__class__ == DictListElement
+    assert children[0].name == "responsible"
+    assert children[0].value.__class__ == list
+