values are updated correctly now in general store

src/newcrawler/crawl.py

@@ -61,6 +61,7 @@ example, a regular expresion could be used to get a date from a file name.
 import sys
 import os
 import yaml
+import re
 import argparse
 from abc import abstractmethod
 from argparse import RawTextHelpFormatter
@@ -92,14 +93,28 @@ class Store(object):
     def __init__(self):
         self.storage = dict()
 
+    def __getitem__(self, key):
+        return self.storage[key]
+
+    def update(self, other):
+        self.storage.update(other)
+
     def create_scoped_copy(self):
-        pass
+        s_copy = Store()
+        s_copy.storage = dict(self.storage)
+        return s_copy
 
 class GeneralStore(Store):
-    pass
+    def create_scoped_copy(self):
+        s_copy = GeneralStore()
+        s_copy.storage = dict(self.storage)
+        return s_copy
 
 class RecordStore(Store):
-    pass
+    def create_scoped_copy(self):
+        s_copy = RecordStore()
+        s_copy.storage = dict(self.storage)
+        return s_copy
 
 
 class Converter(object):
@@ -260,7 +275,8 @@ class DirectoryConverter(Converter):
     def create_values(self,
                       values: GeneralStore,
                       element: StructureElement):
-        pass
+        m = re.match(self.definition["match"], element.name)
+        values.update(m.groupdict())
 
     def create_children(self, generalStore: GeneralStore,
                         element: StructureElement):
@@ -275,12 +291,14 @@ class DirectoryConverter(Converter):
         pass
 
     def typecheck(self, element: StructureElement):
-        return isinstance(element, File)
+        return isinstance(element, Directory)
 
     def match(self, element: StructureElement):
-        pass
+        m = re.match(self.definition["match"], element.name)
+        return m is not None
 
-    def create_children_from_directory(self, element: Directory):
+    @staticmethod
+    def create_children_from_directory(element: Directory):
         """
         Creates a list of files (of type File) and directories (of type Directory) for a
         given directory. No recursion.
@@ -344,7 +362,7 @@ class Crawler(object):
             crawler_definition = yaml.load(f, Loader=yaml.SafeLoader)
 
         self.start_crawling(Directory(os.path.basename(dirname),
-                                      os.path.dirname(dirname)),
+                                      dirname),
                             crawler_definition)
 
 
@@ -365,9 +383,8 @@ class Crawler(object):
             self.converters.append(Converter.converter_factory(
                 value, key))
 
-        
-
-        self.crawl([item], self.converters, [], self.generalStore, self.recordStore)
+        self.crawl(DirectoryConverter.create_children_from_directory(item),
+                   self.converters, [], self.generalStore, self.recordStore)
 
 
     def crawl(self, items: list[StructureElement],

tests/test_tool.py

@@ -11,4 +11,4 @@ def test_crawler():
     crawler = Crawler([])
     crawler.crawl_directory(join(dirname(__file__), "test_directories/examples_article"),
               join(dirname(__file__), "scifolder_cfood.yml"))
-    assert True
+    assert crawler.generalStore["date"] == "test"