ENH: Don't let the crawler query all files

src/caosadvancedtools/cfood.py

@@ -51,6 +51,7 @@ ENTITIES = {}
 PROPERTIES = {}
 RECORDS = {}
 RECORDTYPES = {}
+FILES = {}
 
 logger = logging.getLogger(__name__)
 
@@ -244,11 +245,20 @@ class AbstractCFood(object, metaclass=ABCMeta):
             entity.add_property(prop, value)
 
 
+def add_files(filemap):
+    """add to the file cache"""
+    FILES.update(filemap)
+
+
 def get_entity_for_path(path):
+    if path in FILES:
+        return FILES[path]
     try:
         q = "FIND FILE WHICH IS STORED AT '{}'".format(path)
+        logger.debug(q)
+        FILES[path] = db.execute_query(q, unique=True)
 
-        return db.execute_query(q, unique=True)
+        return FILES[path]
     except EntityDoesNotExistError:
         path_prefix = "**"
 
@@ -257,7 +267,9 @@ def get_entity_for_path(path):
         q = "FIND FILE WHICH IS STORED AT '{}{}'".format(path_prefix, path)
         logger.debug(q)
 
-        return db.execute_query(q, unique=True)
+        FILES[path] = db.execute_query(q, unique=True)
+
+        return FILES[path]
 
 
 class AbstractFileCFood(AbstractCFood):
@@ -488,6 +500,7 @@ def assure_has_property(entity, name, value, to_be_updated=None,
 
             else:
                 to_be_updated.append(entity)
+
             return
 
     properties = entity.get_properties()

src/caosadvancedtools/crawler.py

@@ -50,7 +50,7 @@ import caosdb as db
 from caosdb.exceptions import TransactionError
 
 from .cache import Cache, UpdateCache, get_pretty_xml
-from .cfood import RowCFood, get_ids_for_entities_with_names
+from .cfood import RowCFood, add_files, get_ids_for_entities_with_names
 from .datainconsistency import DataInconsistencyError
 from .datamodel_problems import DataModelProblems
 from .guard import RETRIEVE, ProhibitedException
@@ -611,6 +611,7 @@ class FileCrawler(Crawler):
         """
         super().__init__(**kwargs)
         self.files = files
+        add_files({fi.path: fi for fi in files})
 
     def iteritems(self):
         for idx, p in enumerate(sorted([f.path for f in self.files])):

unittests/test_cfood.py

@@ -25,10 +25,11 @@ import re
 import unittest
 
 import caosdb as db
-from caosadvancedtools.cfood import (AbstractCFood, AbstractFileCFood,
-                                     CMeal, assure_has_parent,
-                                     assure_has_property,
-                                     assure_object_is_in_list)
+from caosadvancedtools.cfood import (AbstractCFood, AbstractFileCFood, CMeal,
+                                     assure_has_parent, assure_has_property,
+                                     assure_object_is_in_list,
+                                     get_entity_for_path)
+from caosadvancedtools.crawler import FileCrawler
 from caosadvancedtools.example_cfood import ExampleCFood
 
 PATTERN = "h.*"
@@ -143,6 +144,7 @@ class InsertionTest(unittest.TestCase):
             "Test bool": ("BOOLEAN", True),
             "Test string": ("STRING", "bla")
         }
+
         for name, ty_val in types_and_values.items():
             entity.add_property(name=name, datatype=ty_val[0],
                                 value=ty_val[1])
@@ -183,3 +185,10 @@ class MealTest(unittest.TestCase):
         self.assertTrue(c.looking_for("/this/other"))
         # class should still match other prefixes
         self.assertTrue(ExampleCFoodMeal.match_item("/that/file"))
+
+
+class FileCacheTest(unittest.TestCase):
+    def test(self):
+        self.assertRaises(Exception, get_entity_for_path, "/lol")
+        FileCrawler(cfood_types=[], files=[db.File(path="/lol")])
+        get_entity_for_path("/lol")