ENH: added validate cache function and test case

src/caosadvancedtools/cache.py

@@ -133,6 +133,25 @@ class AbstractCache(ABC):
             return version_row[0][0]
         finally:
             conn.close()
+
+    def run_sql_commands(self, commands, fetchall=False):
+        """
+        Run a list of SQL commands on self.db_file.
+        
+        commands: list of sql commands (tuples) to execute
+        fetchall: When True, run fetchall as last command and return the results.
+                  Otherwise nothing is returned.
+        """
+        conn = sqlite3.connect(self.db_file)
+        c = conn.cursor()
+        for sql in commands:
+            c.execute(*sql)
+        if fetchall:
+            results = c.fetchall()
+        conn.commit()
+        conn.close()
+        if fetchall:
+            return results
     
 
 # TODO: A better name would be IdentifiablesCache
@@ -163,14 +182,10 @@ class Cache(AbstractCache):
         """
         conn = sqlite3.connect(self.db_file)
         c = conn.cursor()
-        c.execute(
-            '''CREATE TABLE identifiables (digest TEXT PRIMARY KEY, caosdb_id INTEGER, caosdb_version TEXT)''')
-        c.execute(
-            '''CREATE TABLE version (schema INTEGER)''')
-        c.execute("INSERT INTO version VALUES (?)", (self.get_cache_schema_version(),))
-        
-        conn.commit()
-        conn.close()
+        self.run_sql_commands([
+            ('''CREATE TABLE identifiables (digest TEXT PRIMARY KEY, caosdb_id INTEGER, caosdb_version TEXT)''',),
+            ('''CREATE TABLE version (schema INTEGER)''',),
+            ("INSERT INTO version VALUES (?)", (self.get_cache_schema_version(),))])
 
     @staticmethod
     def hash_entity(ent):
@@ -190,12 +205,9 @@ class Cache(AbstractCache):
         ent_id: ID of the entity
         ent_version: Version string of the entity
         """
-        conn = sqlite3.connect(self.db_file)
-        c = conn.cursor()
-        c.execute('''INSERT INTO identifiables VALUES (?, ?, ?)''',
-                  (ent_hash, ent_id, ent_version))
-        conn.commit()
-        conn.close()
+        self.run_sql_commands([
+            ('''INSERT INTO identifiables VALUES (?, ?, ?)''',
+             (ent_hash, ent_id, ent_version))])
 
     def check_existing(self, ent_hash):
         """
@@ -206,18 +218,13 @@ class Cache(AbstractCache):
         Return the ID and the version ID of the hashed entity.
         Return None if no entity with that hash is in the cache.
         """
-        conn = sqlite3.connect(self.db_file)
-        c = conn.cursor()
-        c.execute('''Select * FROM identifiables WHERE digest=?''',
-                  (ent_hash,))
-        res = c.fetchone()
-        conn.commit()
-        conn.close()
+        res = self.run_sql_commands([('''Select * FROM identifiables WHERE digest=?''',
+                                      (ent_hash,))], True)
 
-        if res is None:
-            return res
+        if len(res) == 0:
+            return None
         else:
-            return res[1:]
+            return res[0][1:]
 
     def update_ids_from_cache(self, entities):
         """ sets ids of those entities that are in cache
@@ -253,6 +260,50 @@ class Cache(AbstractCache):
             if self.check_existing(ehash) is None:
                 self.insert(ehash, ent.id, ent.version.id)
 
+    def validate_cache(self, entities=None):
+        """
+        Runs through all entities stored in the cache and checks
+        whether the version still matches the most recent version.
+        Non-matching entities will be removed
+        from the cache.
+
+        entities: When set to a db.Container or a list of Entities
+                  the IDs from the cache will not be retrieved from the CaosDB database,
+                  but the versions from the cache will be checked against the versions
+                  contained in that collection. Only entries in the cache that have
+                  a corresponding version in the collection will be checked, all others
+                  will be ignored. Useful for testing.
+
+        Return a list of invalidated entries or an empty list if no elements have been invalidated.
+        """
+
+        res = self.run_sql_commands([(
+            "SELECT caosdb_id, caosdb_version FROM identifiables", ())], True)
+
+        if entities is None:
+            c = db.Container()
+        else:
+            c = entities
+        v = dict()
+        for c_id, c_version in res:
+            if entities is None:
+                c.append(db.Entity(id=c_id))
+            v[c_id] = c_version
+        if entities is None:
+            c.retrieve()
+        
+
+        invalidate_list = []
+        for ent in c:
+            if ent.version.id != v[ent.id]:
+                invalidate_list.append(ent.id)
+
+        self.run_sql_commands([(
+            "DELETE FROM identifiables WHERE caosdb_id IN ({})".format(
+                ", ".join([str(caosdb_id) for caosdb_id in invalidate_list])), ())])
+                
+        return invalidate_list
+
 
 class UpdateCache(AbstractCache):
     """