From 6dbc7f6a0f5d52b1297c506aed6dce21745c9ae1 Mon Sep 17 00:00:00 2001
From: Alexander Schlemmer <alexander.schlemmer@ds.mpg.de>
Date: Tue, 24 Nov 2020 11:55:23 +0100
Subject: [PATCH] ENH: added validate cache function and test case
---
src/caosadvancedtools/cache.py | 99 +++++++++++++++++++++++++---------
1 file changed, 75 insertions(+), 24 deletions(-)
diff --git a/src/caosadvancedtools/cache.py b/src/caosadvancedtools/cache.py
index 3a8b7626..5dc4f847 100644
--- a/src/caosadvancedtools/cache.py
+++ b/src/caosadvancedtools/cache.py
@@ -133,6 +133,25 @@ class AbstractCache(ABC):
return version_row[0][0]
finally:
conn.close()
+
+ def run_sql_commands(self, commands, fetchall=False):
+ """
+ Run a list of SQL commands on self.db_file.
+
+ commands: list of sql commands (tuples) to execute
+ fetchall: When True, run fetchall as last command and return the results.
+ Otherwise nothing is returned.
+ """
+ conn = sqlite3.connect(self.db_file)
+ c = conn.cursor()
+ for sql in commands:
+ c.execute(*sql)
+ if fetchall:
+ results = c.fetchall()
+ conn.commit()
+ conn.close()
+ if fetchall:
+ return results
# TODO: A better name would be IdentifiablesCache
@@ -163,14 +182,10 @@ class Cache(AbstractCache):
"""
conn = sqlite3.connect(self.db_file)
c = conn.cursor()
- c.execute(
- '''CREATE TABLE identifiables (digest TEXT PRIMARY KEY, caosdb_id INTEGER, caosdb_version TEXT)''')
- c.execute(
- '''CREATE TABLE version (schema INTEGER)''')
- c.execute("INSERT INTO version VALUES (?)", (self.get_cache_schema_version(),))
-
- conn.commit()
- conn.close()
+ self.run_sql_commands([
+ ('''CREATE TABLE identifiables (digest TEXT PRIMARY KEY, caosdb_id INTEGER, caosdb_version TEXT)''',),
+ ('''CREATE TABLE version (schema INTEGER)''',),
+ ("INSERT INTO version VALUES (?)", (self.get_cache_schema_version(),))])
@staticmethod
def hash_entity(ent):
@@ -190,12 +205,9 @@ class Cache(AbstractCache):
ent_id: ID of the entity
ent_version: Version string of the entity
"""
- conn = sqlite3.connect(self.db_file)
- c = conn.cursor()
- c.execute('''INSERT INTO identifiables VALUES (?, ?, ?)''',
- (ent_hash, ent_id, ent_version))
- conn.commit()
- conn.close()
+ self.run_sql_commands([
+ ('''INSERT INTO identifiables VALUES (?, ?, ?)''',
+ (ent_hash, ent_id, ent_version))])
def check_existing(self, ent_hash):
"""
@@ -206,18 +218,13 @@ class Cache(AbstractCache):
Return the ID and the version ID of the hashed entity.
Return None if no entity with that hash is in the cache.
"""
- conn = sqlite3.connect(self.db_file)
- c = conn.cursor()
- c.execute('''Select * FROM identifiables WHERE digest=?''',
- (ent_hash,))
- res = c.fetchone()
- conn.commit()
- conn.close()
+ res = self.run_sql_commands([('''Select * FROM identifiables WHERE digest=?''',
+ (ent_hash,))], True)
- if res is None:
- return res
+ if len(res) == 0:
+ return None
else:
- return res[1:]
+ return res[0][1:]
def update_ids_from_cache(self, entities):
""" sets ids of those entities that are in cache
@@ -253,6 +260,50 @@ class Cache(AbstractCache):
if self.check_existing(ehash) is None:
self.insert(ehash, ent.id, ent.version.id)
+ def validate_cache(self, entities=None):
+ """
+ Runs through all entities stored in the cache and checks
+ whether the version still matches the most recent version.
+ Non-matching entities will be removed
+ from the cache.
+
+ entities: When set to a db.Container or a list of Entities
+ the IDs from the cache will not be retrieved from the CaosDB database,
+ but the versions from the cache will be checked against the versions
+ contained in that collection. Only entries in the cache that have
+ a corresponding version in the collection will be checked, all others
+ will be ignored. Useful for testing.
+
+ Return a list of invalidated entries or an empty list if no elements have been invalidated.
+ """
+
+ res = self.run_sql_commands([(
+ "SELECT caosdb_id, caosdb_version FROM identifiables", ())], True)
+
+ if entities is None:
+ c = db.Container()
+ else:
+ c = entities
+ v = dict()
+ for c_id, c_version in res:
+ if entities is None:
+ c.append(db.Entity(id=c_id))
+ v[c_id] = c_version
+ if entities is None:
+ c.retrieve()
+
+
+ invalidate_list = []
+ for ent in c:
+ if ent.version.id != v[ent.id]:
+ invalidate_list.append(ent.id)
+
+ self.run_sql_commands([(
+ "DELETE FROM identifiables WHERE caosdb_id IN ({})".format(
+ ", ".join([str(caosdb_id) for caosdb_id in invalidate_list])), ())])
+
+ return invalidate_list
+
class UpdateCache(AbstractCache):
"""
--
GitLab