diff --git a/.docker/Dockerfile b/.docker/Dockerfile
index 876f252299991f2fa4410994b73259c3593c2198..5b26c03c927780e804f190ee6c8dee1e6813a18e 100644
--- a/.docker/Dockerfile
+++ b/.docker/Dockerfile
@@ -25,7 +25,7 @@ ADD https://gitlab.com/api/v4/projects/13656973/repository/branches/dev \
RUN git clone https://gitlab.com/caosdb/caosdb-pylib.git && \
cd caosdb-pylib && git checkout dev && pip3 install .
# At least recommonmark 0.6 required.
-RUN pip3 install -U html2text pycodestyle pylint recommonmark sphinx-rtd-theme
+RUN pip3 install -U html2text pycodestyle pylint recommonmark sphinx-rtd-theme gitignore-parser
COPY . /git
RUN rm -r /git/.git \
&& mv /git/.docker/pycaosdb.ini /git/integrationtests
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 73a9abb0b5a525ddb74ddbf33003b03e35c1cacf..68fb90fc027f1d706430358822b7aa8d5c4e2959 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -129,6 +129,7 @@ unittest:
image: $CI_REGISTRY_IMAGE
needs: [build-testenv]
script:
+ - python3 -c "import caosdb; print('CaosDB Version:', caosdb.__version__)"
- tox
# Build the sphinx documentation and make it ready for deployment by Gitlab Pages
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 99e9a1c0d06946c679f8fe1b32f573c3876b867c..b9a8c585cbe657f1ee4463ce908d3ea4bab36f25 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -4,12 +4,24 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+## [0.6.0] - 2022-10-11 ##
+(Florian Spreckelsen)
+
+### Added ###
+
+- Unauthorized inserts can now be cached. Note that the Crawler cannot postpone
+ inserts but the Cache has the functionality now.
+- caosdbignore; You can add one or more `.caosdbignore` files to the directory
+ structure that you want to make available in CaosDB and the run loadFiles.
+ The syntax is that of `.gitignore` files. For more information see `loadFiles`
+ section of the Crawler in the documentation.
+
## [0.5.0] - 2022-09-05 ##
(Florian Spreckelsen)
### Added ###
-- You can now use `python -m caosadvancedtools.models.parser model_file` to
+- You can now use `python -m caosadvancedtools.models.parser model_file` to
parse and potentially synchronize data models.
### Deprecated ###
diff --git a/README_SETUP.md b/README_SETUP.md
index 43047d554afbe8ffba11aef67b20dde44d29bdcf..daab4707679e30cfecf6f995dfb75f327304cb33 100644
--- a/README_SETUP.md
+++ b/README_SETUP.md
@@ -11,7 +11,7 @@ git clone 'https://gitlab.com/caosdb/caosdb-advanced-user-tools'
Dependencies will be installed automatically if you use the below described
procedure.
- `caosdb>=0.6.0`
-- `openpyxl>=3.0.0`
+- `openpyxl>=3.0.7`
- `xlrd>=1.2.0`
- `pandas>=1.2.0`
- `numpy>=1.17.3`
diff --git a/RELEASE_GUIDELINES.md b/RELEASE_GUIDELINES.md
index 7592b02d8084d3a5e6419ae66b61331026f2766c..8e4897e7e42c02e2c2620661c9f51e39630f6b50 100644
--- a/RELEASE_GUIDELINES.md
+++ b/RELEASE_GUIDELINES.md
@@ -41,3 +41,7 @@ guidelines of the CaosDB Project
11. After the merge of main to dev, start a new development version by
setting `ISRELEASED` to `False` and by increasing at least the `MICRO`
version in [setup.py](./setup.py) and preparing CHANGELOG.md.
+
+12. Create releases on gitlab.com and gitlab.indiscale.com that contain (at
+ least) the most recent section of the CHANGELOG as the description and link
+ to the PyPi package.
diff --git a/integrationtests/caosdbignore b/integrationtests/caosdbignore
new file mode 100644
index 0000000000000000000000000000000000000000..84b3636238e655e9ea7569ca3989f0ca73abb27c
--- /dev/null
+++ b/integrationtests/caosdbignore
@@ -0,0 +1,2 @@
+lol
+~README.md
diff --git a/integrationtests/extroot/Software/2010_TestSoftware/2019-02-03_v0.1/~README.md b/integrationtests/extroot/Software/2010_TestSoftware/2019-02-03_v0.1/~README.md
new file mode 100644
index 0000000000000000000000000000000000000000..f2e41136eac73c39554dede1fd7e67b12502d577
--- /dev/null
+++ b/integrationtests/extroot/Software/2010_TestSoftware/2019-02-03_v0.1/~README.md
@@ -0,0 +1 @@
+stuff
diff --git a/integrationtests/extroot/Software/2010_TestSoftware/lol b/integrationtests/extroot/Software/2010_TestSoftware/lol
new file mode 100644
index 0000000000000000000000000000000000000000..afd4a162a20a53aaeebf053a257f147759a52d98
--- /dev/null
+++ b/integrationtests/extroot/Software/2010_TestSoftware/lol
@@ -0,0 +1,2 @@
+lol
+
diff --git a/integrationtests/filldb.sh b/integrationtests/filldb.sh
index 9f55365eb595537b43caa9b197c8bc31ea1e69cb..9ee9060e5078b5d48ea5a9972b8ade119378dbc6 100755
--- a/integrationtests/filldb.sh
+++ b/integrationtests/filldb.sh
@@ -5,7 +5,7 @@ python3 -m caosadvancedtools.loadFiles /opt/caosdb/mnt/extroot/ExperimentalData
python3 -m caosadvancedtools.loadFiles /opt/caosdb/mnt/extroot/DataAnalysis
python3 -m caosadvancedtools.loadFiles /opt/caosdb/mnt/extroot/SimulationData
python3 -m caosadvancedtools.loadFiles /opt/caosdb/mnt/extroot/Publications
-python3 -m caosadvancedtools.loadFiles /opt/caosdb/mnt/extroot/Software
+python3 -m caosadvancedtools.loadFiles -c caosdbignore -l $PWD/extroot/Software /opt/caosdb/mnt/extroot/Software
python3 insert_model.py
python3 insert_some.py
python3 crawl.py /
diff --git a/integrationtests/test.sh b/integrationtests/test.sh
index 5bb013db6e70a3a8393e7e3b7c7993a6da6bf9b9..36730cc948d308659f01f6153f86a917ab1909d0 100755
--- a/integrationtests/test.sh
+++ b/integrationtests/test.sh
@@ -16,7 +16,8 @@ fi
OUT=/tmp/crawler.output
ls
cat pycaosdb.ini
-rm -rf cache.db
+python3 -c "import caosdb; print('CaosDB Version:', caosdb.__version__)"
+rm -rf /tmp/caosdb_identifiable_cache.db
set -e
echo "Clearing database"
python3 clear_database.py
diff --git a/integrationtests/test_crawler_with_cfoods.py b/integrationtests/test_crawler_with_cfoods.py
index 4efef87cef52e4a2a20a615afe210c32f52a276a..19b1f8ff10e365031d6940c7b904e3656eda2861 100755
--- a/integrationtests/test_crawler_with_cfoods.py
+++ b/integrationtests/test_crawler_with_cfoods.py
@@ -33,6 +33,15 @@ def get_entity_with_id(eid):
return db.execute_query("FIND "+str(eid), unique=True)
+class LoadFilesTest(unittest.TestCase):
+ def test_lol(self):
+ # check whether ignored files were insered
+ res = db.execute_query("FIND FILE WHICH IS STORED AT '**/lol'")
+ assert len(res) == 0
+ res = db.execute_query("FIND FILE WHICH IS STORED AT '**/~README.md'")
+ assert len(res) == 0
+
+
class CrawlerTest(unittest.TestCase):
def test_experiment(self):
diff --git a/setup.py b/setup.py
index e9e87e448818d6f8960f0429e548560f9b0aef54..1a1cbd953e0fabb08425b6d1972be301730aa744 100755
--- a/setup.py
+++ b/setup.py
@@ -46,7 +46,7 @@ from setuptools import find_packages, setup
########################################################################
MAJOR = 0
-MINOR = 5
+MINOR = 6
MICRO = 0
PRE = "" # e.g. rc0, alpha.1, 0.beta-23
ISRELEASED = True
@@ -157,11 +157,12 @@ def setup_package():
install_requires=["caosdb>=0.7.0",
"jsonschema>=4.4.0",
"numpy>=1.17.3",
- "openpyxl>=3.0.0",
+ "openpyxl>=3.0.7",
"pandas>=1.2.0",
"xlrd>=2.0",
],
extras_require={"h5-crawler": ["h5py>=3.3.0", ],
+ "gitignore-parser ": ["gitignore-parser >=0.1.0", ],
},
packages=find_packages('src'),
package_dir={'': 'src'},
diff --git a/src/caosadvancedtools/cache.py b/src/caosadvancedtools/cache.py
index ff807f2aba6210d643e675e7e3dd91d7c3b30906..db189b16e5755094ff0d6816aa0806b197b1e883 100644
--- a/src/caosadvancedtools/cache.py
+++ b/src/caosadvancedtools/cache.py
@@ -23,11 +23,14 @@
#
# ** end header
-# TODO this is implementing a cache on client side. Should it be on
-# server side?
+# Note: This is implementing a cache on client side. It would be great if the server would provide
+# something to replace this.
import os
import sqlite3
+from copy import deepcopy
+from abc import ABC, abstractmethod
from hashlib import sha256
+import warnings
import caosdb as db
from lxml import etree
@@ -64,59 +67,181 @@ def get_pretty_xml(cont):
return etree.tounicode(xml, pretty_print=True)
-class Cache(object):
- """
- stores identifiables (as a hash of xml) and their respective ID.
+class AbstractCache(ABC):
+ def __init__(self, db_file=None, force_creation=False):
+ """
+ db_file: The path of the database file.
- This allows to retrieve the Record corresponding to an indentifiable
- without querying.
- """
+ if force_creation is set to True, the file will be created
+ regardless of a file at the same path already exists.
+ """
- def __init__(self, db_file=None):
if db_file is None:
- self.db_file = "cache.db"
+ tmppath = tempfile.gettempdir()
+ self.db_file = os.path.join(tmppath, self.get_default_file_name())
else:
self.db_file = db_file
- if not os.path.exists(self.db_file):
+ if not os.path.exists(self.db_file) or force_creation:
self.create_cache()
+ else:
+ self.check_cache()
+
+ @abstractmethod
+ def get_cache_schema_version(self):
+ """
+ A method that has to be overloaded that sets the version of the
+ SQLITE database schema. The schema is saved in table version column schema.
+ Increase this variable, when changes to the cache tables are made.
+ """
+ pass
+
+ @abstractmethod
def create_cache(self):
+ """
+ Provide an overloaded function here that creates the cache in
+ the most recent version.
+ """
+ pass
+
+ @abstractmethod
+ def get_default_file_name(self):
+ """
+ Supply a default file name for the cache here.
+ """
+ pass
+
+ def check_cache(self):
+ """
+ Check whether the cache in db file self.db_file exists and conforms
+ to the latest database schema.
+
+ If it does not exist, it will be created using the newest database schema.
+
+ If it exists, but the schema is outdated, an exception will be raised.
+ """
+ try:
+ current_schema = self.get_cache_version()
+ except sqlite3.OperationalError:
+ current_schema = 1
+
+ if current_schema > self.get_cache_schema_version():
+ raise RuntimeError(
+ "Cache is corrupt or was created with a future version of this program.")
+ elif current_schema < self.get_cache_schema_version():
+ raise RuntimeError("Cache version too old. Please remove the current cache file:\n"
+ + self.db_file)
+
+ def get_cache_version(self):
+ """
+ Return the version of the cache stored in self.db_file.
+ The version is stored as the only entry in colum schema of table version.
+ """
+ try:
+ conn = sqlite3.connect(self.db_file)
+ c = conn.cursor()
+ c.execute("SELECT schema FROM version")
+ version_row = c.fetchall()
+
+ if len(version_row) != 1:
+ raise RuntimeError("Cache version table broken.")
+
+ return version_row[0][0]
+ finally:
+ conn.close()
+
+ def run_sql_commands(self, commands, fetchall=False):
+ """
+ Run a list of SQL commands on self.db_file.
+
+ commands: list of sql commands (tuples) to execute
+ fetchall: When True, run fetchall as last command and return the results.
+ Otherwise nothing is returned.
+ """
conn = sqlite3.connect(self.db_file)
c = conn.cursor()
- c.execute(
- '''CREATE TABLE identifiables (digest text primary key, caosdb_id integer)''')
+
+ for sql in commands:
+ c.execute(*sql)
+
+ if fetchall:
+ results = c.fetchall()
conn.commit()
conn.close()
+ if fetchall:
+ return results
+
+
+class IdentifiableCache(AbstractCache):
+ """
+ stores identifiables (as a hash of xml) and their respective ID.
+
+ This allows to retrieve the Record corresponding to an indentifiable
+ without querying.
+ """
+
+ def get_cache_schema_version(self):
+ return 2
+
+ def get_default_file_name(self):
+ return "caosdb_identifiable_cache.db"
+
+ def __init__(self, db_file=None, force_creation=False):
+ super().__init__(db_file, force_creation)
+
+ def create_cache(self):
+ """
+ Create a new SQLITE cache file in self.db_file.
+
+ Two tables will be created:
+ - identifiables is the actual cache.
+ - version is a table with version information about the cache.
+ """
+ self.run_sql_commands([
+ ('''CREATE TABLE identifiables (digest TEXT PRIMARY KEY, caosdb_id INTEGER, caosdb_version TEXT)''',),
+ ('''CREATE TABLE version (schema INTEGER)''',),
+ ("INSERT INTO version VALUES (?)", (self.get_cache_schema_version(),))])
+
@staticmethod
def hash_entity(ent):
- xml = get_pretty_xml(ent)
+ """
+ Format an entity as "pretty" XML and return the SHA256 hash.
+ """
+ xml = get_pretty_xml(deepcopy(ent))
digest = sha256(xml.encode("utf-8")).hexdigest()
return digest
- def insert(self, ent_hash, ent_id):
- conn = sqlite3.connect(self.db_file)
- c = conn.cursor()
- c.execute('''INSERT INTO identifiables VALUES (?, ?)''',
- (ent_hash, ent_id))
- conn.commit()
- conn.close()
+ def insert(self, ent_hash, ent_id, ent_version):
+ """
+ Insert a new cache entry.
+
+ ent_hash: Hash of the entity. Should be generated with Cache.hash_entity
+ ent_id: ID of the entity
+ ent_version: Version string of the entity
+ """
+ self.run_sql_commands([
+ ('''INSERT INTO identifiables VALUES (?, ?, ?)''',
+ (ent_hash, ent_id, ent_version))])
def check_existing(self, ent_hash):
- conn = sqlite3.connect(self.db_file)
- c = conn.cursor()
- c.execute('''Select * FROM identifiables WHERE digest=?''',
- (ent_hash,))
- res = c.fetchone()
- conn.commit()
- conn.close()
+ """
+ Check the cache for a hash.
+
+ ent_hash: The hash to search for.
+
+ Return the ID and the version ID of the hashed entity.
+ Return None if no entity with that hash is in the cache.
+ """
+ res = self.run_sql_commands([('''Select * FROM identifiables WHERE digest=?''',
+ (ent_hash,))], True)
- if res is None:
- return res
+ if len(res) == 0:
+ return None
else:
- return res[1]
+ return res[0][1:]
def update_ids_from_cache(self, entities):
""" sets ids of those entities that are in cache
@@ -131,7 +256,7 @@ class Cache(object):
eid = self.check_existing(ehash)
if eid is not None:
- ent.id = eid
+ ent.id = eid[0]
return hashes
@@ -141,25 +266,75 @@ class Cache(object):
The hashes must correspond to the entities in the list
"""
+ # Check whether all entities have IDs and versions:
+
+ for ent in entities:
+ if ent.id is None:
+ raise RuntimeError("Entity has no ID.")
+
+ if ent.version is None or ent.version.id is None:
+ raise RuntimeError("Entity has no version ID.")
+
for ehash, ent in zip(hashes, entities):
if self.check_existing(ehash) is None:
- self.insert(ehash, ent.id)
+ self.insert(ehash, ent.id, ent.version.id)
+
+ def validate_cache(self, entities=None):
+ """
+ Runs through all entities stored in the cache and checks
+ whether the version still matches the most recent version.
+ Non-matching entities will be removed from the cache.
+
+ entities: When set to a db.Container or a list of Entities
+ the IDs from the cache will not be retrieved from the CaosDB database,
+ but the versions from the cache will be checked against the versions
+ contained in that collection. Only entries in the cache that have
+ a corresponding version in the collection will be checked, all others
+ will be ignored. Useful for testing.
+
+ Return a list of invalidated entries or an empty list if no elements have been invalidated.
+ """
+ res = self.run_sql_commands([(
+ "SELECT caosdb_id, caosdb_version FROM identifiables", ())], True)
-class UpdateCache(Cache):
+ if entities is None:
+ # TODO this might become a problem. If many entities are cached,
+ # then all of them are retrieved here...
+ ids = [c_id for c_id, _ in res]
+ ids = set(ids)
+ entities = db.Container()
+ entities.extend([db.Entity(id=c_id) for c_id in ids])
+ entities.retrieve()
+
+ v = {c_id: c_version for c_id, c_version in res}
+
+ invalidate_list = []
+
+ for ent in entities:
+ if ent.version.id != v[ent.id]:
+ invalidate_list.append(ent.id)
+
+ self.run_sql_commands([(
+ "DELETE FROM identifiables WHERE caosdb_id IN ({})".format(
+ ", ".join([str(caosdb_id) for caosdb_id in invalidate_list])), ())])
+
+ return invalidate_list
+
+
+class UpdateCache(AbstractCache):
"""
- stores unauthorized updates
+ stores unauthorized inserts and updates
- If the Guard is set to a mode that does not allow an update, the update can
- be stored in this cache such that it can be authorized and done later.
+ If the Guard is set to a mode that does not allow an insert or update, the insert or update can
+ be stored in this cache such that it can be authorized and performed later.
"""
- def __init__(self, db_file=None):
- if db_file is None:
- tmppath = tempfile.gettempdir()
- tmpf = os.path.join(tmppath, "crawler_update_cache.db")
- db_file = tmpf
- super().__init__(db_file=db_file)
+ def get_cache_schema_version(self):
+ return 3
+
+ def get_default_file_name(self):
+ return "/tmp/crawler_update_cache.db"
@staticmethod
def get_previous_version(cont):
@@ -174,41 +349,65 @@ class UpdateCache(Cache):
return old_ones
- def insert(self, cont, run_id):
- """Insert a pending, unauthorized update
+ def insert(self, cont, run_id, insert=False):
+ """Insert a pending, unauthorized insert or update
Parameters
----------
- cont: Container with the records to be updated containing the desired
+ cont: Container with the records to be inserted or updated containing the desired
version, i.e. the state after the update.
run_id: int
The id of the crawler run
+ insert: bool
+ Whether the entities in the container shall be inserted or updated.
"""
cont = put_in_container(cont)
- old_ones = UpdateCache.get_previous_version(cont)
+
+ if insert:
+ old_ones = ""
+ else:
+ old_ones = UpdateCache.get_previous_version(cont)
new_ones = cont
- old_hash = Cache.hash_entity(old_ones)
+ if insert:
+ old_hash = ""
+ else:
+ old_hash = Cache.hash_entity(old_ones)
new_hash = Cache.hash_entity(new_ones)
- conn = sqlite3.connect(self.db_file)
- c = conn.cursor()
- c.execute('''INSERT INTO updates VALUES (?, ?, ?, ?, ?)''',
- (old_hash, new_hash, str(old_ones), str(new_ones),
- str(run_id)))
- conn.commit()
- conn.close()
+ self.run_sql_commands([('''INSERT INTO updates VALUES (?, ?, ?, ?, ?)''',
+ (old_hash, new_hash, str(old_ones), str(new_ones),
+ str(run_id)))])
def create_cache(self):
""" initialize the cache """
- conn = sqlite3.connect(self.db_file)
- c = conn.cursor()
- c.execute('''CREATE TABLE updates (olddigest text, newdigest text,
- oldrep text, newrep text, run_id text,
- primary key (olddigest, newdigest, run_id))''')
- conn.commit()
- conn.close()
+ self.run_sql_commands([
+ ('''CREATE TABLE updates (olddigest TEXT, newdigest TEXT, oldrep TEXT,
+ newrep TEXT, run_id TEXT, primary key (olddigest, newdigest, run_id))''',),
+ ('''CREATE TABLE version (schema INTEGER)''',),
+ ("INSERT INTO version VALUES (?)", (self.get_cache_schema_version(),))])
+
+ def get(self, run_id, querystring):
+ """ returns the pending updates for a given run id
+
+ Parameters:
+ -----------
+ run_id: the id of the crawler run
+ querystring: the sql query
+ """
+
+ return self.run_sql_commands([(querystring, (str(run_id),))], fetchall=True)
+
+ def get_inserts(self, run_id):
+ """ returns the pending updates for a given run id
+
+ Parameters:
+ -----------
+ run_id: the id of the crawler run
+ """
+
+ return self.get(run_id, '''Select * FROM updates WHERE olddigest='' AND run_id=?''')
def get_updates(self, run_id):
""" returns the pending updates for a given run id
@@ -218,12 +417,10 @@ class UpdateCache(Cache):
run_id: the id of the crawler run
"""
- conn = sqlite3.connect(self.db_file)
- c = conn.cursor()
- c.execute('''Select * FROM updates WHERE run_id=?''',
- (str(run_id),))
- res = c.fetchall()
- conn.commit()
- conn.close()
+ return self.get(run_id, '''Select * FROM updates WHERE olddigest!='' AND run_id=?''')
+
- return res
+class Cache(IdentifiableCache):
+ def __init__(self, *args, **kwargs):
+ warnings.warn(DeprecationWarning("This class is depricated. Please use IdentifiableCache."))
+ super().__init__(*args, **kwargs)
diff --git a/src/caosadvancedtools/crawler.py b/src/caosadvancedtools/crawler.py
index 099b8fd86656bd326c91e7754fa32a3d4ba76564..085cd8d27f261644b38061d26fb10e37ac5465fd 100644
--- a/src/caosadvancedtools/crawler.py
+++ b/src/caosadvancedtools/crawler.py
@@ -50,7 +50,7 @@ from sqlite3 import IntegrityError
import caosdb as db
from caosdb.exceptions import BadQueryError
-from .cache import Cache, UpdateCache, get_pretty_xml
+from .cache import IdentifiableCache, UpdateCache, get_pretty_xml
from .cfood import RowCFood, add_files, get_ids_for_entities_with_names
from .datainconsistency import DataInconsistencyError
from .datamodel_problems import DataModelProblems
@@ -190,7 +190,8 @@ class Crawler(object):
self.filterKnown.reset(cat)
if self.use_cache:
- self.cache = Cache(db_file=cache_file)
+ self.cache = IdentifiableCache(db_file=cache_file)
+ self.cache.validate_cache()
def iteritems(self):
""" generates items to be crawled with an index"""
@@ -208,28 +209,70 @@ class Crawler(object):
run_id: the id of the crawler run
"""
cache = UpdateCache()
+ inserts = cache.get_inserts(run_id)
+ all_inserts = 0
+ all_updates = 0
+ for _, _, _, new, _ in inserts:
+ new_cont = db.Container()
+ new_cont = new_cont.from_xml(new)
+ new_cont.insert(unique=False)
+ logger.info("Successfully inserted {} records!".format(len(new_cont)))
+ all_inserts += len(new_cont)
+ logger.info("Finished with authorized updates.")
+
changes = cache.get_updates(run_id)
for _, _, old, new, _ in changes:
- current = db.Container()
new_cont = db.Container()
new_cont = new_cont.from_xml(new)
+ ids = []
+ tmp = []
+ update_incomplete = False
+ # remove duplicate entities
+ for el in new_cont:
+ if el.id not in ids:
+ ids.append(el.id)
+ tmp.append(el)
+ else:
+ update_incomplete = True
+ new_cont = tmp
+ if new[0].version:
+ valids = db.Container()
+ nonvalids = db.Container()
+
+ for ent in new_cont:
+ remote_ent = db.Entity(id=ent.id).retrieve()
+ if ent.version == remote_ent.version:
+ valids.append(remote_ent)
+ else:
+ update_incomplete = True
+ nonvalids.append(remote_ent)
+ valids.update(unique=False)
+ logger.info("Successfully updated {} records!".format(
+ len(valids)))
+ logger.info("{} Records were not updated because the version in the server "
+ "changed!".format(len(nonvalids)))
+ all_updates += len(valids)
+ else:
+ current = db.Container()
- for ent in new_cont:
- current.append(db.execute_query("FIND {}".format(ent.id),
- unique=True))
- current_xml = get_pretty_xml(current)
+ for ent in new_cont:
+ current.append(db.Entity(id=ent.id).retrieve())
+ current_xml = get_pretty_xml(current)
- # check whether previous version equals current version
- # if not, the update must not be done
+ # check whether previous version equals current version
+ # if not, the update must not be done
- if current_xml != old:
- continue
+ if current_xml != old:
+ continue
- new_cont.update(unique=False)
- logger.info("Successfully updated {} records!".format(
- len(new_cont)))
+ new_cont.update(unique=False)
+ logger.info("Successfully updated {} records!".format(
+ len(new_cont)))
+ all_updates += len(new_cont)
+ logger.info("Some updates could not be applied. Crawler has to rerun.")
logger.info("Finished with authorized updates.")
+ return all_inserts, all_updates
def collect_cfoods(self):
"""
diff --git a/src/caosadvancedtools/loadFiles.py b/src/caosadvancedtools/loadFiles.py
index 445374b3b3bddf94eefd7952e190bf52155765a8..210732aa2bd31b7f493af263df24cc3e7a56611b 100755
--- a/src/caosadvancedtools/loadFiles.py
+++ b/src/caosadvancedtools/loadFiles.py
@@ -25,10 +25,14 @@
import argparse
import logging
+import os
import math
import sys
+import re
from argparse import ArgumentParser
+from tempfile import NamedTemporaryFile
+import shutil
import caosdb as db
logger = logging.getLogger(__name__)
@@ -46,37 +50,135 @@ def convert_size(size):
return '%s %s' % (s, size_name[i])
-def loadpath(path, include, exclude, prefix, dryrun, forceAllowSymlinks):
+def combine_ignore_files(caosdbignore, localignore, dirname=None):
+ """appends the contents of localignore to caosdbignore and saves the result
+ and returns the name
- if dryrun:
- logger.info("Performin a dryrun!")
- files = db.Container().retrieve(
- unique=False,
- raise_exception_on_error=True,
- flags={"InsertFilesInDir": ("-p " + prefix + " " if prefix else "")
- + ("-e " + exclude + " " if exclude else "")
- + ("-i " + include + " " if include else "")
- + ("--force-allow-symlinks " if forceAllowSymlinks else "")
- + path})
+ """
+
+ tmp = NamedTemporaryFile(delete=False, mode="w",
+ dir=dirname, prefix=".caosdbignore")
+ with open(caosdbignore, "r") as base:
+ tmp.write(base.read())
+ with open(localignore, "r") as local:
+ tmp.write(local.read())
+ tmp.close()
+ return tmp.name
+
+
+def compile_file_list(caosdbignore, localpath):
+ """creates a list of files that contain all files under localpath except
+ those excluded by caosdbignore
+
+ """
+
+ from gitignore_parser import parse_gitignore
+ matches = parse_gitignore(caosdbignore)
+ current_ignore = caosdbignore
+ non_ignored_files = []
+ ignore_files = []
+ for root, dirs, files in os.walk(localpath):
+ # remove local ignore files that do no longer apply to the current subtree (branch switch)
+ while len(ignore_files) > 0 and not root.startswith(ignore_files[-1][0]):
+ shutil.os.remove(ignore_files[-1][1])
+ ignore_files.pop()
+
+ # use the global one if there are no more local ones
+ if len(ignore_files) > 0:
+ current_ignore = ignore_files[-1][1]
+ matches = parse_gitignore(current_ignore)
+ else:
+ current_ignore = caosdbignore
+ matches = parse_gitignore(current_ignore)
+
+ # create a new local ignore file
+ if ".caosdbignore" in files:
+ current_ignore = combine_ignore_files(current_ignore,
+ os.path.join(
+ root, ".caosdbignore"),
+ # due to the logic of gitignore_parser the file
+ # has to be written to this folder
+ dirname=root)
+ ignore_files.append((root, current_ignore))
+ matches = parse_gitignore(current_ignore)
+
+ # actually append files that are not ignored
+ for fi in files:
+ fullpath = os.path.join(root, fi)
+ if not matches(fullpath):
+ non_ignored_files.append(fullpath)
+ return non_ignored_files
+
+
+def create_re_for_file_list(files, localroot, remoteroot):
+ """creates a regular expression that matches file paths contained in the
+ files argument and all parent directories. The prefix localroot is replaced
+ by the prefix remoteroot.
+
+ """
+ regexp = ""
+ for fi in files:
+ path = fi
+ reg = ""
+ while path != localroot and path != "/" and path != "":
+ print(path, localroot)
+ reg = "(/"+re.escape(os.path.basename(path)) + reg + ")?"
+ path = os.path.dirname(path)
+ regexp += "|" + re.escape(remoteroot) + reg
+ return "^("+regexp[1:]+")$"
+
+
+def loadpath(path, include, exclude, prefix, dryrun, forceAllowSymlinks, caosdbignore=None,
+ localpath=None):
+
+ if caosdbignore:
+ # create list of files and create regular expression for small chunks
+ filelist = compile_file_list(caosdbignore, localpath)
+ fulllist = filelist
+
+ index = 0
+ step_size = 3
+ includes = []
+ while index < len(fulllist):
+ subset = fulllist[index:min(index+step_size, len(fulllist))]
+ includes.append(create_re_for_file_list(subset, localpath, path))
+ index += step_size
else:
- # new files (inserting them using the insertFilesInDir feature of
- # the server, which inserts files via symlinks)
- files = db.Container().insert(
- unique=False,
- raise_exception_on_error=True,
- flags={"InsertFilesInDir": ("-p " + prefix + " " if prefix else "")
- + ("-e " + exclude + " " if exclude else "")
- + ("-i " + include + " " if include else "")
- + ("--force-allow-symlinks " if forceAllowSymlinks else "")
- + path})
+ includes = [include]
- totalsize = 0 # collecting total size of all new files
+ # if no caosdbignore file is used, this iterates over a single include
+ for include in includes:
+ if dryrun:
+ logger.info("Performin a dryrun!")
+ files = db.Container().retrieve(
+ unique=False,
+ raise_exception_on_error=True,
+ flags={"InsertFilesInDir": ("-p " + prefix + " " if prefix else "")
+ + ("-e " + exclude + " " if exclude else "")
+ + ("-i " + include + " " if include else "")
+ + ("--force-allow-symlinks " if forceAllowSymlinks else "")
+ + path})
+ else:
+ # new files (inserting them using the insertFilesInDir feature of
+ # the server, which inserts files via symlinks)
+ files = db.Container().insert(
+ unique=False,
+ raise_exception_on_error=True,
+ flags={"InsertFilesInDir": ("-p " + prefix + " " if prefix else "")
+ + ("-e " + exclude + " " if exclude else "")
+ + ("-i " + include + " " if include else "")
+ + ("--force-allow-symlinks " if forceAllowSymlinks else "")
+ + path})
- for f in files:
- totalsize += f.size
+ totalsize = 0 # collecting total size of all new files
- logger.info("Made in total {} new files with a combined size of {} "
- "accessible.".format(len(files), convert_size(totalsize)))
+ for f in files:
+ totalsize += f.size
+
+ print("Made in total {} new files with a combined size of {} "
+ "accessible.".format(len(files), convert_size(totalsize)))
+ logger.info("Made in total {} new files with a combined size of {} "
+ "accessible.".format(len(files), convert_size(totalsize)))
return
@@ -91,6 +193,18 @@ def main(argv=None):
# Setup argument parser
parser = ArgumentParser()
+ parser.add_argument("-p", "--prefix", dest="prefix",
+ help="store files with this prefix into the server's"
+ " file system.")
+ parser.add_argument("-c", "--caosdbignore", help="""
+Path to a caosdbignore file that defines which files shall be included and which do not.
+The syntax is the same as in a gitignore file. You must also provide the localpath option
+since the check is done locally.
+"""
+ )
+ parser.add_argument("-l", "--localpath", help="Path to the root directory on this machine. "
+ "This is needed if a caosdbignore file is used since the check is done "
+ "locally")
parser.add_argument("-i", "--include", dest="include",
help="""
only include paths matching this regex pattern.
@@ -104,9 +218,6 @@ exclude is given preference over include.
parser.add_argument("-e", "--exclude", dest="exclude",
help="exclude paths matching this regex pattern.",
metavar="RE")
- parser.add_argument("-p", "--prefix", dest="prefix",
- help="store files with this prefix into the server's"
- " file system.")
parser.add_argument("-d", "--dry-run", dest="dryrun", action="store_true",
help="Just simulate the insertion of the files.")
parser.add_argument('-t', '--timeout', dest="timeout",
@@ -127,6 +238,17 @@ exclude is given preference over include.
"directory tree.", action="store_true")
args = parser.parse_args()
+ if args.caosdbignore and (args.exclude or args.include):
+ raise ValueError(
+ "Do not use a caosdbignore file and in- or exclude simultaneously!")
+
+ if args.caosdbignore and not args.localpath:
+ raise ValueError("To use caosdbignore you must supply a local path!")
+
+ if args.localpath and (args.exclude or args.include):
+ raise ValueError(
+ "Do not use a localpath and in- or exclude simultaneously!")
+
con = db.get_connection()
con.timeout = float(args.timeout)
con._login()
@@ -137,8 +259,9 @@ exclude is given preference over include.
exclude=args.exclude,
prefix=args.prefix,
dryrun=args.dryrun,
-
forceAllowSymlinks=args.forceAllowSymlinks,
+ caosdbignore=args.caosdbignore,
+ localpath=args.localpath,
)
return 0
diff --git a/src/doc/conf.py b/src/doc/conf.py
index 9eb8c08ae5a75cb0561d20d858350b7f6db3001b..8302fd6f8087b7fc2c3b0352d975b3d340be3677 100644
--- a/src/doc/conf.py
+++ b/src/doc/conf.py
@@ -27,9 +27,9 @@ copyright = '2021, IndiScale GmbH'
author = 'Daniel Hornung'
# The short X.Y version
-version = '0.5.0'
+version = '0.6.0'
# The full version, including alpha/beta/rc tags
-release = '0.5.0'
+release = '0.6.0'
# -- General configuration ---------------------------------------------------
diff --git a/src/doc/crawler.rst b/src/doc/crawler.rst
index 4b99c97e6db16e5691f373fa5fb4903e4d078155..aea023192c0b95c784d5ac91ade12d0c30591d42 100644
--- a/src/doc/crawler.rst
+++ b/src/doc/crawler.rst
@@ -121,6 +121,14 @@ as seen by the CaosDB server (The actual path may vary. This is the used
in the LinkAhead distribution of CaosDB). In this case the root file
system as seen from within the CaosDB docker process is used.
+
+You can provide a ``.caosdbignore`` file as a commandline option to the above
+loadFiles command. The syntax of that file is the same as for `gitignore
+<https://git-scm.com/docs/gitignore>`_ files. Note, that you can have additional
+``.caosdbignore`` files at lower levels which are appended to the current ignore
+file and have an effect of the respective subtree.
+
+
Extending the Crawlers
======================
diff --git a/tox.ini b/tox.ini
index dde34b987b9b08bfdfc51a06dd46a9a0e0494f28..42ff0df986d179ce9770937dba4886f3b4e7ad0f 100644
--- a/tox.ini
+++ b/tox.ini
@@ -7,7 +7,8 @@ deps=nose
git+https://gitlab.indiscale.com/caosdb/src/caosdb-pylib.git@dev
pytest
pytest-cov
- openpyxl
+ gitignore-parser
+ openpyxl >= 3.0.7
xlrd == 1.2
h5py
commands=py.test --cov=caosadvancedtools -vv {posargs}
diff --git a/unittests/caosdbignore-example b/unittests/caosdbignore-example
new file mode 100644
index 0000000000000000000000000000000000000000..04500046def6f8d1a7efb1cdceeff7e85f8e1f91
--- /dev/null
+++ b/unittests/caosdbignore-example
@@ -0,0 +1,2 @@
+README.md
+.caosdbignore*
diff --git a/unittests/data/Publications/Posters/2019-02-03_something/.caosdbignore b/unittests/data/Publications/Posters/2019-02-03_something/.caosdbignore
new file mode 100644
index 0000000000000000000000000000000000000000..43b9cb201ed2ae7e193a1755d4e935a8d7b17a23
--- /dev/null
+++ b/unittests/data/Publications/Posters/2019-02-03_something/.caosdbignore
@@ -0,0 +1 @@
+!README.md
diff --git a/unittests/test_cache.py b/unittests/test_cache.py
index 2d7b863fe971dd61a575e24f52853de4f5c4e204..de3430bf2f28a6b05ea36b1047ac11937809ff44 100644
--- a/unittests/test_cache.py
+++ b/unittests/test_cache.py
@@ -24,31 +24,35 @@ import os
import unittest
from copy import deepcopy
from tempfile import NamedTemporaryFile
+import sqlite3
import caosdb as db
-from caosadvancedtools.cache import Cache, cleanXML
+from caosadvancedtools.cache import IdentifiableCache, cleanXML
from lxml import etree
+import pytest
+
class CacheTest(unittest.TestCase):
def setUp(self):
- self.cache = Cache(db_file=NamedTemporaryFile(delete=False).name)
- self.cache.create_cache()
+ self.cache = IdentifiableCache(db_file=NamedTemporaryFile(delete=False).name,
+ force_creation=True)
def test_hash(self):
ent = db.Record()
- assert isinstance(Cache.hash_entity(ent), str)
- assert (Cache.hash_entity(ent) !=
- Cache.hash_entity(db.Record().add_parent("lol")))
+ assert isinstance(IdentifiableCache.hash_entity(ent), str)
+ assert (IdentifiableCache.hash_entity(ent) !=
+ IdentifiableCache.hash_entity(db.Record().add_parent("lol")))
def test_insert(self):
ent = db.Record()
ent2 = db.Record()
ent2.add_parent(name="Experiment")
- ent_hash = Cache.hash_entity(ent)
- ent2_hash = Cache.hash_entity(ent2)
- self.cache.insert(ent2_hash, 1235)
- assert isinstance(self.cache.check_existing(ent2_hash), int)
+ ent_hash = IdentifiableCache.hash_entity(ent)
+ ent2_hash = IdentifiableCache.hash_entity(ent2)
+ self.cache.insert(ent2_hash, 1235, "ajkfljadsklf")
+ assert self.cache.check_existing(ent2_hash)[0] == 1235
+ assert self.cache.check_existing(ent2_hash)[1] == "ajkfljadsklf"
assert self.cache.check_existing(ent_hash) is None
def test_hirarchy(self):
@@ -64,17 +68,29 @@ class CacheTest(unittest.TestCase):
ent3 = db.Record()
ent3.add_parent(name="Analysis")
test_id = 2353243
- self.cache.insert(Cache.hash_entity(ent2), test_id)
+ self.cache.insert(IdentifiableCache.hash_entity(ent2), test_id, "ajdsklfjadslf")
entities = [ent, ent2, ent3]
hashes = self.cache.update_ids_from_cache(entities)
+ self.assertEqual(ent.id, None)
self.assertEqual(ent2.id, test_id)
+ self.assertEqual(ent3.id, None)
+
+ with pytest.raises(RuntimeError, match=r".*no ID.*"):
+ self.cache.insert_list(hashes, entities)
# test
ent.id = 1001
ent3.id = 1003
+ with pytest.raises(RuntimeError, match=r".*no version ID.*"):
+ self.cache.insert_list(hashes, entities)
+
+ ent.version = db.common.versioning.Version("jkadsjfldf")
+ ent2.version = db.common.versioning.Version("jkadsjfldf")
+ ent3.version = db.common.versioning.Version("jkadsjfldf")
+
self.cache.insert_list(hashes, entities)
- self.assertEqual(self.cache.check_existing(hashes[0]), 1001)
- self.assertEqual(self.cache.check_existing(hashes[2]), 1003)
+ self.assertEqual(self.cache.check_existing(hashes[0])[0], 1001)
+ self.assertEqual(self.cache.check_existing(hashes[2])[0], 1003)
def test_clean(self):
xml = etree.XML(
@@ -91,3 +107,138 @@ class CacheTest(unittest.TestCase):
""")
cleanXML(xml)
assert len(xml.findall('TransactionBenchmark')) == 0
+
+
+def create_sqlite_file(commands):
+ """
+ A temporary file will be used
+ commands: list of sql commands (tuples) to execute after creation
+ Name of the file is returned
+ """
+ db_file = NamedTemporaryFile(delete=False).name
+ conn = sqlite3.connect(db_file)
+ c = conn.cursor()
+ for sql in commands:
+ c.execute(*sql)
+ conn.commit()
+ conn.close()
+ return db_file
+
+
+class CacheTest2(unittest.TestCase):
+ """
+ Test the schema version.
+ """
+
+ def setUp(self):
+ # Correct version:
+ self.cache = IdentifiableCache(db_file=NamedTemporaryFile(delete=False).name,
+ force_creation=True)
+
+ self.db_file_defect = []
+ self.db_file_defect.extend([
+ # Version without version table (old version):
+ create_sqlite_file(
+ [('''CREATE TABLE identifiables (digest TEXT PRIMARY KEY, caosdb_id INTEGER)''',)]),
+ # Version with version table with wrong version:
+ create_sqlite_file(
+ [('''CREATE TABLE identifiables (digest TEXT PRIMARY KEY, caosdb_id INTEGER, caosdb_version TEXT)''',),
+ ('''CREATE TABLE version (schema INTEGER)''',),
+ ("INSERT INTO version VALUES (?)", (1,))]),
+ # Version with version table with wrong version:
+ create_sqlite_file(
+ [('''CREATE TABLE identifiables (digest TEXT PRIMARY KEY, caosdb_id INTEGER, caosdb_version TEXT)''',),
+ ('''CREATE TABLE version (schema INTEGER)''',),
+ ("INSERT INTO version VALUES (?)", (3,))]),
+ # Version with version table with missing version:
+ create_sqlite_file(
+ [('''CREATE TABLE identifiables (digest TEXT PRIMARY KEY, caosdb_id INTEGER, caosdb_version TEXT)''',),
+ ('''CREATE TABLE version (schema INTEGER)''',)]),
+ # Version with version table with too many versions:
+ create_sqlite_file(
+ [('''CREATE TABLE identifiables (digest TEXT PRIMARY KEY, caosdb_id INTEGER, caosdb_version TEXT)''',),
+ ('''CREATE TABLE version (schema INTEGER)''',),
+ ("INSERT INTO version VALUES (?)", (1,)),
+ ("INSERT INTO version VALUES (?)", (3,))])])
+
+ def test_schema(self):
+ # Test whether new cache is created correctly:
+ assert os.path.exists(self.cache.db_file)
+ # Test whether it can be opened
+ test_cache_2 = IdentifiableCache(db_file=self.cache.db_file)
+ assert test_cache_2.get_cache_version() == 2
+
+ with pytest.raises(RuntimeError, match="Cache version too old.") as e_info:
+ test_cache_2 = IdentifiableCache(db_file=self.db_file_defect[0])
+
+ with pytest.raises(RuntimeError, match="Cache version too old.") as e_info:
+ test_cache_2 = IdentifiableCache(db_file=self.db_file_defect[1])
+
+ with pytest.raises(RuntimeError, match=r".*future version.*") as e_info:
+ test_cache_2 = IdentifiableCache(db_file=self.db_file_defect[2])
+
+ with pytest.raises(RuntimeError, match=r".*table broken.*") as e_info:
+ test_cache_2 = IdentifiableCache(db_file=self.db_file_defect[3])
+
+ with pytest.raises(RuntimeError, match=r".*table broken.*") as e_info:
+ test_cache_2 = IdentifiableCache(db_file=self.db_file_defect[4])
+
+ def tearDown(self):
+ os.remove(self.cache.db_file)
+
+ for db_fn_defect in self.db_file_defect:
+ os.remove(db_fn_defect)
+
+
+class InvalidationTest(unittest.TestCase):
+ """
+ Test invalidation of cache entries.
+ """
+
+ def setUp(self):
+ # Correct version:
+ self.cache = IdentifiableCache(db_file=NamedTemporaryFile(delete=False).name,
+ force_creation=True)
+
+ def tearDown(self):
+ os.remove(self.cache.db_file)
+
+ def test_invalid(self):
+ ent = db.Record()
+ ent2 = db.Record()
+ ent2.add_parent(name="Experiment")
+ ent3 = db.Record()
+ ent3.add_parent(name="Analysis")
+ ent.id = 117
+ ent2.id = 328
+ ent3.id = 224
+
+ ent.version = db.common.versioning.Version("a")
+ ent2.version = db.common.versioning.Version("b")
+ ent3.version = db.common.versioning.Version("a")
+
+ el = [ent, ent2, ent3]
+
+ for e in el:
+ self.cache.insert(IdentifiableCache.hash_entity(e), e.id, e.version.id)
+
+ for e in el:
+ res = self.cache.check_existing(IdentifiableCache.hash_entity(e))
+ assert e.id == res[0]
+ assert e.version.id == res[1]
+
+ ent2.version.id = "c"
+ ent3.version.id = "b"
+
+ for e in el[1:]:
+ res = self.cache.check_existing(IdentifiableCache.hash_entity(e))
+ assert res is None
+
+ invalidated_entries = self.cache.validate_cache(el)
+ assert 328 in invalidated_entries
+ assert 224 in invalidated_entries
+ assert 117 not in invalidated_entries
+
+ res = self.cache.run_sql_commands([
+ ("SELECT * FROM identifiables", ())], fetchall=True)
+ assert len(res) == 1
diff --git a/unittests/test_caosdbignore.py b/unittests/test_caosdbignore.py
new file mode 100644
index 0000000000000000000000000000000000000000..9394bf0c8b177bc6df132ee0b43dcb2753d70f28
--- /dev/null
+++ b/unittests/test_caosdbignore.py
@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+# encoding: utf-8
+#
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2022 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2022 Henrik tom Wörden <h.tomwoerden@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+
+
+import os
+import re
+from tempfile import NamedTemporaryFile
+import unittest
+
+from caosadvancedtools.loadFiles import compile_file_list, create_re_for_file_list, combine_ignore_files
+
+BASEDIR = os.path.dirname(os.path.realpath(__file__))
+
+
+class Caosdbignore(unittest.TestCase):
+ def setUp(self):
+ pass
+
+ def test_compile(self):
+ files = compile_file_list(os.path.join(BASEDIR, "caosdbignore-example"),
+ os.path.join(BASEDIR, "data"))
+ assert len(files) == 3
+ assert os.path.join(BASEDIR, "data", "datatypes.xlsx") in files
+ assert os.path.join(BASEDIR, "data", "README.xlsx") in files
+ assert os.path.join(BASEDIR, "data", "Publications/Posters/2019-02-03_something/README.md") in files
+
+ def test_regex(self):
+ files = [r"/dies/ist/simple", r"/dies/eh(er)/nich?t"]
+ regex = create_re_for_file_list(files, "/dies", "/dies")
+ assert re.match(regex, files[0]) is not None
+ assert re.match(regex, files[1]) is not None
+ assert re.match(regex, "/dies/ist") is not None
+ assert re.match(regex, "/die") is None
+ assert re.match(regex, files[0]+files[1]) is None
+ assert re.match(regex, "d") is None
+
+ def test_combine(self):
+ fi1 = NamedTemporaryFile(delete=False, mode="w")
+ fi1.write("ha")
+ fi1.close()
+ fi2 = NamedTemporaryFile(delete=False, mode="w")
+ fi2.write("ha")
+ fi2.close()
+ fi_new = combine_ignore_files(fi1.name, fi2.name)
+ with open(fi_new, "r") as fi:
+ assert "haha" == fi.read()
diff --git a/unittests/test_cfood.py b/unittests/test_cfood.py
index 7055bc7c51962c0cbc487f29bcdacb391218a7d3..62e4b114a1c5cd8f3631f774637f3876c545afd3 100644
--- a/unittests/test_cfood.py
+++ b/unittests/test_cfood.py
@@ -336,6 +336,7 @@ class MealTest(unittest.TestCase):
class FileCacheTest(unittest.TestCase):
def test(self):
+ # if there is no connection to a server an exception should be raised.
self.assertRaises(Exception, get_entity_for_path, "/lol")
FileCrawler(cfood_types=[], files=[db.File(path="/lol")])
get_entity_for_path("/lol")
diff --git a/unittests/test_update_cache.py b/unittests/test_update_cache.py
index 4720f23de0b651b90e3b74ee13e06088462c5e31..8376da482b4828dd09de2ac6f3aca4fb9617c08d 100644
--- a/unittests/test_update_cache.py
+++ b/unittests/test_update_cache.py
@@ -42,8 +42,8 @@ class CacheTest(unittest.TestCase):
return c
def setUp(self):
- self.cache = UpdateCache(db_file=NamedTemporaryFile(delete=False).name)
- self.cache.create_cache()
+ self.cache = UpdateCache(db_file=NamedTemporaryFile(delete=False).name,
+ force_creation=True)
self.run_id = "235234"
def test_insert(self):