Skip to content
Snippets Groups Projects
Commit bbd6a022 authored by Henrik tom Wörden's avatar Henrik tom Wörden
Browse files

safer use of guard

parent e9cc1739
No related branches found
No related tags found
No related merge requests found
...@@ -38,6 +38,7 @@ import re ...@@ -38,6 +38,7 @@ import re
import caosdb as db import caosdb as db
from .guard import global_guard as guard
from .verbosity import INFO, VERBOSE from .verbosity import INFO, VERBOSE
ENTITIES = {} ENTITIES = {}
...@@ -62,8 +63,7 @@ class AbstractCFood(object): ...@@ -62,8 +63,7 @@ class AbstractCFood(object):
# function match() # function match()
_pattern = None _pattern = None
def __init__(self, crawled_file, access=lambda x: x, verbosity=INFO, def __init__(self, crawled_file, access=lambda x: x, verbosity=INFO):
guard=None):
""" Abstract base class for Crawler food (CFood). """ Abstract base class for Crawler food (CFood).
Parameters Parameters
...@@ -82,7 +82,6 @@ class AbstractCFood(object): ...@@ -82,7 +82,6 @@ class AbstractCFood(object):
self.identifiables = db.Container() self.identifiables = db.Container()
self.verbosity = verbosity self.verbosity = verbosity
self.attached_ones = [] self.attached_ones = []
self.guard = guard
@staticmethod @staticmethod
def get_re(): def get_re():
...@@ -169,7 +168,7 @@ class AbstractCFood(object): ...@@ -169,7 +168,7 @@ class AbstractCFood(object):
for el in self.to_be_updated: for el in self.to_be_updated:
print(el.name if el.name is not None else el.id) print(el.name if el.name is not None else el.id)
print("/"*60) print("/"*60)
self.guard.safe_update(self.to_be_updated) guard.safe_update(self.to_be_updated)
def attach(self, crawled_file): def attach(self, crawled_file):
self.attached_ones.append(crawled_file) self.attached_ones.append(crawled_file)
...@@ -229,15 +228,18 @@ def assure_object_is_in_list(obj, containing_object, property_name, ...@@ -229,15 +228,18 @@ def assure_object_is_in_list(obj, containing_object, property_name,
If obj is a list, every element is added If obj is a list, every element is added
""" """
if datatype is None:
datatype = db.LIST(property_name)
if containing_object.get_property(property_name) is None: if containing_object.get_property(property_name) is None:
if datatype is None:
datatype = db.LIST(property_name)
containing_object.add_property(property_name, value=[], containing_object.add_property(property_name, value=[],
datatype=datatype) datatype=datatype)
current_list = containing_object.get_property(property_name).value
if not isinstance(current_list, list): if not isinstance(containing_object.get_property(property_name).value, list):
current_list = [current_list] containing_object.get_property(property_name).value = [containing_object.get_property(property_name).value]
containing_object.get_property(property_name).value
containing_object.get_property(property_name).datatype = datatype
current_list = containing_object.get_property(property_name).value
if not isinstance(obj, list): if not isinstance(obj, list):
objects = [obj] objects = [obj]
...@@ -278,7 +280,7 @@ def assure_object_is_in_list(obj, containing_object, property_name, ...@@ -278,7 +280,7 @@ def assure_object_is_in_list(obj, containing_object, property_name,
to_be_updated.append(containing_object) to_be_updated.append(containing_object)
def assure_has_parent(entity, parent, to_be_updated=None, guard=None, verbosity=INFO): def assure_has_parent(entity, parent, to_be_updated=None, verbosity=INFO):
""" """
Checks whether `entity` has a parent with name `parent`. Checks whether `entity` has a parent with name `parent`.
...@@ -312,16 +314,13 @@ def assure_has_parent(entity, parent, to_be_updated=None, guard=None, verbosity= ...@@ -312,16 +314,13 @@ def assure_has_parent(entity, parent, to_be_updated=None, guard=None, verbosity=
if to_be_updated is None: if to_be_updated is None:
get_ids_for_entities_with_names([entity]) get_ids_for_entities_with_names([entity])
if guard is None: guard.safe_update(entity, unique=False)
entity.update(unique=False)
else:
guard.safe_update(entity, unique=False)
else: else:
to_be_updated.append(entity) to_be_updated.append(entity)
def assure_has_property(entity, name, value, to_be_updated=None, def assure_has_property(entity, name, value, to_be_updated=None,
verbosity=INFO, guard=None, datatype=None): verbosity=INFO, datatype=None):
""" """
Checks whether `entity` has a property `name` with the value `value`. Checks whether `entity` has a property `name` with the value `value`.
...@@ -345,8 +344,7 @@ def assure_has_property(entity, name, value, to_be_updated=None, ...@@ -345,8 +344,7 @@ def assure_has_property(entity, name, value, to_be_updated=None,
if to_be_updated is None: if to_be_updated is None:
get_ids_for_entities_with_names([entity]) get_ids_for_entities_with_names([entity])
if guard is None: guard.safe_update(entity, unique=False)
entity.update(unique=False)
return return
...@@ -384,10 +382,7 @@ def assure_has_property(entity, name, value, to_be_updated=None, ...@@ -384,10 +382,7 @@ def assure_has_property(entity, name, value, to_be_updated=None,
if to_be_updated is None: if to_be_updated is None:
get_ids_for_entities_with_names([entity]) get_ids_for_entities_with_names([entity])
if guard is None: guard.safe_update(entity, unique=False)
entity.update(unique=False)
else:
guard.safe_update(entity, unique=False)
else: else:
to_be_updated.append(entity) to_be_updated.append(entity)
...@@ -427,8 +422,8 @@ class CMeal(object): ...@@ -427,8 +422,8 @@ class CMeal(object):
for group in cls.matching_groups: for group in cls.matching_groups:
if (group not in match.groupdict() or if (group not in match.groupdict() or
group not in cfood.match.groupdict()or group not in cfood.match.groupdict() or
match.group(group) == cfood.match.group(group)): match.group(group) != cfood.match.group(group)):
suitable = False suitable = False
if suitable: if suitable:
......
...@@ -43,7 +43,8 @@ import caosdb as db ...@@ -43,7 +43,8 @@ import caosdb as db
from caosdb.exceptions import TransactionError from caosdb.exceptions import TransactionError
from .cache import Cache from .cache import Cache
from .guard import INSERT, RETRIEVE, UPDATE, Guard from .guard import INSERT, RETRIEVE, UPDATE
from .guard import global_guard as guard
from .verbosity import DEBUG, INFO, VERBOSE from .verbosity import DEBUG, INFO, VERBOSE
...@@ -76,7 +77,7 @@ class Crawler(object): ...@@ -76,7 +77,7 @@ class Crawler(object):
def crawl(self, files, security_level=RETRIEVE): def crawl(self, files, security_level=RETRIEVE):
errors_occured = False errors_occured = False
tbs = [] tbs = []
self.guard = Guard(level=security_level) guard.set_level(level=security_level)
files = sorted(files, key=lambda x: x.path) files = sorted(files, key=lambda x: x.path)
cfoods = [] cfoods = []
...@@ -102,9 +103,11 @@ class Crawler(object): ...@@ -102,9 +103,11 @@ class Crawler(object):
Cfood.__class__.__name__, Cfood.__class__.__name__,
crawled_file.path)) crawled_file.path))
try: try:
cfoods.append(Cfood.cook(crawled_file, access=self.access, cfood = Cfood.cook(crawled_file, access=self.access,
verbosity=self.verbosity, verbosity=self.verbosity)
guard=self.guard))
if cfood is not None:
cfoods.append(cfood)
except Exception as e: except Exception as e:
traceback.print_exc() traceback.print_exc()
print(e) print(e)
...@@ -161,7 +164,6 @@ class Crawler(object): ...@@ -161,7 +164,6 @@ class Crawler(object):
self.find_or_insert_identifiables(cfood.identifiables, self.find_or_insert_identifiables(cfood.identifiables,
self.verbosity, self.verbosity,
guard=self.guard
) )
if self.use_cache: if self.use_cache:
...@@ -190,7 +192,7 @@ class Crawler(object): ...@@ -190,7 +192,7 @@ class Crawler(object):
# TODO remove static? # TODO remove static?
@staticmethod @staticmethod
def find_or_insert_identifiables(identifiables, verbosity=INFO, guard=None): def find_or_insert_identifiables(identifiables, verbosity=INFO):
""" Sets the ids of identifiables (that do not have already an id from the """ Sets the ids of identifiables (that do not have already an id from the
cache) based on searching CaosDB and retrieves those entities. cache) based on searching CaosDB and retrieves those entities.
The remaining entities (those which can not be retrieved) have no The remaining entities (those which can not be retrieved) have no
......
...@@ -39,3 +39,9 @@ class Guard(object): ...@@ -39,3 +39,9 @@ class Guard(object):
raise Exception("not allowed") raise Exception("not allowed")
else: else:
obj.update(**kwargs) obj.update(**kwargs)
def set_level(self, level):
self.level = level
global_guard = Guard()
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment