diff --git a/CHANGELOG.md b/CHANGELOG.md
index f03b7c4c8ea737af4f5afda6c31e6af0956db811..cf324dd1b18cb6a839df3a9010ddde7b080b0c7e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,18 +8,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased] ##
### Added ###
+- Identifiable class to represent the information used to identify Records.
+- Added some StructureElements: BooleanElement, FloatElement, IntegerElement,
+ ListElement, DictElement
### Changed ###
+- Some StructureElements changed (see "How to upgrade" in the docs):
+ - Dict, DictElement and DictDictElement were merged into DictElement.
+ - DictTextElement and TextElement were merged into TextElement. The "match"
+ keyword is now invalid for TextElements.
-- There are no DictXYElements any more. For example, DictTextElement is now
- simply a TextElement. The behavior of the following classes changed:
- - DictElementConverter (old: DictConverter) now can use "match" keywords. If
- none are in the definition, the behavior is as before.
- - TextElement used the 'match' keyword before, which was applied to the
- value. This is will in future be applied to the key instead and is now
- forbidden to used. Please use 'match_name' or 'match_value'.
### Deprecated ###
+- The DictXYElements are now depricated and are now synonyms for the
+ XYElements.
### Removed ###
diff --git a/README.md b/README.md
index 109285abd91898b810b9f7b95104358af9e12537..6c94473c066439b1645712c0046cd890b6b38715 100644
--- a/README.md
+++ b/README.md
@@ -1,29 +1,47 @@
-# caoscrawler
+# CaosDB-Crawler
-A new crawler for CaosDB.
+## Welcome
+This is the repository of the CaosDB-Crawler, a tool for automatic data
+insertion into [CaosDB](https://gitlab.com/caosdb/caosdb-meta).
-This package has yaml-header-tools as a dependency:
-https://gitlab.com/salexan/yaml-header-tools
+This is a new implementation resolving problems of the original implementation
+in [caosdb-advancedtools](https://gitlab.com/caosdb/caosdb-advanced-user-tools)
+## Setup
+Please read the [README_SETUP.md](README_SETUP.md) for instructions on how to
+setup this code.
-This python package can be installed using `pip`, e.g.:
-```bash
-pip install --user .
-```
-# Usage
+## Further Reading
-work in progress
+Please refer to the [official documentation](https://docs.indiscale.com/caosdb-crawler/) of the CaosDB-Crawler for more information.
-# Running the tests
+## Contributing
-After installation of the package run (within the project folder):
+Thank you very much to all contributers—[past, present](https://gitlab.com/caosdb/caosdb/-/blob/dev/HUMANS.md), and prospective ones.
+
+### Code of Conduct
+
+By participating, you are expected to uphold our [Code of Conduct](https://gitlab.com/caosdb/caosdb/-/blob/dev/CODE_OF_CONDUCT.md).
+
+### How to Contribute
+
+* You found a bug, have a question, or want to request a feature? Please
+[create an issue](https://gitlab.com/caosdb/caosdb-crawler).
+* You want to contribute code?
+ * **Forking:** Please fork the repository and create a merge request in GitLab and choose this repository as
+ target. Make sure to select "Allow commits from members who can merge the target branch" under
+ Contribution when creating the merge request. This allows our team to work with you on your
+ request.
+ * **Code style:** This project adhers to the PEP8 recommendations, you can test your code style
+ using the `autopep8` tool (`autopep8 -i -r ./`). Please write your doc strings following the
+ [NumpyDoc](https://numpydoc.readthedocs.io/en/latest/format.html) conventions.
+* You can also contact us at **info (AT) caosdb.de** and join the
+ CaosDB community on
+ [#caosdb:matrix.org](https://matrix.to/#/!unwwlTfOznjEnMMXxf:matrix.org).
-```bash
-pytest
-```
There is the file `unittests/records.xml` that servers as a dummy for a server state with files.
You can recreate this by uncommenting a section in `integrationtests/basic_example/test_basic.py`
@@ -33,7 +51,7 @@ and rerunning the integration test.
see `integrationtests/README.md`
-# Contributers
+## Contributers
The original authors of this package are:
@@ -41,10 +59,10 @@ The original authors of this package are:
- Henrik tom Wörden
- Florian Spreckelsen
-# License
+## License
-Copyright (C) 2021-2022 Research Group Biomedical Physics, Max Planck Institute for
-Dynamics and Self-Organization Göttingen.
+Copyright (C) 2021-2022 Research Group Biomedical Physics, Max Planck Institute
+ for Dynamics and Self-Organization Göttingen.
Copyright (C) 2021-2022 IndiScale GmbH
All files in this repository are licensed under a [GNU Affero General Public
diff --git a/README_SETUP.md b/README_SETUP.md
new file mode 120000
index 0000000000000000000000000000000000000000..d478016ecde09dab8820d398b15df325f4159380
--- /dev/null
+++ b/README_SETUP.md
@@ -0,0 +1 @@
+src/doc/README_SETUP.md
\ No newline at end of file
diff --git a/integrationtests/basic_example/test_basic.py b/integrationtests/basic_example/test_basic.py
index a4261cd92fec32b757f3abd210881d49e6a5e381..0c847b08a729f3b112cbdf3c38bac31309cda125 100755
--- a/integrationtests/basic_example/test_basic.py
+++ b/integrationtests/basic_example/test_basic.py
@@ -33,6 +33,7 @@ import argparse
import sys
from argparse import RawTextHelpFormatter
from caoscrawler import Crawler, SecurityMode
+from caoscrawler.identifiable import Identifiable
import caosdb as db
from caoscrawler.identifiable_adapters import CaosDBIdentifiableAdapter
import pytest
@@ -106,6 +107,15 @@ def crawler_extended(ident):
return cr
+def test_ambigious_lookup(clear_database, usemodel, crawler, ident):
+ ins, ups = crawler.synchronize()
+
+ proj = db.execute_query("FIND Project WITH identifier='SpeedOfLight'", unique=True)
+ with pytest.raises(RuntimeError, match=".*unambigiously.*"):
+ print(crawler.identifiableAdapter.retrieve_identified_record_for_identifiable(
+ Identifiable(properties={'project': proj.id})))
+
+
def test_single_insertion(clear_database, usemodel, crawler, ident):
ins, ups = crawler.synchronize()
diff --git a/src/caoscrawler/crawl.py b/src/caoscrawler/crawl.py
index bc482c311d4a38137236f1348b77d4d77efbbc17..e6b5f65cde7bd9159ab104f763597c3ded1c3903 100644
--- a/src/caoscrawler/crawl.py
+++ b/src/caoscrawler/crawl.py
@@ -56,6 +56,7 @@ from caosdb.apiutils import (compare_entities, EntityMergeConflictError,
from caosdb.common.datatype import is_reference
from .converters import Converter, DirectoryConverter
+from .identifiable import Identifiable
from .identifiable_adapters import (IdentifiableAdapter,
LocalStorageIdentifiableAdapter,
CaosDBIdentifiableAdapter)
@@ -511,21 +512,36 @@ class Crawler(object):
return self._synchronize(self.crawled_data, commit_changes, unique_names=unique_names)
- def has_reference_value_without_id(self, record: db.Record):
+ def _has_reference_value_without_id(self, ident: Identifiable) -> bool:
"""
- Returns True if there is at least one property in `record` which:
+ Returns True if there is at least one value in the properties attribute of ``ident`` which:
+
a) is a reference property AND
- b) where the value is set to a db.Entity (instead of an ID) AND
- c) where the ID of the value (the db.Entity object in b)) is not set (to an integer)
+ b) where the value is set to a
+ :external+caosdb-pylib:py:class:`db.Entity <caosdb.common.models.Entity>`
+ (instead of an ID) AND
+ c) where the ID of the value (the
+ :external+caosdb-pylib:py:class:`db.Entity <caosdb.common.models.Entity>` object in b))
+ is not set (to an integer)
+
+ Returns
+ -------
+ bool
+ True if there is a value without id (see above)
- Returns False otherwise.
+ Raises
+ ------
+ ValueError
+ If no Identifiable is given.
"""
- for p in record.properties:
- if isinstance(p.value, list):
- for el in p.value:
+ if ident is None:
+ raise ValueError("Identifiable has to be given as argument")
+ for pname, pvalue in ident.properties.items():
+ if isinstance(pvalue, list):
+ for el in pvalue:
if isinstance(el, db.Entity) and el.id is None:
return True
- elif isinstance(p.value, db.Entity) and p.value.id is None:
+ elif isinstance(pvalue, db.Entity) and pvalue.id is None:
return True
return False
@@ -552,23 +568,25 @@ class Crawler(object):
flat.append(p.value)
Crawler.create_flat_list([p.value], flat)
- def has_missing_object_in_references(self, record: db.Record):
+ def _has_missing_object_in_references(self, ident: Identifiable):
"""
- returns False if any property value is a db.Entity object that
- is contained in the `remote_missing_cache`. If the record has such an object in the
- reference properties, it means that it references another Entity, where we checked
+ returns False if any value in the properties attribute is a db.Entity object that
+ is contained in the `remote_missing_cache`. If ident has such an object in
+ properties, it means that it references another Entity, where we checked
whether it exists remotely and it was not found.
"""
- for p in record.properties:
+ if ident is None:
+ raise ValueError("Identifiable has to be given as argument")
+ for pname, pvalue in ident.properties.items():
# if (is_reference(p)
# Entity instead of ID and not cached locally
- if (isinstance(p.value, list)):
- for el in p.value:
- if (isinstance(el, db.Entity)
- and self.get_from_remote_missing_cache(el) is not None):
+ if (isinstance(pvalue, list)):
+ for el in pvalue:
+ if (isinstance(el, db.Entity) and self.get_from_remote_missing_cache(
+ self.identifiableAdapter.get_identifiable(el)) is not None):
return True
- if (isinstance(p.value, db.Entity)
- and self.get_from_remote_missing_cache(p.value) is not None):
+ if (isinstance(pvalue, db.Entity) and self.get_from_remote_missing_cache(
+ self.identifiableAdapter.get_identifiable(pvalue)) is not None):
# might be checked when reference is resolved
return True
return False
@@ -585,7 +603,7 @@ class Crawler(object):
for el in p.value:
if (isinstance(el, db.Entity) and el.id is None):
cached = self.get_from_any_cache(
- el)
+ self.identifiableAdapter.get_identifiable(el))
if cached is None:
raise RuntimeError("Not in cache.")
if not check_identical(cached, el, True):
@@ -599,7 +617,8 @@ class Crawler(object):
lst.append(el)
p.value = lst
if (isinstance(p.value, db.Entity) and p.value.id is None):
- cached = self.get_from_any_cache(p.value)
+ cached = self.get_from_any_cache(
+ self.identifiableAdapter.get_identifiable(p.value))
if cached is None:
raise RuntimeError("Not in cache.")
if not check_identical(cached, p.value, True):
@@ -610,38 +629,28 @@ class Crawler(object):
raise RuntimeError("Not identical.")
p.value = cached
- def get_from_remote_missing_cache(self, record: db.Record):
+ def get_from_remote_missing_cache(self, identifiable: Identifiable):
"""
- returns the identifiable if an identifiable with the same values already exists locally
+ returns the identified record if an identifiable with the same values already exists locally
(Each identifiable that is not found on the remote server, is 'cached' locally to prevent
that the same identifiable exists twice)
"""
- if self.identifiableAdapter is None:
- raise RuntimeError("Should not happen.")
- identifiable = self.identifiableAdapter.get_identifiable(record)
if identifiable is None:
- # TODO: check whether the same idea as below works here
- identifiable = record
- # return None
+ raise ValueError("Identifiable has to be given as argument")
if identifiable in self.remote_missing_cache:
return self.remote_missing_cache[identifiable]
else:
return None
- def get_from_any_cache(self, record: db.Record):
+ def get_from_any_cache(self, identifiable: Identifiable):
"""
returns the identifiable if an identifiable with the same values already exists locally
(Each identifiable that is not found on the remote server, is 'cached' locally to prevent
that the same identifiable exists twice)
"""
- if self.identifiableAdapter is None:
- raise RuntimeError("Should not happen.")
- identifiable = self.identifiableAdapter.get_identifiable(record)
if identifiable is None:
- # TODO: check whether the same idea as below works here
- identifiable = record
- # return None
+ raise ValueError("Identifiable has to be given as argument")
if identifiable in self.remote_existing_cache:
return self.remote_existing_cache[identifiable]
@@ -652,54 +661,31 @@ class Crawler(object):
def add_to_remote_missing_cache(self, record: db.Record):
"""
- adds the given identifiable to the local cache
+ stores the given Record in the remote_missing_cache.
- No identifiable with the same values must exist locally.
- (Each identifiable that is not found on the remote server, is 'cached' locally to prevent
- that the same identifiable exists twice)
-
- Return False if there is no identifiable for this record and True otherwise.
+ If no identifiable can be created for the given Record, the Record is NOT stored.
"""
self.add_to_cache(record=record, cache=self.remote_missing_cache)
def add_to_remote_existing_cache(self, record: db.Record):
"""
- adds the given identifiable to the local cache
-
- No identifiable with the same values must exist locally.
- (Each identifiable that is not found on the remote server, is 'cached' locally to prevent
- that the same identifiable exists twice)
+ stores the given Record in the remote_existing_cache.
- Return False if there is no identifiable for this record and True otherwise.
+ If no identifiable can be created for the given Record, the Record is NOT stored.
"""
self.add_to_cache(record=record, cache=self.remote_existing_cache)
- def add_to_cache(self, record: db.Record, cache):
+ def add_to_cache(self, record: db.Record, cache) -> Union[Identifiable, None]:
"""
- adds the given identifiable to the local cache
-
- No identifiable with the same values must exist locally.
- (Each identifiable that is not found on the remote server, is 'cached' locally to prevent
- that the same identifiable exists twice)
+ stores the given Record in the given cache.
- Return False if there is no identifiable for this record and True otherwise.
+ If no identifiable can be created for the given Record, the Record is NOT stored.
"""
if self.identifiableAdapter is None:
raise RuntimeError("Should not happen.")
identifiable = self.identifiableAdapter.get_identifiable(record)
- if identifiable is None:
- # TODO: this error report is bad
- # we need appropriate handling for records without an identifiable
- # or at least a simple fallback definition if tehre is no identifiable.
-
- # print(record)
- # raise RuntimeError("No identifiable for record.")
-
- # TODO: check whether that holds:
- # if there is no identifiable, for the cache that is the same
- # as if the complete entity is the identifiable:
- identifiable = record
- cache.add(identifiable=identifiable, record=record)
+ if identifiable is not None:
+ cache.add(identifiable=identifiable, record=record)
@staticmethod
def bend_references_to_new_object(old, new, entities):
@@ -736,17 +722,48 @@ class Crawler(object):
while resolved_references and len(flat) > 0:
resolved_references = False
+ # For each element we try to find out whether we can find it in the server or whether
+ # it does not yet exist. Since a Record may reference other unkown Records it might not
+ # be possible to answer this right away.
+ # The following checks are done on each Record:
+ # 1. Can it be identified via an ID?
+ # 2. Can it be identified via a path?
+ # 3. Is it in the cache of already checked Records?
+ # 4. Can it be checked on the remote server?
+ # 5. Does it have to be new since a needed reference is missing?
for i in reversed(range(len(flat))):
record = flat[i]
+ identifiable = self.identifiableAdapter.get_identifiable(record)
# TODO remove if the exception is never raised
- if (record.id is not None or record in to_be_inserted):
+ if record in to_be_inserted:
raise RuntimeError("This should not be reached since treated elements"
"are removed from the list")
- # Check whether this record is a duplicate that can be removed
- elif self.get_from_any_cache(record) is not None:
+ # 1. Can it be identified via an ID?
+ elif record.id is not None:
+ to_be_updated.append(record)
+ self.add_to_remote_existing_cache(record)
+ del flat[i]
+ # 2. Can it be identified via a path?
+ elif record.path is not None:
+ existing = self._get_entity_by_path(record.path)
+ if existing is None:
+ to_be_inserted.append(record)
+ self.add_to_remote_missing_cache(record)
+ del flat[i]
+ else:
+ record.id = existing.id
+ # TODO check the following copying of _size and _checksum
+ # Copy over checksum and size too if it is a file
+ record._size = existing._size
+ record._checksum = existing._checksum
+ to_be_updated.append(record)
+ self.add_to_remote_existing_cache(record)
+ del flat[i]
+ # 3. Is it in the cache of already checked Records?
+ elif self.get_from_any_cache(identifiable) is not None:
# We merge the two in order to prevent loss of information
- newrecord = self.get_from_any_cache(record)
+ newrecord = self.get_from_any_cache(identifiable)
try:
merge_entities(newrecord, record)
except EntityMergeConflictError:
@@ -757,13 +774,11 @@ class Crawler(object):
del flat[i]
resolved_references = True
- # can we check whether the record(identifiable) exists on the remote server?
- elif not self.has_reference_value_without_id(
- self.identifiableAdapter.get_identifiable(record)):
- # TODO: remove deepcopy?
+ # 4. Can it be checked on the remote server?
+ elif not self._has_reference_value_without_id(identifiable):
identified_record = (
- self.identifiableAdapter.retrieve_identified_record_for_record(
- deepcopy(record)))
+ self.identifiableAdapter.retrieve_identified_record_for_identifiable(
+ identifiable))
if identified_record is None:
# identifiable does not exist remotely -> record needs to be inserted
to_be_inserted.append(record)
@@ -772,20 +787,15 @@ class Crawler(object):
else:
# side effect
record.id = identified_record.id
- # Copy over checksum and size too if it is a file
- if isinstance(record, db.File):
- record._size = identified_record._size
- record._checksum = identified_record._checksum
-
to_be_updated.append(record)
self.add_to_remote_existing_cache(record)
del flat[i]
resolved_references = True
- # is it impossible to check this record because an identifiable references a
- # missing record?
- elif self.has_missing_object_in_references(
- self.identifiableAdapter.get_identifiable(record)):
+ # 5. Does it have to be new since a needed reference is missing?
+ # (Is it impossible to check this record because an identifiable references a
+ # missing record?)
+ elif self._has_missing_object_in_references(identifiable):
to_be_inserted.append(record)
self.add_to_remote_missing_cache(record)
del flat[i]
@@ -923,6 +933,13 @@ class Crawler(object):
def _get_entity_by_name(name):
return db.Entity(name=name).retrieve()
+ @staticmethod
+ def _get_entity_by_path(path):
+ try:
+ return db.execute_query(f"FIND FILE WHICH IS STORED AT '{path}'", unique=True)
+ except db.exceptions.EmptyUniqueQueryError:
+ return None
+
@staticmethod
def _get_entity_by_id(id):
return db.Entity(id=id).retrieve()
@@ -997,8 +1014,7 @@ class Crawler(object):
self.replace_entities_with_ids(el)
identified_records = [
- self.identifiableAdapter.retrieve_identified_record_for_record(
- record)
+ self.identifiableAdapter.retrieve_identified_record_for_record(record)
for record in to_be_updated]
# Merge with existing data to prevent unwanted overwrites
to_be_updated = self._merge_properties_from_remote(to_be_updated,
diff --git a/src/caoscrawler/identifiable.py b/src/caoscrawler/identifiable.py
new file mode 100644
index 0000000000000000000000000000000000000000..5b2b215263b0edc7817c18ad9eecf85ad648f10f
--- /dev/null
+++ b/src/caoscrawler/identifiable.py
@@ -0,0 +1,129 @@
+#!/usr/bin/env python3
+# encoding: utf-8
+#
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2022 Henrik tom Wörden
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+
+from __future__ import annotations
+import caosdb as db
+from datetime import datetime
+from hashlib import sha256
+from typing import Union
+
+
+class Identifiable():
+ """
+ The fingerprint of a Record in CaosDB.
+
+ This class contains the information that is used by the CaosDB Crawler to identify Records.
+ On one hand, this can be the ID or a Record or the path of a File.
+ On the other hand, in order to check whether a Record exits in the CaosDB Server, a query can
+ be created using the information contained in the Identifiable.
+
+ Parameters
+ ----------
+ record_type: str, this RecordType has to be a parent of the identified object
+ name: str, the name of the identified object
+ properties: dict, keys are names of Properties; values are Property values
+ Note, that lists are not checked for equality but are interpreted as multiple
+ conditions for a single Property.
+ path: str, In case of files: The path where the file is stored.
+ backrefs: list, TODO future
+ """
+
+ def __init__(self, record_id: int = None, path: str = None, record_type: str = None,
+ name: str = None, properties: dict = None,
+ backrefs: list[Union[int, str]] = None):
+ if (record_id is None and path is None and name is None and backrefs is None and (
+ properties is None or len(properties) == 0)):
+ raise ValueError("There is no identifying information. You need to add a path or "
+ "properties or other identifying attributes.")
+ if properties is not None and 'name' in [k.lower() for k in properties.keys()]:
+ raise ValueError("Please use the separete 'name' keyword instead of the properties "
+ "dict for name")
+ self.record_id = record_id
+ self.path = path
+ self.record_type = record_type
+ self.name = name
+ self.properties: dict = {}
+ if properties is not None:
+ self.properties = properties
+ self.backrefs: list = []
+ if backrefs is not None:
+ self.backrefs = backrefs
+
+ def get_representation(self) -> str:
+ return sha256(Identifiable._create_hashable_string(self).encode('utf-8')).hexdigest()
+
+ @staticmethod
+ def _value_representation(value) -> str:
+ """returns the string representation of property values to be used in the hash function
+
+ The string is the path of a File Entity, the CaosDB ID or Python ID of other Entities
+ (Python Id only if there is no CaosDB ID) and the string representation of bool, float, int
+ and str.
+ """
+
+ if value is None:
+ return "None"
+ elif isinstance(value, db.File):
+ return str(value.path)
+ elif isinstance(value, db.Entity):
+ if value.id is not None:
+ return str(value.id)
+ else:
+ return "PyID=" + str(id(value))
+ elif isinstance(value, list):
+ return "[" + ", ".join([Identifiable._value_representation(el) for el in value]) + "]"
+ elif (isinstance(value, str) or isinstance(value, int) or isinstance(value, float)
+ or isinstance(value, datetime)):
+ return str(value)
+ else:
+ raise ValueError(f"Unknown datatype of the value: {value}")
+
+ @staticmethod
+ def _create_hashable_string(identifiable: Identifiable) -> str:
+ """
+ creates a string from the attributes of an identifiable that can be hashed
+ String has the form "P<parent>N<name>a:5b:10"
+ """
+ rec_string = "P<{}>N<{}>".format(identifiable.record_type, identifiable.name)
+ # TODO this structure neglects Properties if multiple exist for the same name
+ for pname in sorted(identifiable.properties.keys()):
+ rec_string += ("{}:".format(pname) +
+ Identifiable._value_representation(identifiable.properties[pname]))
+ return rec_string
+
+ def __eq__(self, other) -> bool:
+ """
+ Identifiables are equal if they belong to the same Record. Since ID and path are on their
+ own enough to identify the Record it is sufficient if those attributes are equal.
+ 1. both IDs are set (not None) -> equal if IDs are equal
+ 2. both paths are set (not None) -> equal if paths are equal
+ 3. equal if attribute representations are equal
+ """
+ if not isinstance(other, Identifiable):
+ raise ValueError("Identifiable can only be compared to other Identifiable objects.")
+ elif self.record_id is not None and other.record_id is not None:
+ return self.record_id == other.record_id
+ elif self.path is not None and other.path is not None:
+ return self.path == other.path
+ elif self.get_representation() == other.get_representation():
+ return True
+ else:
+ return False
diff --git a/src/caoscrawler/identifiable_adapters.py b/src/caoscrawler/identifiable_adapters.py
index 1c6da34759a76c7a2272ff115239af9f989a2c24..0abeab6252e031bf8493a1ad04dc960f70de4e4c 100644
--- a/src/caoscrawler/identifiable_adapters.py
+++ b/src/caoscrawler/identifiable_adapters.py
@@ -26,6 +26,7 @@
import yaml
from datetime import datetime
+from .identifiable import Identifiable
import caosdb as db
import logging
from abc import abstractmethod, ABCMeta
@@ -86,7 +87,7 @@ class IdentifiableAdapter(metaclass=ABCMeta):
"""
@staticmethod
- def create_query_for_identifiable(ident: db.Record):
+ def create_query_for_identifiable(ident: Identifiable):
"""
This function is taken from the old crawler:
caosdb-advanced-user-tools/src/caosadvancedtools/crawler.py
@@ -94,35 +95,29 @@ class IdentifiableAdapter(metaclass=ABCMeta):
uses the properties of ident to create a query that can determine
whether the required record already exists.
"""
+ query_string = "FIND Record "
+ if ident.record_type is not None:
+ query_string += ident.record_type
- if len(ident.parents) != 1:
- raise RuntimeError(
- "Multiple parents for identifiables not supported.")
-
- query_string = "FIND Record " + ident.get_parents()[0].name
query_string += " WITH "
- if ident.name is None and len(ident.get_properties()) == 0:
- raise ValueError(
- "The identifiable must have features to identify it.")
-
if ident.name is not None:
query_string += "name='{}'".format(ident.name)
- if len(ident.get_properties()) > 0:
+ if len(ident.properties) > 0:
query_string += " AND "
query_string += IdentifiableAdapter.create_property_query(ident)
return query_string
@staticmethod
- def create_property_query(entity: db.Entity):
+ def create_property_query(entity: Identifiable):
query_string = ""
- for p in entity.get_properties():
- if p.value is None:
- query_string += "'" + p.name + "' IS NULL AND "
- elif isinstance(p.value, list):
- for v in p.value:
- query_string += ("'" + p.name + "'='" +
+ for pname, pvalue in entity.properties.items():
+ if pvalue is None:
+ query_string += "'" + pname + "' IS NULL AND "
+ elif isinstance(pvalue, list):
+ for v in pvalue:
+ query_string += ("'" + pname + "'='" +
convert_value(v) + "' AND ")
# TODO: (for review)
@@ -136,8 +131,8 @@ class IdentifiableAdapter(metaclass=ABCMeta):
# IdentifiableAdapter.create_property_query(p.value) +
# ") AND ")
else:
- query_string += ("'" + p.name + "'='" +
- convert_value(p.value) + "' AND ")
+ query_string += ("'" + pname + "'='" +
+ convert_value(pvalue) + "' AND ")
# remove the last AND
return query_string[:-4]
@@ -160,79 +155,61 @@ class IdentifiableAdapter(metaclass=ABCMeta):
"""
pass
- def get_identifiable_for_file(self, record: db.File):
- """
- Retrieve an identifiable for a file.
-
- Currently an identifiable for a file ist just a File object
- with a specific path. In the future, this could be extended
- to allow for names, parents and custom properties.
- """
- identifiable = db.File()
- identifiable.path = record.path
- return identifiable
-
def get_identifiable(self, record: db.Record):
"""
retrieve the registred identifiable and fill the property values to create an
identifiable
"""
- if record.role == "File":
- return self.get_identifiable_for_file(record)
-
registered_identifiable = self.get_registered_identifiable(record)
- if registered_identifiable is None:
- return None
-
- identifiable = db.Record(name=record.name)
- if len(registered_identifiable.parents) != 1:
- raise RuntimeError("Multiple parents for identifiables"
- "not supported.")
- identifiable.add_parent(registered_identifiable.parents[0])
property_name_list_A = []
property_name_list_B = []
-
- # fill the values:
- for prop in registered_identifiable.properties:
- if prop.name == "name":
- # The name can be an identifiable, but it isn't a property
- continue
- # problem: what happens with multi properties?
- # case A: in the registered identifiable
- # case B: in the identifiable
-
- record_prop = record.get_property(prop.name)
- if record_prop is None:
- # TODO: how to handle missing values in identifiables
- # raise an exception?
- raise NotImplementedError(
- f"The following record is missing an identifying property:"
- f"RECORD\n{record}\nIdentifying PROPERTY\n{prop.name}"
- )
- newval = record_prop.value
- record_prop_new = db.Property(name=record_prop.name,
- id=record_prop.id,
- description=record_prop.description,
- datatype=record_prop.datatype,
- value=newval,
- unit=record_prop.unit)
- identifiable.add_property(record_prop_new)
- property_name_list_A.append(prop.name)
-
- # check for multi properties in the record:
- for prop in property_name_list_A:
- property_name_list_B.append(prop)
- if (len(set(property_name_list_B)) != len(property_name_list_B) or len(
- set(property_name_list_A)) != len(property_name_list_A)):
- raise RuntimeError(
- "Multi properties used in identifiables can cause unpredictable results.")
-
- return identifiable
+ identifiable_props = {}
+
+ if registered_identifiable is not None:
+ # fill the values:
+ for prop in registered_identifiable.properties:
+ if prop.name == "name":
+ # The name can be an identifiable, but it isn't a property
+ continue
+ # problem: what happens with multi properties?
+ # case A: in the registered identifiable
+ # case B: in the identifiable
+
+ record_prop = record.get_property(prop.name)
+ if record_prop is None:
+ # TODO: how to handle missing values in identifiables
+ # raise an exception?
+ raise NotImplementedError(
+ f"The following record is missing an identifying property:"
+ f"RECORD\n{record}\nIdentifying PROPERTY\n{prop.name}"
+ )
+ identifiable_props[record_prop.name] = record_prop.value
+ property_name_list_A.append(prop.name)
+
+ # check for multi properties in the record:
+ for prop in property_name_list_A:
+ property_name_list_B.append(prop)
+ if (len(set(property_name_list_B)) != len(property_name_list_B) or len(
+ set(property_name_list_A)) != len(property_name_list_A)):
+ raise RuntimeError(
+ "Multi properties used in identifiables could cause unpredictable results and "
+ "are not allowed. You might want to consider a Property with a list as value.")
+
+ # use the RecordType of the registred Identifiable if it exists
+ # We do not use parents of Record because it might have multiple
+ return Identifiable(
+ record_id=record.id,
+ record_type=(registered_identifiable.parents[0].name
+ if registered_identifiable else None),
+ name=record.name,
+ properties=identifiable_props,
+ path=record.path
+ )
@abstractmethod
- def retrieve_identified_record_for_identifiable(self, identifiable: db.Record):
+ def retrieve_identified_record_for_identifiable(self, identifiable: Identifiable):
"""
Retrieve identifiable record for a given identifiable.
@@ -256,10 +233,7 @@ class IdentifiableAdapter(metaclass=ABCMeta):
"""
identifiable = self.get_identifiable(record)
- if identifiable is None:
- return None
-
- if identifiable.role == "File":
+ if identifiable.path is not None:
return self.get_file(identifiable)
return self.retrieve_identified_record_for_identifiable(identifiable)
@@ -280,7 +254,7 @@ class LocalStorageIdentifiableAdapter(IdentifiableAdapter):
def get_records(self):
return self._records
- def get_file(self, identifiable: db.File):
+ def get_file(self, identifiable: Identifiable):
"""
Just look in records for a file with the same path.
"""
@@ -338,7 +312,7 @@ class LocalStorageIdentifiableAdapter(IdentifiableAdapter):
return None
return identifiable_candidates[0]
- def check_record(self, record: db.Record, identifiable: db.Record):
+ def check_record(self, record: db.Record, identifiable: Identifiable):
"""
Check for a record from the local storage (named "record") if it is
the identified record for an identifiable which was created by
@@ -348,13 +322,11 @@ class LocalStorageIdentifiableAdapter(IdentifiableAdapter):
record is the record from the local database to check against.
identifiable is the record that was created during the crawler run.
"""
- if len(identifiable.parents) != 1:
- raise RuntimeError(
- "Multiple parents for identifiables not supported.")
- if not has_parent(record, identifiable.parents[0].name):
+ if (identifiable.record_type is not None
+ and not has_parent(record, identifiable.record_type)):
return False
- for prop in identifiable.properties:
- prop_record = record.get_property(prop.name)
+ for propname, propvalue in identifiable.properties.items():
+ prop_record = record.get_property(propname)
if prop_record is None:
return False
@@ -362,18 +334,18 @@ class LocalStorageIdentifiableAdapter(IdentifiableAdapter):
# there are two different cases:
# a) prop_record.value has a registered identifiable:
# in this case, fetch the identifiable and set the value accordingly
- if isinstance(prop.value, db.Entity): # lists are not checked here
+ if isinstance(propvalue, db.Entity): # lists are not checked here
otherid = prop_record.value
if isinstance(prop_record.value, db.Entity):
otherid = prop_record.value.id
- if prop.value.id != otherid:
+ if propvalue.id != otherid:
return False
- elif prop.value != prop_record.value:
+ elif propvalue != prop_record.value:
return False
return True
- def retrieve_identified_record_for_identifiable(self, identifiable: db.Record):
+ def retrieve_identified_record_for_identifiable(self, identifiable: Identifiable):
candidates = []
for record in self._records:
if self.check_record(record, identifiable):
@@ -426,7 +398,7 @@ class CaosDBIdentifiableAdapter(IdentifiableAdapter):
def register_identifiable(self, name: str, definition: db.RecordType):
self._registered_identifiables[name] = definition
- def get_file(self, identifiable: db.File):
+ def get_file(self, identifiable: Identifiable):
if identifiable.path is None:
raise RuntimeError("Path must not be None for File retrieval.")
candidates = db.execute_query("FIND File which is stored at {}".format(
@@ -444,6 +416,9 @@ class CaosDBIdentifiableAdapter(IdentifiableAdapter):
It is assumed, that there is exactly one identifiable for each RecordType. Only the first
parent of the given Record is considered; others are ignored
"""
+ if len(record.parents) == 0:
+ return None
+ # TODO We need to treat the case where multiple parents exist properly.
rt_name = record.parents[0].name
for name, definition in self._registered_identifiables.items():
if definition.parents[0].name.lower() == rt_name.lower():
@@ -458,7 +433,7 @@ class CaosDBIdentifiableAdapter(IdentifiableAdapter):
return record
return record.id
- def retrieve_identified_record_for_identifiable(self, identifiable: db.Record):
+ def retrieve_identified_record_for_identifiable(self, identifiable: Identifiable):
query_string = self.create_query_for_identifiable(identifiable)
candidates = db.execute_query(query_string)
if len(candidates) > 1:
diff --git a/src/caoscrawler/identified_cache.py b/src/caoscrawler/identified_cache.py
index b5db8ba04fc813ea8aca0131c6265adab666b2e7..aa2d82f8e66c738e737c62f3cc68eaf60127e28b 100644
--- a/src/caoscrawler/identified_cache.py
+++ b/src/caoscrawler/identified_cache.py
@@ -25,87 +25,41 @@
"""
-This module is a cache for Records where we checked the existence in a remote server using
-identifiables. If the Record was found, this means that we identified the corresponding Record
-in the remote server and the ID of the local object can be set.
-To prevent querying the server again and again for the same objects, this cache allows storing
-Records that were found on a remote server and those that were not (typically in separate caches).
-The look up in the cache is done using a hash of a string representation.
-
-TODO: We need a general review:
-- How are entities identified with each other?
-- What happens if the identification fails?
-
-Checkout how this was done in the old crawler.
+see class docstring
"""
+from .identifiable import Identifiable
import caosdb as db
-from hashlib import sha256
-from datetime import datetime
-
-
-def _value_representation(value):
- """returns the string representation of property values to be used in the hash function """
-
- # TODO: (for review)
- # This expansion of the hash function was introduced recently
- # to allow the special case of Files as values of properties.
- # We need to review the completeness of all the cases here, as the cache
- # is crucial for correct identification of insertion and updates.
- if value is None:
- return "None"
- elif isinstance(value, db.File):
- return str(value.path)
- elif isinstance(value, db.Entity):
- if value.id is not None:
- return str(value.id)
- else:
- return "PyID="+str(id(value))
- elif isinstance(value, list):
- return "["+", ".join([_value_representation(el) for el in value])+"]"
- elif (isinstance(value, str) or isinstance(value, int) or isinstance(value, float)
- or isinstance(value, datetime)):
- return str(value)
- else:
- raise ValueError(f"Unknown datatype of the value: {value}")
-
-def _create_hashable_string(identifiable: db.Record):
+class IdentifiedCache(object):
"""
- creates a string from the attributes of an identifiable that can be hashed
+ This class is like a dictionary where the keys are Identifiables. When you check whether an
+ Identifiable exists as key this class returns True not only if that exact Python object is
+ used as a key, but if an Identifiable is used as key that is **equal** to the one being
+ considered (see __eq__ function of Identifiable). Similarly, if you do `cache[identifiable]`
+ you get the Record where the key is an Identifiable that is equal to the one in the rectangular
+ brackets.
+
+ This class is used for Records where we checked the existence in a remote server using
+ identifiables. If the Record was found, this means that we identified the corresponding Record
+ in the remote server and the ID of the local object can be set.
+ To prevent querying the server again and again for the same objects, this cache allows storing
+ Records that were found on a remote server and those that were not (typically in separate
+ caches).
"""
- if identifiable.role == "File":
- # Special treatment for files:
- return "P<>N<>{}:{}".format("path", identifiable.path)
- if len(identifiable.parents) != 1:
- # TODO: extend this
- # maybe something like this:
- # parent_names = ",".join(
- # sorted([p.name for p in identifiable.parents])
- raise RuntimeError("Cache entry can only be generated for entities with 1 parent.")
- rec_string = "P<{}>N<{}>".format(identifiable.parents[0].name, identifiable.name)
- # TODO this structure neglects Properties if multiple exist for the same name
- for pname in sorted([p.name for p in identifiable.properties]):
-
- rec_string += ("{}:".format(pname) +
- _value_representation(identifiable.get_property(pname).value))
- return rec_string
-
-def _create_hash(identifiable: db.Record) -> str:
- return sha256(_create_hashable_string(identifiable).encode('utf-8')).hexdigest()
-
-
-class IdentifiedCache(object):
def __init__(self):
self._cache = {}
+ self._identifiables = []
- def __contains__(self, identifiable: db.Record):
- return _create_hash(identifiable) in self._cache
+ def __contains__(self, identifiable: Identifiable):
+ return identifiable in self._identifiables
def __getitem__(self, identifiable: db.Record):
- return self._cache[_create_hash(identifiable)]
+ index = self._identifiables.index(identifiable)
+ return self._cache[id(self._identifiables[index])]
- def add(self, record: db.Record, identifiable: db.Record):
- self._cache[_create_hash(identifiable)] = record
+ def add(self, record: db.Record, identifiable: Identifiable):
+ self._cache[id(identifiable)] = record
+ self._identifiables.append(identifiable)
diff --git a/src/doc/README_SETUP.md b/src/doc/README_SETUP.md
index b6995c9a2d950ecd1e832d5b49dac9ed88a7e455..1f6e15d408e10e38bce0d9b9fe9b6197ec69bfc3 100644
--- a/src/doc/README_SETUP.md
+++ b/src/doc/README_SETUP.md
@@ -2,9 +2,6 @@
## Installation ##
-### Requirements ###
-
-
### How to install ###
#### Linux ####
@@ -59,17 +56,12 @@ pip3 install --user .
**Note**: In the near future, this package will also be made available on PyPi.
-## Configuration ##
-
-
-
-## Try it out ##
-
-
## Run Unit Tests
## Documentation ##
+We use sphinx to create the documentation. Docstrings in the code should comply
+with the Googly style (see link below).
Build documentation in `src/doc` with `make html`.
@@ -79,4 +71,10 @@ Build documentation in `src/doc` with `make html`.
- `sphinx-autoapi`
- `recommonmark`
-### Troubleshooting ###
+### How to contribute ###
+
+- [Google Style Python Docstrings](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html)
+- [Google Style Python Docstrings 2nd reference](https://github.com/google/styleguide/blob/gh-pages/pyguide.md#38-comments-and-docstrings)
+- [References to other documentation](https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html#role-external)
+
+
diff --git a/src/doc/concepts.rst b/src/doc/concepts.rst
index c0f21cbaa322caddabed8e045f7b6fc4253d2959..012f393bca2caf1f0107ea462fdc531a1a8c799e 100644
--- a/src/doc/concepts.rst
+++ b/src/doc/concepts.rst
@@ -11,7 +11,8 @@ of the existing data (For example could a tree of Python file objects
(StructureElements) represent a file tree that exists on some file server).
Relevant sources in:
-src/structure_elements.py
+
+- ``src/structure_elements.py``
Converters
++++++++++
@@ -22,20 +23,54 @@ the above named tree. The definition of a Converter also contains what
Converters shall be used to treat the generated child-StructureElements. The
definition is therefore a tree itself.
-See `:doc:converters<converters>` for details.
+See :std:doc:`converters<converters>` for details.
Relevant sources in:
-src/converters.py
+- ``src/converters.py``
Identifiables
+++++++++++++
-Relevant sources in:
-src/identifiable_adapters.py
+An Identifiable of a Record is like the fingerprint of a Record.
+
+The identifiable contains the information that is used by the CaosDB Crawler to identify Records.
+In order to check whether a Record exits in the CaosDB Server, the CaosDB Crawler creates a query
+using the information contained in the Identifiable.
+
+For example, suppose a certain experiment is at most done once per day, then the identifiable could
+consist of the RecordType "SomeExperiment" (as a parent) and the Property "date".
+
+You can think of the properties that are used by the identifiable as a dictionary. For each property
+name there can be one value. However, this value can be a list such that the created query can look
+like "FIND RECORD ParamenterSet WITH a=5 AND a=6". This is meaningful if there is a ParamenterSet
+with two Properties with the name 'a' (multi property) or if 'a' is a list containing at least the values 5 and 6.
+
+The path of a File object can serve as a Property that identifies files and similarly the name of
+Records can be used.
+
+In the current implementation an identifiable can only use one RecordType even though the identified Records might have multiple
+Parents.
+
+Relevant sources in
+
+- ``src/identifiable_adapters.py``
+- ``src/identifiable.py``
+
+RegisteredIdentifiables
++++++++++++++++++++++++
+A Registered Identifiable is the blue print for Identifiables.
+You can think of registered identifiables as identifiables without concrete values for properties.
+RegisteredIdentifiables are
+associated with RecordTypes and define of what information an identifiable for that RecordType
+exists. There can be multiple Registered Identifiables for one RecordType.
+
+Identified Records
+++++++++++++++++++
+TODO
The Crawler
+++++++++++
@@ -45,7 +80,8 @@ The crawler can be considered the main program doing the synchronization in basi
#. Compare the current state of the CaosDB instance with the set of CaosDB Entities created in step 1, taking into account the :ref:`registered identifiables<Identifiables>`. Insert or update entites accordingly.
Relevant sources in:
-src/crawl.py
+
+- ``src/crawl.py``
diff --git a/src/doc/how-to-upgrade.md b/src/doc/how-to-upgrade.md
new file mode 100644
index 0000000000000000000000000000000000000000..56298e695bc4aa7ee83e407fffd39a5d0d8c21f5
--- /dev/null
+++ b/src/doc/how-to-upgrade.md
@@ -0,0 +1,16 @@
+
+# How to upgrade
+
+## 0.2.x to 0.3.0
+DictElementConverter (old: DictConverter) now can use "match" keywords. If
+none are in the definition, the behavior is as before. If you had "match",
+"match_name" or "match_value" in the definition of a
+DictConverter (StructureElement: Dict) before,
+you probably want to remove those. They were ignored before and are now used.
+
+TextElement used the 'match' keyword before, which was applied to the
+value. This is will in future be applied to the key instead and is now
+forbidden to used. If you used the 'match'
+keyword in the definition of TextElementConverter
+(StructureElement: TextElement) before, you need to change the key from "match"
+to "match_name" in order to preserve the behavior.
diff --git a/src/doc/index.rst b/src/doc/index.rst
index 724bcc543dd1cf0b9af451c487b1b3aab7fa95ca..b4e30e4728068cabb92626cfac986ab858a0bbb6 100644
--- a/src/doc/index.rst
+++ b/src/doc/index.rst
@@ -1,5 +1,5 @@
-Crawler 2.0 Documentation
-=========================
+CaosDB-Crawler Documentation
+============================
.. toctree::
@@ -13,22 +13,23 @@ Crawler 2.0 Documentation
CFoods (Crawler Definitions)<cfood>
Macros<macros>
Tutorials<tutorials/index>
+ How to upgrade<how-to-upgrade>
API documentation<_apidoc/modules>
-This is the documentation for the crawler (previously known as crawler 2.0) for CaosDB, ``caosdb-crawler``.
+This is the documentation for CaosDB-Crawler (previously known as crawler 2.0)
+the main tool for automatic data insertion into CaosDB.
-The crawler is the main date integration tool for CaosDB.
Its task is to automatically synchronize data found on file systems or in other
sources of data with the semantic data model of CaosDB.
More specifically, data that is contained in a hierarchical structure is converted to a data
structure that is consistent with a predefined semantic data model.
-The hierarchical sturcture can be for example a file tree. However it can be
-also something different like the contents of a json file or a file tree with
-json files.
+The hierarchical structure can be for example a file tree. However it can be
+also something different like the contents of a JSON file or a file tree with
+JSON files.
This documentation helps you to :doc:`get started<README_SETUP>`, explains the most important
:doc:`concepts<concepts>` and offers a range of :doc:`tutorials<tutorials/index>`.
@@ -40,5 +41,3 @@ Indices and tables
* :ref:`genindex`
* :ref:`modindex`
* :ref:`search`
-
-
diff --git a/unittests/test_cache.py b/unittests/test_cache.py
deleted file mode 100644
index 135316b92fda0ac1e43f4e5f2c4f28fbf1272494..0000000000000000000000000000000000000000
--- a/unittests/test_cache.py
+++ /dev/null
@@ -1,56 +0,0 @@
-#!/bin/python
-# Tests for entity comparison
-# A. Schlemmer, 06/2021
-
-import caosdb as db
-from pytest import raises
-
-from caoscrawler.identified_cache import _create_hashable_string as create_hash_string
-
-
-def test_normal_hash_creation():
- # Test the initial functionality:
- # hash comprises only one parent, name and properties:
-
- r1 = db.Record()
- r1.add_property(name="test")
- r1.add_parent("bla")
- hash1 = create_hash_string(r1)
-
- r2 = db.Record()
- r2.add_property(name="test2")
- r2.add_parent("bla")
- hash2 = create_hash_string(r2)
-
- assert hash1 != hash2
-
- r3 = db.Record()
- r3.add_property(name="test")
- r3.add_parent("bla bla")
- hash3 = create_hash_string(r3)
- assert hash1 != hash3
- assert hash2 != hash3
-
- # no name and no properties and no parents:
- r4 = db.Record()
- with raises(RuntimeError, match=".*1 parent.*"):
- create_hash_string(r4)
-
- # should work
- r4.add_parent("bla")
- assert len(create_hash_string(r4)) > 0
- r4.add_property(name="test")
- assert len(create_hash_string(r4)) > 0
-
- r4.add_parent("bla bla")
- with raises(RuntimeError, match=".*1 parent.*"):
- create_hash_string(r4)
-
-
-def test_file_hash_creation():
- f1 = db.File(path="/bla/bla/test1.txt")
- hash1 = create_hash_string(f1)
- f2 = db.File(path="/bla/bla/test2.txt")
- hash2 = create_hash_string(f2)
-
- assert hash1 != hash2
diff --git a/unittests/test_file_identifiables.py b/unittests/test_file_identifiables.py
index b0b9801993dc68fe473e788b8ca79a2244912676..c7821f396a3f55634042f74dbe5e6f2d7e223811 100644
--- a/unittests/test_file_identifiables.py
+++ b/unittests/test_file_identifiables.py
@@ -8,63 +8,44 @@ import pytest
from pytest import raises
from caoscrawler.identifiable_adapters import LocalStorageIdentifiableAdapter
+from caoscrawler.identifiable import Identifiable
def test_file_identifiable():
ident = LocalStorageIdentifiableAdapter()
- file_obj = db.File()
+ # Without a path there is no identifying information
+ with raises(ValueError):
+ ident.get_identifiable(db.File())
+
+ fp = "/test/bla/bla.txt"
+ file_obj = db.File(path=fp)
identifiable = ident.get_identifiable(file_obj)
- identifiable2 = ident.get_identifiable_for_file(file_obj)
- # these are two different objects:
- assert identifiable != identifiable2
- assert file_obj != identifiable
- # ... but the path is equal:
- assert identifiable.path == identifiable2.path
- # ... and very boring:
- assert identifiable.path is None
- # Test functionality of retrieving the files:
- identified_file = ident.get_file(identifiable)
- identified_file2 = ident.get_file(file_obj)
- # The both should be None currently as there are no files in the local store yet:
- assert identified_file is None
- assert identified_file2 is None
+ # the path is copied to the identifiable
+ assert fp == identifiable.path
+ assert isinstance(identifiable, Identifiable)
- # Let's make it more interesting:
- file_obj.path = "/test/bla/bla.txt"
- file_obj._checksum = "abcd"
- identifiable = ident.get_identifiable(file_obj)
- assert file_obj != identifiable
- assert file_obj.path == identifiable.path
- # Checksum is not part of the identifiable:
- assert file_obj.checksum != identifiable.checksum
+ # __eq__ function is only defined for Identifiable objects
+ with raises(ValueError):
+ file_obj != identifiable
- # This is the wrong method, so it should definitely return None:
- identified_file = ident.retrieve_identified_record_for_identifiable(
- identifiable)
- assert identified_file is None
- # This is the correct method to use:
- identified_file = ident.get_file(identifiable)
- # or directly using:
- identified_file2 = ident.get_file(file_obj)
- # The both should be None currently as there are no files in the local store yet:
- assert identified_file is None
- assert identified_file2 is None
+ # since the path does not exist in the data in ident, the follwoing functions return None
+ assert ident.retrieve_identified_record_for_record(file_obj) is None
+ assert ident.get_file(identifiable) is None
# Try again with actual files in the store:
records = ident.get_records()
- test_record_wrong_path = db.File(
- path="/bla/bla/test.txt")
- test_record_correct_path = db.File(
- path="/test/bla/bla.txt")
- test_record_alsocorrect_path = db.File(
- path="/test/bla/bla.txt")
+ test_record_wrong_path = db.File(path="/bla/bla/test.txt")
+ test_record_correct_path = db.File(path="/test/bla/bla.txt")
+ test_record_alsocorrect_path = db.File(path="/test/bla/bla.txt")
records.append(test_record_wrong_path)
+ # Now, there is a file, but still wrong path -> result is still None
identified_file = ident.get_file(file_obj)
assert identified_file is None
records.append(test_record_correct_path)
+ # now there is a match
identified_file = ident.get_file(file_obj)
assert identified_file is not None
assert identified_file.path == file_obj.path
diff --git a/unittests/test_identifiable.py b/unittests/test_identifiable.py
new file mode 100644
index 0000000000000000000000000000000000000000..c06ef0ec4b0ab7a91e0397884d95a9e496240df2
--- /dev/null
+++ b/unittests/test_identifiable.py
@@ -0,0 +1,82 @@
+#!/usr/bin/env python3
+# encoding: utf-8
+#
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2021 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2021 Henrik tom Wörden <h.tomwoerden@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+
+"""
+test identifiable module
+"""
+
+import pytest
+import caosdb as db
+from caoscrawler.identifiable import Identifiable
+from caoscrawler.identified_cache import IdentifiedCache
+
+
+def test_create_hashable_string():
+ assert Identifiable._create_hashable_string(
+ Identifiable(name="A", record_type="B")) == "P<B>N<A>"
+ assert Identifiable._create_hashable_string(
+ Identifiable(name="A", record_type="B", properties={'a': 5})) == "P<B>N<A>a:5"
+ a = Identifiable._create_hashable_string(
+ Identifiable(name="A", record_type="B", properties={'a': 4, 'b': 5}))
+ b = Identifiable._create_hashable_string(
+ Identifiable(name="A", record_type="B", properties={'b': 5, 'a': 4}))
+ assert a == b
+ assert (
+ Identifiable._create_hashable_string(
+ Identifiable(name="A", record_type="B", properties={'a': db.Record(id=12)})
+ ) == "P<B>N<A>a:12")
+ a = Identifiable._create_hashable_string(
+ Identifiable(name="A", record_type="B", properties={'a': [db.Record(id=12)]}))
+ assert (a == "P<B>N<A>a:[12]")
+ assert (Identifiable._create_hashable_string(
+ Identifiable(name="A", record_type="B", properties={'a': [12]})) == "P<B>N<A>a:[12]")
+ assert (
+ Identifiable._create_hashable_string(
+ Identifiable(name="A", record_type="B", properties={'a': [db.Record(id=12), 11]})
+ ) == "P<B>N<A>a:[12, 11]")
+ assert (
+ Identifiable._create_hashable_string(
+ Identifiable(record_type="B", properties={'a': [db.Record()]})
+ ) != Identifiable._create_hashable_string(
+ Identifiable(record_type="B", properties={'a': [db.Record()]})))
+
+
+def test_name():
+ with pytest.raises(ValueError):
+ Identifiable(properties={"Name": 'li'})
+
+
+def test_equality():
+ assert Identifiable(
+ record_id=12, properties={"a": 0}) == Identifiable(record_id=12, properties={"a": 1})
+ assert Identifiable(
+ record_id=12, properties={"a": 0}) != Identifiable(record_id=13, properties={"a": 0})
+ assert Identifiable(
+ record_id=12, properties={"a": 0}) == Identifiable(properties={"a": 0})
+ assert Identifiable(
+ path="a", properties={"a": 0}) != Identifiable(path="b", properties={"a": 0})
+ assert Identifiable(
+ path="a", properties={"a": 0}) == Identifiable(path="a", properties={"a": 1})
+ assert Identifiable(
+ path="a", properties={"a": 0}) == Identifiable(properties={"a": 0})
+ assert Identifiable(properties={"a": 0}) == Identifiable(properties={"a": 0})
+ assert Identifiable(properties={"a": 0}) != Identifiable(properties={"a": 1})
diff --git a/unittests/test_identifiable_adapters.py b/unittests/test_identifiable_adapters.py
index ef7998a460c07342d30a3f769fd609c1045a9cca..743f64c19653806519074d5ef6fb272ca2847e7f 100644
--- a/unittests/test_identifiable_adapters.py
+++ b/unittests/test_identifiable_adapters.py
@@ -31,33 +31,32 @@ import os
from datetime import datetime
from caoscrawler.identifiable_adapters import (
CaosDBIdentifiableAdapter, IdentifiableAdapter)
+from caoscrawler.identifiable import Identifiable
import caosdb as db
def test_create_query_for_identifiable():
query = IdentifiableAdapter.create_query_for_identifiable(
- db.Record().add_parent("Person")
- .add_property("first_name", value="A")
- .add_property("last_name", value="B"))
+ Identifiable(record_type="Person", properties={"first_name": "A", "last_name": "B"}))
assert query.lower() == "find record person with 'first_name'='a' and 'last_name'='b' "
query = IdentifiableAdapter.create_query_for_identifiable(
- db.Record(name="A").add_parent("B")
- .add_property("c", value="c")
- .add_property("d", value=5)
- .add_property("e", value=5.5)
- .add_property("f", value=datetime(2020, 10, 10))
- .add_property("g", value=True)
- .add_property("h", value=db.Record(id=1111))
- .add_property("i", value=db.File(id=1112))
- .add_property("j", value=[2222, db.Record(id=3333)]))
+ Identifiable(name="A", record_type="B", properties={
+ "c": "c",
+ "d": 5,
+ "e": 5.5,
+ "f": datetime(2020, 10, 10),
+ "g": True,
+ "h": db.Record(id=1111),
+ "i": db.File(id=1112),
+ "j": [2222, db.Record(id=3333)]}))
assert (query.lower() == "find record b with name='a' and 'c'='c' and 'd'='5' and 'e'='5.5'"
" and 'f'='2020-10-10t00:00:00' and 'g'='true' and 'h'='1111' and 'i'='1112' and "
"'j'='2222' and 'j'='3333' ")
# The name can be the only identifiable
query = IdentifiableAdapter.create_query_for_identifiable(
- db.Record(name="TestRecord").add_parent("TestType"))
+ Identifiable(name="TestRecord", record_type="TestType"))
assert query.lower() == "find record testtype with name='testrecord'"
diff --git a/unittests/test_identified_cache.py b/unittests/test_identified_cache.py
index aeb5f0afcd9fc9912579bf5320bbb36b52899f07..4ed7c55c7326415308917e20e9f391b17b07ad87 100644
--- a/unittests/test_identified_cache.py
+++ b/unittests/test_identified_cache.py
@@ -27,44 +27,18 @@
test identified_cache module
"""
-from caoscrawler.identified_cache import _create_hashable_string, IdentifiedCache
import caosdb as db
-
-
-def test_create_hash():
- assert _create_hashable_string(
- db.Record("A").add_parent("B")) == "P<B>N<A>"
- assert _create_hashable_string(db.Record("A")
- .add_parent("B").add_property('a', 5)) == "P<B>N<A>a:5"
- assert (_create_hashable_string(
- db.Record("A").add_parent("B")
- .add_property('a', 4).add_property('b', 5)) == _create_hashable_string(
- db.Record("A").add_parent("B")
- .add_property('b', 5).add_property('a', 4)))
- assert (_create_hashable_string(db.Record("A")
- .add_parent("B")
- .add_property('a', db.Record(id=12))) == "P<B>N<A>a:12")
- assert (_create_hashable_string(db.Record("A")
- .add_parent("B")
- .add_property('a', [db.Record(id=12)])) == "P<B>N<A>a:[12]")
- assert (_create_hashable_string(db.Record("A")
- .add_parent("B").add_property('a', [12])) == "P<B>N<A>a:[12]")
- assert (_create_hashable_string(
- db.Record("A")
- .add_parent("B")
- .add_property('a', [db.Record(id=12), 11])) == "P<B>N<A>a:[12, 11]")
- assert (_create_hashable_string(
- db.Record().add_parent("B").add_property('a', [db.Record()]))
- != _create_hashable_string(
- db.Record().add_parent("B").add_property('a', [db.Record()])))
+from caoscrawler.identifiable import Identifiable
+from caoscrawler.identified_cache import IdentifiedCache
def test_IdentifiedCache():
- ident = db.Record("A").add_parent("B")
+ ident = Identifiable(name="A", record_type="B")
record = db.Record("A").add_parent("B").add_property('b', 5)
cache = IdentifiedCache()
assert ident not in cache
cache.add(record=record, identifiable=ident)
assert ident in cache
- assert record not in cache
assert cache[ident] is record
+ assert Identifiable(name="A", record_type="C") != Identifiable(name="A", record_type="B")
+ assert Identifiable(name="A", record_type="C") not in cache
diff --git a/unittests/test_tool.py b/unittests/test_tool.py
index e1a155f29c5f3537ca9a33fa775e5497f4b15cc8..9f49bfb9ebfd47dfaf1df120039e04f7ced06ed2 100755
--- a/unittests/test_tool.py
+++ b/unittests/test_tool.py
@@ -4,6 +4,7 @@
# A. Schlemmer, 06/2021
from caoscrawler.crawl import Crawler, SecurityMode
+from caoscrawler.identifiable import Identifiable
from caoscrawler.structure_elements import File, DictTextElement, DictListElement
from caoscrawler.identifiable_adapters import IdentifiableAdapter, LocalStorageIdentifiableAdapter
from simulated_server_data import full_data
@@ -182,15 +183,6 @@ def test_record_structure_generation(crawler):
# ident.store_state(rfp("records.xml"))
-def test_ambigious_records(crawler, ident):
- ident.get_records().clear()
- ident.get_records().extend(crawler.crawled_data)
- r = ident.get_records()
- id_r0 = ident.get_identifiable(r[0])
- with raises(RuntimeError, match=".*unambigiously.*"):
- ident.retrieve_identified_record_for_identifiable(id_r0)
-
-
def test_crawler_update_list(crawler, ident):
# If the following assertions fail, that is a hint, that the test file records.xml has changed
# and this needs to be updated:
@@ -219,13 +211,12 @@ def test_crawler_update_list(crawler, ident):
break
id_r0 = ident.get_identifiable(r_cur)
- assert r_cur.parents[0].name == id_r0.parents[0].name
+ assert r_cur.parents[0].name == id_r0.record_type
assert r_cur.get_property(
- "first_name").value == id_r0.get_property("first_name").value
+ "first_name").value == id_r0.properties["first_name"]
assert r_cur.get_property(
- "last_name").value == id_r0.get_property("last_name").value
+ "last_name").value == id_r0.properties["last_name"]
assert len(r_cur.parents) == 1
- assert len(id_r0.parents) == 1
assert len(r_cur.properties) == 2
assert len(id_r0.properties) == 2
@@ -240,14 +231,13 @@ def test_crawler_update_list(crawler, ident):
break
id_r1 = ident.get_identifiable(r_cur)
- assert r_cur.parents[0].name == id_r1.parents[0].name
+ assert r_cur.parents[0].name == id_r1.record_type
assert r_cur.get_property(
- "identifier").value == id_r1.get_property("identifier").value
- assert r_cur.get_property("date").value == id_r1.get_property("date").value
+ "identifier").value == id_r1.properties["identifier"]
+ assert r_cur.get_property("date").value == id_r1.properties["date"]
assert r_cur.get_property(
- "project").value == id_r1.get_property("project").value
+ "project").value == id_r1.properties["project"]
assert len(r_cur.parents) == 1
- assert len(id_r1.parents) == 1
assert len(r_cur.properties) == 4
assert len(id_r1.properties) == 3
@@ -262,21 +252,6 @@ def test_crawler_update_list(crawler, ident):
"responsible").value == idr_r1.get_property("responsible").value
assert r_cur.description == idr_r1.description
- # test whether compare_entites function works in this context:
- comp = compare_entities(r_cur, id_r1)
- assert len(comp[0]["parents"]) == 0
- assert len(comp[1]["parents"]) == 0
- assert len(comp[0]["properties"]) == 1
- assert len(comp[1]["properties"]) == 0
- assert "responsible" in comp[0]["properties"]
- assert "description" in comp[0]
-
- comp = compare_entities(r_cur, idr_r1)
- assert len(comp[0]["parents"]) == 0
- assert len(comp[1]["parents"]) == 0
- assert len(comp[0]["properties"]) == 0
- assert len(comp[1]["properties"]) == 0
-
def test_synchronization(crawler, ident):
insl, updl = crawler.synchronize(commit_changes=False)
@@ -286,9 +261,9 @@ def test_synchronization(crawler, ident):
def test_identifiable_adapter():
query = IdentifiableAdapter.create_query_for_identifiable(
- db.Record().add_parent("Person")
- .add_property("first_name", value="A")
- .add_property("last_name", value="B"))
+ Identifiable(record_type="Person",
+ properties={"first_name": "A",
+ "last_name": "B"}))
assert query.lower() == "find record person with 'first_name'='a' and 'last_name'='b' "
@@ -359,6 +334,10 @@ def test_provenance_debug_data(crawler):
assert check_key_count("Person") == 14
+def test_split_into_inserts_and_updates_trivial(crawler):
+ crawler.split_into_inserts_and_updates([])
+
+
def basic_retrieve_by_name_mock_up(rec, known):
""" returns a stored Record if rec.name is an existing key, None otherwise """
if rec.name in known:
@@ -377,27 +356,26 @@ def crawler_mocked_identifiable_retrieve(crawler):
# There is only a single known Record with name A
crawler.identifiableAdapter.retrieve_identified_record_for_record = Mock(side_effect=partial(
basic_retrieve_by_name_mock_up, known={"A": db.Record(id=1111, name="A")}))
+ crawler.identifiableAdapter.retrieve_identified_record_for_identifiable = Mock(
+ side_effect=partial(
+ basic_retrieve_by_name_mock_up, known={"A": db.Record(id=1111, name="A")}))
return crawler
-def test_split_into_inserts_and_updates_trivial(crawler):
- # Try trivial argument
- crawler.split_into_inserts_and_updates([])
-
-
def test_split_into_inserts_and_updates_single(crawler_mocked_identifiable_retrieve):
crawler = crawler_mocked_identifiable_retrieve
+ identlist = [Identifiable(name="A", record_type="C"), Identifiable(name="B", record_type="C")]
entlist = [db.Record(name="A").add_parent(
"C"), db.Record(name="B").add_parent("C")]
- assert crawler.get_from_any_cache(entlist[0]) is None
- assert crawler.get_from_any_cache(entlist[1]) is None
- assert not crawler.has_reference_value_without_id(entlist[0])
- assert not crawler.has_reference_value_without_id(entlist[1])
+ assert crawler.get_from_any_cache(identlist[0]) is None
+ assert crawler.get_from_any_cache(identlist[1]) is None
+ assert not crawler._has_reference_value_without_id(identlist[0])
+ assert not crawler._has_reference_value_without_id(identlist[1])
assert crawler.identifiableAdapter.retrieve_identified_record_for_record(
- entlist[0]).id == 1111
+ identlist[0]).id == 1111
assert crawler.identifiableAdapter.retrieve_identified_record_for_record(
- entlist[1]) is None
+ identlist[1]) is None
insert, update = crawler.split_into_inserts_and_updates(deepcopy(entlist))
assert len(insert) == 1
@@ -406,7 +384,7 @@ def test_split_into_inserts_and_updates_single(crawler_mocked_identifiable_retri
assert update[0].name == "A"
# if this ever fails, the mock up may be removed
crawler.identifiableAdapter.get_registered_identifiable.assert_called()
- crawler.identifiableAdapter.retrieve_identified_record_for_record.assert_called()
+ crawler.identifiableAdapter.retrieve_identified_record_for_identifiable.assert_called()
def test_split_into_inserts_and_updates_with_duplicate(crawler_mocked_identifiable_retrieve):
@@ -424,7 +402,7 @@ def test_split_into_inserts_and_updates_with_duplicate(crawler_mocked_identifiab
assert update[0].name == "A"
# if this ever fails, the mock up may be removed
crawler.identifiableAdapter.get_registered_identifiable.assert_called()
- crawler.identifiableAdapter.retrieve_identified_record_for_record.assert_called()
+ crawler.identifiableAdapter.retrieve_identified_record_for_identifiable.assert_called()
def test_split_into_inserts_and_updates_with_ref(crawler_mocked_identifiable_retrieve):
@@ -440,8 +418,8 @@ def test_split_into_inserts_and_updates_with_ref(crawler_mocked_identifiable_ret
assert len(update) == 1
assert update[0].name == "A"
# if this ever fails, the mock up may be removed
- crawler.identifiableAdapter.retrieve_identified_record_for_record.assert_called()
crawler.identifiableAdapter.get_registered_identifiable.assert_called()
+ crawler.identifiableAdapter.retrieve_identified_record_for_identifiable.assert_called()
def test_split_into_inserts_and_updates_with_circ(crawler):
@@ -476,7 +454,7 @@ def test_split_into_inserts_and_updates_with_complex(crawler_mocked_identifiable
assert update[0].name == "A"
# if this ever fails, the mock up may be removed
crawler.identifiableAdapter.get_registered_identifiable.assert_called()
- crawler.identifiableAdapter.retrieve_identified_record_for_record.assert_called()
+ crawler.identifiableAdapter.retrieve_identified_record_for_identifiable.assert_called()
# TODO write test where the unresoled entity is not part of the identifiable
@@ -495,7 +473,7 @@ def test_split_into_inserts_and_updates_with_copy_attr(crawler_mocked_identifiab
assert update[0].get_property("foo").value == 1
# if this ever fails, the mock up may be removed
crawler.identifiableAdapter.get_registered_identifiable.assert_called()
- crawler.identifiableAdapter.retrieve_identified_record_for_record.assert_called()
+ crawler.identifiableAdapter.retrieve_identified_record_for_identifiable.assert_called()
def test_has_missing_object_in_references(crawler):
@@ -509,56 +487,55 @@ def test_has_missing_object_in_references(crawler):
}))
# one reference with id -> check
- assert not crawler.has_missing_object_in_references(db.Record(name="C")
- .add_parent("RTC").add_property('d', 123))
+ assert not crawler._has_missing_object_in_references(
+ Identifiable(name="C", record_type="RTC", properties={'d': 123}))
# one ref with Entity with id -> check
- assert not crawler.has_missing_object_in_references(db.Record(name="C")
- .add_parent("RTC")
- .add_property('d', db.Record(id=123)
- .add_parent("C")))
+ assert not crawler._has_missing_object_in_references(
+ Identifiable(name="C", record_type="RTC", properties={'d': db.Record(id=123)
+ .add_parent("C")}))
# one ref with id one with Entity with id (mixed) -> check
- assert not crawler.has_missing_object_in_references(db.Record(name="C").add_parent("RTD")
- .add_property('d', 123)
- .add_property('b', db.Record(id=123)
- .add_parent("RTC")))
+ assert not crawler._has_missing_object_in_references(
+ Identifiable(name="C", record_type="RTD",
+ properties={'d': 123, 'b': db.Record(id=123).add_parent("RTC")}))
# entity to be referenced in the following
a = db.Record(name="C").add_parent("C").add_property("d", 12311)
# one ref with id one with Entity without id (but not identifying) -> fail
- assert not crawler.has_missing_object_in_references(db.Record(name="C").add_parent("RTC")
- .add_property('d', 123)
- .add_property('e', a))
+ assert not crawler._has_missing_object_in_references(
+ Identifiable(name="C", record_type="RTC", properties={'d': 123, 'e': a}))
# one ref with id one with Entity without id (mixed) -> fail
- assert not crawler.has_missing_object_in_references(db.Record(name="D").add_parent("RTD")
- .add_property('d', 123)
- .add_property('e', a))
+ assert not crawler._has_missing_object_in_references(
+ Identifiable(name="D", record_type="RTD", properties={'d': 123, 'e': a}))
+
crawler.add_to_remote_missing_cache(a)
# one ref with id one with Entity without id but in cache -> check
- assert crawler.has_missing_object_in_references(db.Record(name="D").add_parent("RTD")
- .add_property('d', 123)
- .add_property('e', a))
+ assert crawler._has_missing_object_in_references(
+ Identifiable(name="D", record_type="RTD", properties={'d': 123, 'e': a}))
+
# if this ever fails, the mock up may be removed
crawler.identifiableAdapter.get_registered_identifiable.assert_called()
+@pytest.mark.xfail()
def test_references_entities_without_ids(crawler, ident):
- assert not crawler.has_reference_value_without_id(db.Record().add_parent("Person")
- .add_property('last_name', 123)
- .add_property('first_name', 123))
+ assert not crawler._has_reference_value_without_id(db.Record().add_parent("Person")
+ .add_property('last_name', 123)
+ .add_property('first_name', 123))
# id and rec with id
- assert not crawler.has_reference_value_without_id(db.Record().add_parent("Person")
- .add_property('first_name', 123)
- .add_property('last_name', db.Record(id=123)))
+ assert not crawler._has_reference_value_without_id(db.Record().add_parent("Person")
+ .add_property('first_name', 123)
+ .add_property('last_name',
+ db.Record(id=123)))
# id and rec with id and one unneeded prop
- assert crawler.has_reference_value_without_id(db.Record().add_parent("Person")
- .add_property('first_name', 123)
- .add_property('stuff', db.Record())
- .add_property('last_name', db.Record(id=123)))
+ assert crawler._has_reference_value_without_id(db.Record().add_parent("Person")
+ .add_property('first_name', 123)
+ .add_property('stuff', db.Record())
+ .add_property('last_name', db.Record(id=123)))
# one identifying prop is missing
- assert crawler.has_reference_value_without_id(db.Record().add_parent("Person")
- .add_property('first_name', 123)
- .add_property('last_name', db.Record()))
+ assert crawler._has_reference_value_without_id(db.Record().add_parent("Person")
+ .add_property('first_name', 123)
+ .add_property('last_name', db.Record()))
def test_replace_entities_with_ids(crawler):
@@ -604,6 +581,9 @@ def reset_mocks(mocks):
def change_identifiable_prop(ident):
+ """
+ This function is supposed to change a non identifiing property.
+ """
for ent in ident._records:
if len(ent.parents) == 0 or ent.parents[0].name != "Measurement":
continue
@@ -613,10 +593,14 @@ def change_identifiable_prop(ident):
# change one element; This removes a responsible which is not part of the identifiable
prop.value = "2022-01-04"
return
+ # If it does not work, this test is not implemented properly
raise RuntimeError("Did not find the property that should be changed.")
def change_non_identifiable_prop(ident):
+ """
+ This function is supposed to change a non identifiing property.
+ """
for ent in ident._records:
if len(ent.parents) == 0 or ent.parents[0].name != "Measurement":
continue