diff --git a/integrationtests/README.md b/integrationtests/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..c1f96606a46de4dd96f90fd4a1e46957100e68b3
--- /dev/null
+++ b/integrationtests/README.md
@@ -0,0 +1,3 @@
+1. Clear database
+2. Insert model
+3. Run test.py
diff --git a/integrationtests/clear_database.py b/integrationtests/clear_database.py
new file mode 100644
index 0000000000000000000000000000000000000000..138cf4e6abb256d5710cd2b32f55a1fb51f3fbed
--- /dev/null
+++ b/integrationtests/clear_database.py
@@ -0,0 +1,46 @@
+# -*- coding: utf-8 -*-
+#
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2020 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2020 Florian Spreckelsen <f.spreckelsen@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public
+# License along with this program. If not, see
+# <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+"""Clear the database before and after the integration tests."""
+import caosdb as db
+
+
+def clear_all():
+ """First remove Records, then RecordTypes, then Properties, finally
+ files. Since there may be no entities, execute all deletions
+ without raising errors.
+
+ """
+ db.execute_query("FIND Record").delete(
+ raise_exception_on_error=False)
+ db.execute_query("FIND RecordType").delete(
+ raise_exception_on_error=False)
+ db.execute_query("FIND Property").delete(
+ raise_exception_on_error=False)
+ db.execute_query("FIND File").delete(
+ raise_exception_on_error=False)
+
+
+if __name__ == "__main__":
+ clear_all()
diff --git a/integrationtests/insert_model.py b/integrationtests/insert_model.py
new file mode 100755
index 0000000000000000000000000000000000000000..45bdb6c837c36c999b289548e0f685519cd3aa85
--- /dev/null
+++ b/integrationtests/insert_model.py
@@ -0,0 +1,33 @@
+#!/usr/bin/env python3
+# encoding: utf-8
+#
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2021 Henrik tom Wörden
+# 2021 Alexander Schlemmer
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+import caosdb as db
+from caosadvancedtools.models.data_model import DataModel
+from caosadvancedtools.models.parser import parse_model_from_yaml
+
+
+def main():
+ model = parse_model_from_yaml("model.yml")
+ model.sync_data_model(noquestion=True)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/integrationtests/model.yml b/integrationtests/model.yml
new file mode 100644
index 0000000000000000000000000000000000000000..7d78ac7ef4bc792f54594b29a8ac311479f41a59
--- /dev/null
+++ b/integrationtests/model.yml
@@ -0,0 +1,88 @@
+Experiment:
+ obligatory_properties:
+ date:
+ datatype: DATETIME
+ description: 'date of the experiment'
+ identifier:
+ datatype: TEXT
+ description: 'identifier of the experiment'
+ # TODO empty recommended_properties is a problem
+ #recommended_properties:
+ responsible:
+ datatype: LIST<Person>
+Project:
+SoftwareVersion:
+ recommended_properties:
+ version:
+ datatype: TEXT
+ description: 'Version of the software.'
+ binaries:
+ sourceCode:
+ Software:
+DepthTest:
+ obligatory_properties:
+ temperature:
+ datatype: DOUBLE
+ description: 'temp'
+ depth:
+ datatype: DOUBLE
+ description: 'temp'
+Person:
+ obligatory_properties:
+ first_name:
+ datatype: TEXT
+ description: 'First name of a Person.'
+ last_name:
+ datatype: TEXT
+ description: 'LastName of a Person.'
+ recommended_properties:
+ email:
+ datatype: TEXT
+ description: 'Email of a Person.'
+revisionOf:
+ datatype: REFERENCE
+results:
+ datatype: LIST<REFERENCE>
+sources:
+ datatype: LIST<REFERENCE>
+scripts:
+ datatype: LIST<REFERENCE>
+single_attribute:
+ datatype: LIST<INTEGER>
+Simulation:
+ obligatory_properties:
+ date:
+ identifier:
+ responsible:
+Analysis:
+ obligatory_properties:
+ date:
+ identifier:
+ responsible:
+ suggested_properties:
+ mean_value:
+ datatype: DOUBLE
+Publication:
+Thesis:
+ inherit_from_suggested:
+ - Publication
+Article:
+ inherit_from_suggested:
+ - Publication
+Poster:
+ inherit_from_suggested:
+ - Publication
+Presentation:
+ inherit_from_suggested:
+ - Publication
+Report:
+ inherit_from_suggested:
+ - Publication
+hdf5File:
+ datatype: REFERENCE
+extern:
+ - TestRT1
+ - TestP1
+Measurement:
+ recommended_properties:
+ date:
diff --git a/integrationtests/test.py b/integrationtests/test.py
new file mode 100644
index 0000000000000000000000000000000000000000..782687be27863e479186717d698b9965f7be8c64
--- /dev/null
+++ b/integrationtests/test.py
@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+# encoding: utf-8
+#
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2021 Indiscale GmbH <info@indiscale.com>
+# 2021 Henrik tom Wörden <h.tomwoerden@indiscale.com>
+# 2021 Alexander Schlemmer
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+
+"""
+module description
+"""
+
+import argparse
+import sys
+from argparse import RawTextHelpFormatter
+from newcrawler import Crawler
+from unittest.mock import Mock
+import caosdb as db
+from newcrawler.identifiable_adapters import CaosDBIdentifiableAdapter
+
+import os
+
+
+def rfp(*pathcomponents):
+ """
+ Return full path.
+ Shorthand convenience function.
+ """
+ return os.path.join(os.path.dirname(__file__), *pathcomponents)
+
+
+def main(args):
+ ident_adapt = CaosDBIdentifiableAdapter()
+ # TODO place this definition of identifiables elsewhere
+ ident_adapt.register_identifiable(
+ "Person", db.RecordType()
+ .add_parent(name="Person")
+ .add_property(name="first_name")
+ .add_property(name="last_name"))
+ ident_adapt.register_identifiable(
+ "Measurement", db.RecordType()
+ .add_parent(name="Measurement")
+ .add_property(name="identifier")
+ .add_property(name="date")
+ .add_property(name="project"))
+ ident_adapt.register_identifiable(
+ "Project", db.RecordType()
+ .add_parent(name="Project")
+ .add_property(name="date")
+ .add_property(name="identifier"))
+
+ crawler = Crawler(debug=True, identifiableAdapter=ident_adapt)
+ crawler.copy_attributes = Mock()
+ crawler.crawl_directory(rfp("../unittests/test_directories", "examples_article"),
+ rfp("../unittests/scifolder_cfood.yml"))
+ ins, ups = crawler.synchronize()
+ assert len(ins) == 18
+ assert len(ups) == 0
+
+
+def parse_args():
+ parser = argparse.ArgumentParser(description=__doc__,
+ formatter_class=RawTextHelpFormatter)
+ # parser.add_argument("path",
+ # help="the subtree of files below the given path will "
+ # "be considered. Use '/' for everything.")
+
+ return parser.parse_args()
+
+
+if __name__ == "__main__":
+ args = parse_args()
+ sys.exit(main(args))
diff --git a/src/newcrawler/crawl.py b/src/newcrawler/crawl.py
index 17833eea4017ad1846dcddf9310918f781e446c8..be2a6792ac29dd7ee565bae2de66aaaa8be7f1dc 100644
--- a/src/newcrawler/crawl.py
+++ b/src/newcrawler/crawl.py
@@ -301,8 +301,21 @@ class Crawler(object):
# This record is a duplicate that can be removed. Make sure we do not lose
# information
# Update an (local) identified record that will be inserted
+ newrecord = self.get_identified_record_from_local_cache(record)
self.copy_attributes(
- fro=record, to=self.get_identified_record_from_local_cache(record))
+ fro=record, to=newrecord)
+ # Bend references to the other object
+ # TODO refactor this
+ for el in flat + to_be_inserted + to_be_updated:
+ for p in el.properties:
+ if isinstance(p.value, list):
+ for index, val in enumerate(p.value):
+ if val is record:
+ p.value[index] = newrecord
+ else:
+ if p.value is record:
+ p.value = newrecord
+
del flat[i]
continue
@@ -346,15 +359,23 @@ class Crawler(object):
if isinstance(val, db.Entity):
el.value[index] = val.id
- def remove_unnecessary_updates(self, updateList: list[db.Record]):
+ @staticmethod
+ def remove_unnecessary_updates(updateList: list[db.Record],
+ identified_records: list[db.Record]):
"""
checks whether all relevant attributes (especially Property values) are equal
+
+ Returns (in future)
+ -------
+ update list without unecessary updates
+
"""
+ if len(updateList) != len(identified_records):
+ raise RuntimeError("The lists of updates and of identified records need to be of the "
+ "same length!")
+ # TODO this can now easily be changed to a function without side effect
for i in reversed(range(len(updateList))):
- record = updateList[i]
- identifiable = self.identifiableAdapter.retrieve_identifiable(record)
-
- comp = compare_entities(record, identifiable)
+ comp = compare_entities(updateList[i], identified_records[i])
identical = True
for j in range(2):
# TODO: should be implemented elsewhere (?)
@@ -366,12 +387,15 @@ class Crawler(object):
break
for key in comp[0]["properties"]:
for attribute in ("datatype", "importance", "unit"):
- if (attribute in comp[0]["properties"][key] and
- comp[0]["properties"][key][attribute] is not None and
- comp[0]["properties"][key][attribute] !=
- comp[1]["properties"][key][attribute]):
- identical = False
- break
+ # only make an update for those attributes if there is a value difference and
+ # the value in the updateList is not None
+ if attribute in comp[0]["properties"][key]:
+ attr_val = comp[0]["properties"][key][attribute]
+ other_attr_val = (comp[1]["properties"][key][attribute]
+ if attribute in comp[1]["properties"][key] else None)
+ if attr_val is not None and atrr_val != other_attr_val:
+ identical = False
+ break
if "value" in comp[0]["properties"][key]:
identical = False
@@ -385,6 +409,16 @@ class Crawler(object):
else:
pass
+ @staticmethod
+ def execute_inserts_in_list(to_be_inserted):
+ if len(to_be_inserted) > 0:
+ db.Container().extend(to_be_inserted).insert()
+
+ @staticmethod
+ def execute_updates_in_list(to_be_updated):
+ if len(to_be_updated) > 0:
+ db.Container().extend(to_be_updated).update()
+
def _synchronize(self, updateList: list[db.Record]):
"""
This function applies several stages:
@@ -407,11 +441,12 @@ class Crawler(object):
for el in to_be_updated:
self.replace_entities_by_ids(el)
- self.remove_unnecessary_updates(to_be_updated)
+ identified_records = [self.identifiableAdapter.retrieve_identifiable(record) for record
+ in to_be_updated]
+ self.remove_unnecessary_updates(to_be_updated, identified_records)
- # TODO
- # self.execute_inserts_in_list(to_be_inserted)
- # self.execute_updates_in_list(to_be_updated)
+ self.execute_inserts_in_list(to_be_inserted)
+ self.execute_updates_in_list(to_be_updated)
return (to_be_inserted, to_be_updated)
diff --git a/src/newcrawler/identifiable_adapters.py b/src/newcrawler/identifiable_adapters.py
index 01eb55bbb3ec4b54a49633ef839b2ed99ab5b398..f11a7fc101225db4fd3bdd15f3ad397425930d08 100644
--- a/src/newcrawler/identifiable_adapters.py
+++ b/src/newcrawler/identifiable_adapters.py
@@ -23,9 +23,34 @@
# ** end header
#
+from datetime import datetime
import caosdb as db
from abc import abstractmethod
from .utils import get_value, has_parent
+from caosdb.common.datatype import is_reference
+from .utils import has_parent
+
+
+def convert_value(value):
+ """ Returns a string representation of the value that is suitable to be used in the query
+ looking for the identified record.
+
+ Parameters
+ ----------
+ value : The property of which the value shall be returned.
+
+ Returns
+ -------
+ out : the string reprensentation of the value
+
+ """
+
+ if isinstance(value, db.Entity):
+ return str(value.id)
+ elif isinstance(value, datetime):
+ return value.isoformat()
+ else:
+ return str(value)
class IdentifiableAdapter(object):
@@ -67,7 +92,6 @@ class IdentifiableAdapter(object):
if len(ident.parents) != 1:
raise RuntimeError("Multiple parents for identifiables not supported.")
- # TODO prevent multiple parents
query_string = "FIND Record " + ident.get_parents()[0].name
query_string += " WITH "
@@ -76,17 +100,16 @@ class IdentifiableAdapter(object):
"The identifiable must have features to identify it.")
if ident.name is not None:
- query_string += "name='{}' AND".format(ident.name)
+ query_string += "name='{}' AND ".format(ident.name)
for p in ident.get_properties():
- # TODO this is badly wrong :-|
-
- if p.datatype is not None and p.datatype.startswith("LIST<"):
+ if isinstance(p.value, list):
for v in p.value:
- query_string += ("references " + str(v.id if isinstance(v, db.Entity)
- else v) + " AND ")
+ query_string += ("'" + p.name + "'='" +
+ convert_value(v) + "' AND ")
else:
- query_string += ("'" + p.name + "'='" + str(get_value(p)) + "' AND ")
+ query_string += ("'" + p.name + "'='" +
+ convert_value(p.value) + "' AND ")
# remove the last AND
return query_string[:-4]
@@ -231,7 +254,7 @@ class LocalStorageIdentifiableAdapter(IdentifiableAdapter):
def get_registered_identifiable(self, record: db.Record):
identifiable_candidates = []
- for name, definition in self._registered_identifiables.items():
+ for _, definition in self._registered_identifiables.items():
if self.is_identifiable_for_record(definition, record):
identifiable_candidates.append(definition)
if len(identifiable_candidates) > 1:
@@ -303,3 +326,39 @@ class LocalStorageIdentifiableAdapter(IdentifiableAdapter):
raise RuntimeError("The entity has not been assigned an ID.")
return value_identifiable.id
+
+
+class CaosDBIdentifiableAdapter(IdentifiableAdapter):
+ """
+ Identifiable adapter which can be used for production.
+
+
+ TODO: store registred identifiables not locally
+ """
+
+ def __init__(self):
+ self._registered_identifiables = dict()
+
+ def register_identifiable(self, name: str, definition: db.RecordType):
+ self._registered_identifiables[name] = definition
+
+ def get_registered_identifiable(self, record: db.Record):
+ """
+ returns the registred identifiable for the given Record
+
+ It is assumed, that there is exactly one identifiable for each RecordType. Only the first
+ parent of the given Record is considered; others are ignored
+ """
+ rt_name = record.parents[0].name
+ for name, definition in self._registered_identifiables.items():
+ if definition.parents[0].name.lower() == rt_name.lower():
+ return definition
+
+ def retrieve_identified_record(self, identifiable: db.Record):
+ query_string = self.create_query_for_identifiable(identifiable)
+ candidates = db.execute_query(query_string)
+ if len(candidates) > 1:
+ raise RuntimeError("Identifiable was not defined unambigiously.")
+ if len(candidates) == 0:
+ return None
+ return candidates[0]
diff --git a/src/newcrawler/utils.py b/src/newcrawler/utils.py
index c60b7f871db32d66c12781e5f0cfb246bc41c8fe..35fefe6719d579bc8e8a39489f8a872c0cca11b8 100644
--- a/src/newcrawler/utils.py
+++ b/src/newcrawler/utils.py
@@ -40,27 +40,3 @@ def has_parent(entity: db.Entity, name: str):
if parent.name == name:
return True
return False
-
-
-def get_value(prop):
- """ Returns the value of a Property
-
- This function is taken from the old crawler:
- caosdb-advanced-user-tools/src/caosadvancedtools/crawler.py
-
- Parameters
- ----------
- prop : The property of which the value shall be returned.
-
- Returns
- -------
- out : The value of the property; if the value is an entity, its ID.
-
- """
-
- if isinstance(prop.value, db.Entity):
- return prop.value.id
- elif isinstance(prop.value, datetime):
- return prop.value.isoformat()
- else:
- return prop.value
diff --git a/unittests/test_converters.py b/unittests/test_converters.py
index ab8107398a3fbd2a27e3d174d5bc892ec7d8af1e..3ec1764631c4de7b5a7cc247cc559d0dc5f5939c 100644
--- a/unittests/test_converters.py
+++ b/unittests/test_converters.py
@@ -29,6 +29,7 @@ test the converters module
from newcrawler.converters import Converter
from newcrawler.stores import GeneralStore
+from newcrawler.converters import MarkdownFileConverter
from newcrawler.structure_elements import Directory
from test_tool import rfp
@@ -65,3 +66,57 @@ def testDirectoryConverter():
assert len(elements) == 1
assert isinstance(elements[0], Directory)
assert elements[0].name == "examples_article"
+
+
+def test_markdown_converter():
+ test_readme = File("README.md", rfp(
+ "test_directories", "examples_article", "DataAnalysis",
+ "2020_climate-model-predict", "2020-02-08_prediction-errors", "README.md"))
+
+ converter = MarkdownFileConverter({
+ "match": "(.*)"
+ }, "TestMarkdownFileConverter")
+
+ m = converter.match(File("test_tool.py", rfp(
+ "test_tool.py")))
+ assert m is None
+
+ m = converter.match(test_readme)
+ assert m is not None
+ assert m.__class__ == dict
+ assert len(m) == 0
+
+ converter = MarkdownFileConverter({
+ "match": "README.md"
+ }, "TestMarkdownFileConverter")
+
+ m = converter.match(test_readme)
+ assert m is not None
+ assert len(m) == 0
+
+ children = converter.create_children(None, test_readme)
+ assert len(children) == 5
+ assert children[1].__class__ == DictTextElement
+ assert children[1].name == "description"
+ assert children[1].value.__class__ == str
+
+ assert children[0].__class__ == DictTextElement
+ assert children[0].name == "responsible"
+ assert children[0].value.__class__ == str
+
+ test_readme2 = File("README.md", rfp("test_directories", "examples_article",
+ "ExperimentalData", "2020_SpeedOfLight", "2020-01-01_TimeOfFlight", "README.md"))
+
+ m = converter.match(test_readme2)
+ assert m is not None
+ assert len(m) == 0
+
+ children = converter.create_children(None, test_readme2)
+ assert len(children) == 2
+ assert children[1].__class__ == DictTextElement
+ assert children[1].name == "description"
+ assert children[1].value.__class__ == str
+
+ assert children[0].__class__ == DictListElement
+ assert children[0].name == "responsible"
+ assert children[0].value.__class__ == list
diff --git a/unittests/test_identifiable_adapters.py b/unittests/test_identifiable_adapters.py
new file mode 100644
index 0000000000000000000000000000000000000000..9730461020c6c582188db58df6524246c0a1042c
--- /dev/null
+++ b/unittests/test_identifiable_adapters.py
@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+# encoding: utf-8
+#
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2021 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2021 Henrik tom Wörden <h.tomwoerden@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+
+"""
+test identifiable_adapters module
+"""
+
+from datetime import datetime
+from newcrawler.identifiable_adapters import IdentifiableAdapter
+import caosdb as db
+
+
+def test_create_query_for_identifiable():
+ query = IdentifiableAdapter.create_query_for_identifiable(
+ db.Record().add_parent("Person")
+ .add_property("first_name", value="A")
+ .add_property("last_name", value="B"))
+ assert query.lower() == "find record person with 'first_name'='a' and 'last_name'='b' "
+
+ query = IdentifiableAdapter.create_query_for_identifiable(
+ db.Record(name="A").add_parent("B")
+ .add_property("c", value="c")
+ .add_property("d", value=5)
+ .add_property("e", value=5.5)
+ .add_property("f", value=datetime(2020, 10, 10))
+ .add_property("g", value=True)
+ .add_property("h", value=db.Record(id=1111))
+ .add_property("i", value=db.File(id=1112))
+ .add_property("j", value=[2222, db.Record(id=3333)]))
+ assert (query.lower() == "find record b with name='a' and 'c'='c' and 'd'='5' and 'e'='5.5'"
+ " and 'f'='2020-10-10t00:00:00' and 'g'='true' and 'h'='1111' and 'i'='1112' and "
+ "'j'='2222' and 'j'='3333' ")
diff --git a/unittests/test_tool.py b/unittests/test_tool.py
index 23912ff133fdb7dceb1805907d24a57578fc63ee..4b55de8749d3651e4e21482bd903c7da62a96d58 100755
--- a/unittests/test_tool.py
+++ b/unittests/test_tool.py
@@ -4,7 +4,6 @@
# A. Schlemmer, 06/2021
from newcrawler import Crawler
-from newcrawler.converters import MarkdownFileConverter
from newcrawler.structure_elements import File, DictTextElement, DictListElement
from newcrawler.identifiable_adapters import IdentifiableAdapter, LocalStorageIdentifiableAdapter
from functools import partial
@@ -69,7 +68,8 @@ def ident(crawler):
.add_property(name="identifier"))
return ident
-def test_crawler(crawler):
+
+def test_record_structure_generation(crawler):
subd = crawler.debug_tree[dircheckstr("DataAnalysis")]
subc = crawler.debug_metadata["copied"][dircheckstr("DataAnalysis")]
assert len(subd) == 2
@@ -138,59 +138,6 @@ def test_crawler(crawler):
assert subc[0]["identifier"] is False
-def test_markdown_converter():
- test_readme = File("README.md", rfp(
- "test_directories", "examples_article", "DataAnalysis",
- "2020_climate-model-predict", "2020-02-08_prediction-errors", "README.md"))
-
- converter = MarkdownFileConverter({
- "match": "(.*)"
- }, "TestMarkdownFileConverter")
-
- m = converter.match(File("test_tool.py", rfp(
- "test_tool.py")))
- assert m is None
-
- m = converter.match(test_readme)
- assert m is not None
- assert m.__class__ == dict
- assert len(m) == 0
-
- converter = MarkdownFileConverter({
- "match": "README.md"
- }, "TestMarkdownFileConverter")
-
- m = converter.match(test_readme)
- assert m is not None
- assert len(m) == 0
-
- children = converter.create_children(None, test_readme)
- assert len(children) == 5
- assert children[1].__class__ == DictTextElement
- assert children[1].name == "description"
- assert children[1].value.__class__ == str
-
- assert children[0].__class__ == DictTextElement
- assert children[0].name == "responsible"
- assert children[0].value.__class__ == str
-
- test_readme2 = File("README.md", rfp("test_directories", "examples_article",
- "ExperimentalData", "2020_SpeedOfLight", "2020-01-01_TimeOfFlight", "README.md"))
-
- m = converter.match(test_readme2)
- assert m is not None
- assert len(m) == 0
-
- children = converter.create_children(None, test_readme2)
- assert len(children) == 2
- assert children[1].__class__ == DictTextElement
- assert children[1].name == "description"
- assert children[1].value.__class__ == str
-
- assert children[0].__class__ == DictListElement
- assert children[0].name == "responsible"
- assert children[0].value.__class__ == list
-
# def prepare_test_record_file():
# ident = LocalStorageIdentifiableAdapter()
# crawler = Crawler(debug=True, identifiableAdapter=ident)
@@ -217,8 +164,8 @@ def test_ambigious_records(crawler, ident):
def test_crawler_update_list(crawler, ident):
crawler.copy_attributes = Mock()
- # If the following assertions fail, that is a hint, that the test file records.xml is
- # incorrect:
+ # If the following assertions fail, that is a hint, that the test file records.xml has changed
+ # and this needs to be updated:
assert len(ident.get_records()) == 18
assert len([r for r in ident.get_records() if r.parents[0].name == "Person"]) == 5
assert len([r for r in ident.get_records() if r.parents[0].name == "Measurement"]) == 11
@@ -295,37 +242,41 @@ def test_crawler_update_list(crawler, ident):
assert len(updl) == 0
-def test_identifiable_update(crawler, ident):
- # change one value in updateList and then run the synchronization:
- meas = [r for r in crawler.updateList if r.parents[0].name == "Measurement"][0]
- meas.get_property("responsible").value = []
- insl, updl = crawler.synchronize()
- assert len(updl) == 1
-
-
-def test_identifiable_update2(crawler, ident):
- # change one unit in updateList and then run the synchronization:
- meas = [r for r in crawler.updateList if r.parents[0].name == "Measurement"][0]
- meas.get_property("description").unit = "cm"
- insl, updl = crawler.synchronize()
- assert len(updl) == 1
-
-
-def test_identifiable_update3(crawler, ident):
- # change values of multiple records in updateList and then run the synchronization:
- meas = [r for r in crawler.updateList if r.parents[0].name == "Measurement"]
- meas[0].get_property("responsible").value = []
- meas[3].get_property("responsible").value = []
- insl, updl = crawler.synchronize()
- assert len(updl) == 2
-
-
-def test_identifiable_adapter():
- query = IdentifiableAdapter.create_query_for_identifiable(
- db.Record().add_parent("Person")
- .add_property("first_name", value="A")
- .add_property("last_name", value="B"))
- assert query.lower() == "find record person with 'first_name'='a' and 'last_name'='b' "
+def test_remove_unnecessary_updates():
+ # test trvial case
+ upl = [db.Record().add_parent("A")]
+ irs = [db.Record().add_parent("A")]
+ Crawler.remove_unnecessary_updates(upl, irs)
+ assert len(upl) == 0
+
+ # test property difference case
+ # TODO this should work right?
+ #upl = [db.Record().add_parent("A").add_property("a", 3)]
+ # irs = [db.Record().add_parent("A")] # ID should be s
+ #Crawler.remove_unnecessary_updates(upl, irs)
+ #assert len(upl) == 1
+
+ # test value difference case
+ upl = [db.Record().add_parent("A").add_property("a", 5)]
+ irs = [db.Record().add_parent("A").add_property("a")]
+ Crawler.remove_unnecessary_updates(upl, irs)
+ assert len(upl) == 1
+ upl = [db.Record().add_parent("A").add_property("a", 5)]
+ irs = [db.Record().add_parent("A").add_property("a", 5)]
+ Crawler.remove_unnecessary_updates(upl, irs)
+ assert len(upl) == 0
+
+ # test unit difference case
+ upl = [db.Record().add_parent("A").add_property("a", unit='cm')]
+ irs = [db.Record().add_parent("A").add_property("a")]
+ Crawler.remove_unnecessary_updates(upl, irs)
+ assert len(upl) == 1
+
+ # test None difference case
+ upl = [db.Record().add_parent("A").add_property("a")]
+ irs = [db.Record().add_parent("A").add_property("a", 5)]
+ Crawler.remove_unnecessary_updates(upl, irs)
+ assert len(upl) == 1
@pytest.mark.xfail