TST: add test for query and some minor changes

src/newcrawler/crawl.py

@@ -364,10 +364,12 @@ class Crawler(object):
                     break
             for key in comp[0]["properties"]:
                 for attribute in ("datatype", "importance", "unit"):
-                    if (attribute in comp[0]["properties"][key] and
-                        comp[0]["properties"][key][attribute] is not None and
-                        comp[0]["properties"][key][attribute] !=
-                            comp[1]["properties"][key][attribute]):
+
+                    if attribute in comp[0]["properties"][key]:
+                        attr_val = comp[0]["properties"][key][attribute]
+                        other_attr_val = (comp[1]["properties"][key][attribute]
+                                          if attribute in comp[1]["properties"][key] else None)
+                        if attr_val is not None and atrr_val != other_attr_val:
                             identical = False
                             break
 

src/newcrawler/identifiable_adapters.py

@@ -23,9 +23,32 @@
 # ** end header
 #
 
+from datetime import datetime
 import caosdb as db
 from abc import abstractmethod
-from .utils import get_value, has_parent
+from .utils import has_parent
+
+
+def convert_value(value):
+    """ Returns a string representation of the value that is suitable to be used in the query
+    looking for the identified record.
+
+    Parameters
+    ----------
+    value : The property of which the value shall be returned.
+
+    Returns
+    -------
+    out : the string reprensentation of the value
+
+    """
+
+    if isinstance(value, db.Entity):
+        return str(value.id)
+    elif isinstance(value, datetime):
+        return value.isoformat()
+    else:
+        return str(value)
 
 
 class IdentifiableAdapter(object):
@@ -67,7 +90,6 @@ class IdentifiableAdapter(object):
         if len(ident.parents) != 1:
             raise RuntimeError("Multiple parents for identifiables not supported.")
 
-        # TODO prevent multiple parents
         query_string = "FIND Record " + ident.get_parents()[0].name
         query_string += " WITH "
 
@@ -79,14 +101,13 @@ class IdentifiableAdapter(object):
             query_string += "name='{}' AND ".format(ident.name)
 
         for p in ident.get_properties():
-            # TODO this is badly wrong :-|
-
-            if p.datatype is not None and p.datatype.startswith("LIST<"):
+            if isinstance(p.value, list):
                 for v in p.value:
-                    query_string += ("references " + str(v.id if isinstance(v, db.Entity)
-                                                         else v) + " AND ")
+                    query_string += ("'" + p.name + "'='" +
+                                     convert_value(v) + "' AND ")
             else:
-                query_string += ("'" + p.name + "'='" + str(get_value(p)) + "' AND ")
+                query_string += ("'" + p.name + "'='" +
+                                 convert_value(p.value) + "' AND ")
         # remove the last AND
         return query_string[:-4]
 

src/newcrawler/utils.py

@@ -40,27 +40,3 @@ def has_parent(entity: db.Entity, name: str):
         if parent.name == name:
             return True
     return False
-
-
-def get_value(prop):
-    """ Returns the value of a Property
-
-    This function is taken from the old crawler:
-    caosdb-advanced-user-tools/src/caosadvancedtools/crawler.py
-
-    Parameters
-    ----------
-    prop : The property of which the value shall be returned.
-
-    Returns
-    -------
-    out : The value of the property; if the value is an entity, its ID.
-
-    """
-
-    if isinstance(prop.value, db.Entity):
-        return prop.value.id
-    elif isinstance(prop.value, datetime):
-        return prop.value.isoformat()
-    else:
-        return prop.value

unittests/test_converters.py

@@ -29,6 +29,7 @@ test the converters module
 
 from newcrawler.converters import Converter
 from newcrawler.stores import GeneralStore
+from newcrawler.converters import MarkdownFileConverter
 from newcrawler.structure_elements import Directory
 
 from test_tool import rfp
@@ -65,3 +66,57 @@ def testDirectoryConverter():
     assert len(elements) == 1
     assert isinstance(elements[0], Directory)
     assert elements[0].name == "examples_article"
+
+
+def test_markdown_converter():
+    test_readme = File("README.md", rfp(
+        "test_directories", "examples_article", "DataAnalysis",
+        "2020_climate-model-predict", "2020-02-08_prediction-errors", "README.md"))
+
+    converter = MarkdownFileConverter({
+        "match": "(.*)"
+    }, "TestMarkdownFileConverter")
+
+    m = converter.match(File("test_tool.py", rfp(
+        "test_tool.py")))
+    assert m is None
+
+    m = converter.match(test_readme)
+    assert m is not None
+    assert m.__class__ == dict
+    assert len(m) == 0
+
+    converter = MarkdownFileConverter({
+        "match": "README.md"
+    }, "TestMarkdownFileConverter")
+
+    m = converter.match(test_readme)
+    assert m is not None
+    assert len(m) == 0
+
+    children = converter.create_children(None, test_readme)
+    assert len(children) == 5
+    assert children[1].__class__ == DictTextElement
+    assert children[1].name == "description"
+    assert children[1].value.__class__ == str
+
+    assert children[0].__class__ == DictTextElement
+    assert children[0].name == "responsible"
+    assert children[0].value.__class__ == str
+
+    test_readme2 = File("README.md", rfp("test_directories", "examples_article",
+                        "ExperimentalData", "2020_SpeedOfLight", "2020-01-01_TimeOfFlight", "README.md"))
+
+    m = converter.match(test_readme2)
+    assert m is not None
+    assert len(m) == 0
+
+    children = converter.create_children(None, test_readme2)
+    assert len(children) == 2
+    assert children[1].__class__ == DictTextElement
+    assert children[1].name == "description"
+    assert children[1].value.__class__ == str
+
+    assert children[0].__class__ == DictListElement
+    assert children[0].name == "responsible"
+    assert children[0].value.__class__ == list

unittests/test_identifiable_adapters.py

+#!/usr/bin/env python3
+# encoding: utf-8
+#
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2021 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2021 Henrik tom Wörden <h.tomwoerden@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+
+"""
+test identifiable_adapters module
+"""
+
+from datetime import datetime
+from newcrawler.identifiable_adapters import IdentifiableAdapter
+import caosdb as db
+
+
+def test_create_query_for_identifiable():
+    query = IdentifiableAdapter.create_query_for_identifiable(
+        db.Record().add_parent("Person")
+        .add_property("first_name", value="A")
+        .add_property("last_name", value="B"))
+    assert query.lower() == "find record person with 'first_name'='a' and 'last_name'='b' "
+
+    query = IdentifiableAdapter.create_query_for_identifiable(
+        db.Record(name="A").add_parent("B")
+        .add_property("c", value="c")
+        .add_property("d", value=5)
+        .add_property("e", value=5.5)
+        .add_property("f", value=datetime(2020, 10, 10))
+        .add_property("g", value=True)
+        .add_property("h", value=db.Record(id=1111))
+        .add_property("i", value=[2222, db.Record(id=3333)]))
+    assert (query.lower() == "find record b with name='a' and 'c'='c' and 'd'='5' and 'e'='5.5'"
+            " and 'f'='2020-10-10t00:00:00' and 'g'='true' and 'h'='1111' and 'i'='2222' and "
+            "'i'='3333' ")

unittests/test_tool.py

@@ -4,7 +4,6 @@
 # A. Schlemmer, 06/2021
 
 from newcrawler import Crawler
-from newcrawler.converters import MarkdownFileConverter
 from newcrawler.structure_elements import File, DictTextElement, DictListElement
 from newcrawler.identifiable_adapters import IdentifiableAdapter, LocalStorageIdentifiableAdapter
 from functools import partial
@@ -69,7 +68,8 @@ def ident(crawler):
         .add_property(name="identifier"))
     return ident
 
-def test_crawler(crawler):
+
+def test_record_structure_generation(crawler):
     subd = crawler.debug_tree[dircheckstr("DataAnalysis")]
     subc = crawler.debug_metadata["copied"][dircheckstr("DataAnalysis")]
     assert len(subd) == 2
@@ -138,59 +138,6 @@ def test_crawler(crawler):
     assert subc[0]["identifier"] is False
 
 
-def test_markdown_converter():
-    test_readme = File("README.md", rfp(
-        "test_directories", "examples_article", "DataAnalysis",
-        "2020_climate-model-predict", "2020-02-08_prediction-errors", "README.md"))
-
-    converter = MarkdownFileConverter({
-        "match": "(.*)"
-    }, "TestMarkdownFileConverter")
-
-    m = converter.match(File("test_tool.py", rfp(
-        "test_tool.py")))
-    assert m is None
-
-    m = converter.match(test_readme)
-    assert m is not None
-    assert m.__class__ == dict
-    assert len(m) == 0
-
-    converter = MarkdownFileConverter({
-        "match": "README.md"
-    }, "TestMarkdownFileConverter")
-
-    m = converter.match(test_readme)
-    assert m is not None
-    assert len(m) == 0
-
-    children = converter.create_children(None, test_readme)
-    assert len(children) == 5
-    assert children[1].__class__ == DictTextElement
-    assert children[1].name == "description"
-    assert children[1].value.__class__ == str
-
-    assert children[0].__class__ == DictTextElement
-    assert children[0].name == "responsible"
-    assert children[0].value.__class__ == str
-
-    test_readme2 = File("README.md", rfp("test_directories", "examples_article",
-                        "ExperimentalData", "2020_SpeedOfLight", "2020-01-01_TimeOfFlight", "README.md"))
-
-    m = converter.match(test_readme2)
-    assert m is not None
-    assert len(m) == 0
-
-    children = converter.create_children(None, test_readme2)
-    assert len(children) == 2
-    assert children[1].__class__ == DictTextElement
-    assert children[1].name == "description"
-    assert children[1].value.__class__ == str
-
-    assert children[0].__class__ == DictListElement
-    assert children[0].name == "responsible"
-    assert children[0].value.__class__ == list
-
 # def prepare_test_record_file():
 #     ident = LocalStorageIdentifiableAdapter()
 #     crawler = Crawler(debug=True, identifiableAdapter=ident)
@@ -217,8 +164,8 @@ def test_ambigious_records(crawler, ident):
 
 def test_crawler_update_list(crawler, ident):
     crawler.copy_attributes = Mock()
-    # If the following assertions fail, that is a hint, that the test file records.xml is
-    # incorrect:
+    # If the following assertions fail, that is a hint, that the test file records.xml has changed
+    # and this needs to be updated:
     assert len(ident.get_records()) == 18
     assert len([r for r in ident.get_records() if r.parents[0].name == "Person"]) == 5
     assert len([r for r in ident.get_records() if r.parents[0].name == "Measurement"]) == 11
@@ -320,14 +267,6 @@ def test_identifiable_update3(crawler, ident):
     assert len(updl) == 2
 
 
-def test_identifiable_adapter():
-    query = IdentifiableAdapter.create_query_for_identifiable(
-        db.Record().add_parent("Person")
-        .add_property("first_name", value="A")
-        .add_property("last_name", value="B"))
-    assert query.lower() == "find record person with 'first_name'='a' and 'last_name'='b' "
-
-
 @pytest.mark.xfail
 def test_identifiable_adapter_no_identifiable(crawler, ident):
     del ident._registered_identifiables["Person"]