F upate overwrite

Summary

Confirmation for #47 (closed) (https://gitlab.com/caosdb/caosdb-crawler/-/issues/23). Adds an xfailing test that should verify that crawler updates don't overwrite existing properties.

Focus

New integration test.

Test Environment

Start LinkAhead>=0.9 with the integrationtests/test-profile/profile.yml and run

cd integrationtests
pytest -vvv test_issues.py

and verify that it xfails correctly (you can also remove the xfail and check that name and prop_a are overwritten wrongly).

Check List for the Author

Please, prepare your MR for a review. Be sure to write a summary and a focus and create gitlab comments for the reviewer. They should guide the reviewer through the changes, explain your changes and also point out open questions. For further good practices have a look at our review guidelines

• All automated tests pass
• Reference related issues
• Up-to-date CHANGELOG.md (or not necessary)
• Annotations in code (Gitlab comments)
  • Intent of new code
  • Problems with old code
  • Why this implementation?

Check List for the Reviewer

• I understand the intent of this MR
• All automated tests pass
• Up-to-date CHANGELOG.md (or not necessary)
• The test environment setup works and the intended behavior is reproducible in the test environment
• In-code documentation and comments are up-to-date.
• Check: Are there specifications? Are they satisfied?

For further good practices have a look at our review guidelines.

integrationtests/test_issues.py

+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2022 Indiscale GmbH <info@indiscale.com>
+#               2022 Florian Spreckelsen <f.spreckelsen@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+from pytest import fixture, mark
+
+import caosdb as db
+
+from caoscrawler.crawl import Crawler
+from caoscrawler.identifiable_adapters import CaosDBIdentifiableAdapter
+from caoscrawler.structure_elements import Dict
+
+
+@fixture
+def clear_database():
+    db.execute_query("FIND ENTITY").delete(raise_exception_on_error=False)
+
+
+@mark.xfail(reason="See issue https://gitlab.com/caosdb/caosdb-crawler/-/issues/23")
+def test_issue_23(clear_database):
+    """Test that an update leaves existing properties, that were not found by
+    the crawler, unchanged.
+
+    """
+
+    # insert a simplistic model an arecord of type TestType with identifying
+    # property and prop_a, but not prop_b.
+    prop_ident = db.Property(name="identifying_prop", datatype=db.TEXT)
+    prop_a = db.Property(name="prop_a", datatype=db.TEXT)
+    prop_b = db.Property(name="prop_b", datatype=db.TEXT)
+    rt = db.RecordType(name="TestType")
+    rec = db.Record(name="TestRec").add_parent(rt)
+    rec.add_property(name="identifying_prop", value="identifier")
+    rec.add_property(name="prop_a", value="something")
+    db.Container().extend([prop_ident, prop_a, prop_b, rt, rec]).insert()
+
+    # set up crawler, first cfood defining a TestType record with
+    # identifying_prop and prop_b, but not prop_a ...
+    crawler_definition = {
+        "DictTest": {
+            "type": "Dict",
+            "match": "(.*)",
+            "records": {
+                "TestType": {}
+            },
+            "subtree": {
+                "identifying_element": {
+                    "type": "DictTextElement",
+                    "match_name": "ident",
+                    "match_value": "(?P<ident_value>.*)",
+                    "records": {
+                        "TestType": {
+                            "identifying_prop": "$ident_value"
+                        }
+                    }
+                },
+                "other_element": {
+                    "type": "DictTextElement",
+                    "match_name": "prop_b",
+                    "match_value": "(?P<other_value>.*)",
+                    "records": {
+                        "TestType": {
+                            "prop_b": "$other_value"
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    # register identifiable for TestType
+    ident = CaosDBIdentifiableAdapter()
+    ident.register_identifiable("TestType", db.RecordType().add_parent(
+        name="TestType").add_property(name="identifying_prop"))
+
+    crawler = Crawler(debug=True, identifiableAdapter=ident)
+    converter_registry = crawler.load_converters(crawler_definition)
+
+    # the dictionary to be crawled...
+    test_dict = {
+        "ident": "identifier",
+        "prop_b": "something_else"
+    }
+
+    records = crawler.start_crawling(
+        Dict("TestDict", test_dict), crawler_definition, converter_registry)
+
+    assert len(records) == 1
+    rec_crawled = records[0]
+    assert rec_crawled.parents[0].name == "TestType"
+    assert rec_crawled.get_property("identifying_prop") is not None
+    assert rec_crawled.get_property("identifying_prop").value == "identifier"
+    assert rec_crawled.get_property("prop_b") is not None
+    assert rec_crawled.get_property("prop_b").value == "something_else"
+    # no interaction with the database yet, so the rrecord shouldn't have a prop_a yet
+    assert rec_crawled.get_property("prop_a") is None
+
+    # synchronize with database and update the record
+    ins, ups = crawler.synchronize()
+    assert len(ins) == 0
+    assert len(ups) == 1
+
+    # retrieve and check that name and properties have been combined correctly
+    rec_retrieved = db.Record(id=rec.id).retrieve()
+    assert rec_retrieved.name == rec.name
+    assert rec_retrieved.get_property(
+        "identifying_prop").value == rec.get_property("identifying_prop").value
+    assert rec_retrieved.get_property(
+        "prop_a").value == rec.get_property("prop_a").value
+    assert rec_retrieved.get_property(
+        "identifying_prop").value == rec_crawled.get_property("identifying_prop").value
+    assert rec_retrieved.get_property(
+        "prop_b").value == rec_crawled.get_property("prop_b").value