FIX: Fixed previously undetected updates due to property additions and removals.

integrationtests/test.py

@@ -132,7 +132,6 @@ def test_insertion(clear_database, usemodel, ident, crawler):
 def test_insertion_and_update(clear_database, usemodel, ident, crawler):
     ins, ups = crawler.synchronize()
 
-    # Do a second run on the same data, there should be no changes:
     cr = Crawler(debug=True, identifiableAdapter=ident)
     crawl_standard_test_directory(cr, "example_insert")
     ins, ups = cr.synchronize()
@@ -161,7 +160,7 @@ def test_identifiable_update(clear_database, usemodel, ident, crawler):
             record.add_property(name="email", value="testperson@testaccount.test")
             print("one change")
             break
-    ins, ups = cr.synchronize()
     breakpoint()
+    ins, ups = cr.synchronize()
     assert len(ins) == 0
     assert len(ups) == 1

src/newcrawler/crawl.py

@@ -419,8 +419,6 @@ class Crawler(object):
                                "same length!")
         # TODO this can now easily be changed to a function without side effect
         for i in reversed(range(len(updateList))):
-            if updateList[i].get_property("email") is not None:
-                breakpoint()
             comp = compare_entities(updateList[i], identified_records[i])
             identical = True
             for j in range(2):
@@ -432,6 +430,9 @@ class Crawler(object):
                 if not identical:
                     break
             for key in comp[0]["properties"]:
+                if len(comp[0]["properties"][key]) == 0:
+                    # This is a new property
+                    identical = False
                 for attribute in ("datatype", "importance", "unit"):
                     # only make an update for those attributes if there is a value difference and
                     # the value in the updateList is not None
@@ -449,6 +450,13 @@ class Crawler(object):
                 if not identical:
                     break
 
+            # Check for removed properties:
+            for key in comp[1]["properties"]:
+                if len(comp[1]["properties"][key]) == 0:
+                    # This is a removed property
+                    identical = False
+                    break
+
             if identical:
                 del updateList[i]
                 continue