BUG: replacement ids interfering with external ids

Summary

Fixes several issues with the flattening of deeply nested properties, the correct storage of data types and minor issues. Most of these issues have been revealed due to the changes in the back end and probably originate from the major refactoring of the server code in the 0.11.0 release (external/internal ids).

Corresponding back end changes: caosdb-mysqlbackend!21 (merged)

Related: caosdb-mysqlbackend#48 (closed)

Focus

The fix of the original issue happened in the back end. However, the fix revealed another bug which had to be fixed in the back end and the server (concerning the property's data type name and id) and another bug in the server (DatabaseUtils) which failed to implement the flattening/de-flattening of the entity properties correctly. This lead to failing integrations tests because the $ prefix of internal ids was leaked into the server's xml response.

Test Environment

See caosdb-mysqlbackend!21 (merged)

Check List for the Author

Please, prepare your MR for a review. Be sure to write a summary and a focus and create gitlab comments for the reviewer. They should guide the reviewer through the changes, explain your changes and also point out open questions. For further good practices have a look at our review guidelines

• All automated tests pass
• Reference related issues
• Up-to-date CHANGELOG.md (or not necessary)
• Up-to-date JSON schema (or not necessary)
• Appropriate user and developer documentation (or not necessary)
  • How do I use the software? Assume "stupid" users.
  • How do I develop or debug the software? Assume novice developers.
• Annotations in code (Gitlab comments)
  • Intent of new code
  • Problems with old code
  • Why this implementation?

Check List for the Reviewer

• I understand the intent of this MR
• All automated tests pass
• Up-to-date CHANGELOG.md (or not necessary)
• Appropriate user and developer documentation (or not necessary)
• The test environment setup works and the intended behavior is reproducible in the test environment
• In-code documentation and comments are up-to-date.
• Check: Are there specifications? Are they satisfied?

For further good practices have a look at our review guidelines.

src/main/java/org/caosdb/server/database/backend/implementation/MySQL/DatabaseUtils.java

@@ -87,19 +87,19 @@ public class DatabaseUtils {
 
   private static void processPropertiesStage1(
       final List<Property> stage1Inserts, final Property p, final EntityInterface e) {
-
     if (!p.isDescOverride()
         && !p.isNameOverride()
         && !p.isDatatypeOverride()
         && (!p.hasProperties() || hasUniquePropertyId(p, e))
         && !(p.getDatatype() instanceof AbstractCollectionDatatype)) {
       stage1Inserts.add(p);
+      processSubPropertiesStage1(stage1Inserts, p);
     } else {
       Replacement r = new Replacement(p);
       stage1Inserts.add(r);
       stage1Inserts.add(r.replacement);
+      processSubPropertiesStage1(stage1Inserts, r);
     }
-    processSubPropertiesStage1(stage1Inserts, p);
   }
 
   public static void deriveStage2Inserts(
@@ -109,19 +109,17 @@ public class DatabaseUtils {
       EntityInterface entity) {
     for (final Property p : stage1Inserts) {
       if (p instanceof Replacement) {
-
-        EntityID replacementId = replacementIds.pop();
-        ((Replacement) p).setReplacementId(replacementId);
-
-        if (p.hasProperties()) {
-          for (Property subP : p.getProperties()) {
-            subP.setDomain(p);
-            stage2Inserts.add(subP);
-          }
+        if (!p.hasId() || p.getId().isTemporary()) {
+          EntityID replacementId = replacementIds.pop();
+          ((Replacement) p).setReplacementId(replacementId);
         }
-      } else {
-        if (p.hasProperties()) {
-          for (Property subP : p.getProperties()) {
+      }
+      if (p.hasProperties()) {
+        if (p instanceof Replacement && ((Replacement) p).isStage2Replacement()) {
+          stage2Inserts.add(((Replacement) p).replacement);
+        }
+        for (Property subP : p.getProperties()) {
+          if (!subP.hasProperties()) {
             stage2Inserts.add(subP);
           }
         }
@@ -142,9 +140,12 @@ public class DatabaseUtils {
   private static void processSubPropertiesStage1(
       final List<Property> stage1Inserts, final EntityInterface p) {
     for (final Property subP : p.getProperties()) {
+      subP.setDomain(p);
       if (subP.hasProperties()) {
-        stage1Inserts.add(new Replacement(subP));
-        processSubPropertiesStage1(stage1Inserts, subP);
+        Replacement r = new Replacement(subP);
+        r.setStage2Replacement(true);
+        stage1Inserts.add(r);
+        processSubPropertiesStage1(stage1Inserts, r);
       }
     }
   }
@@ -164,9 +165,10 @@ public class DatabaseUtils {
           if (desc != null) {
             p.desc = desc;
           }
-          final String type = bytes2UTF8(rs.getBytes("type_override"));
-          if (type != null) {
-            p.type = type;
+          final String type_id = bytes2UTF8(rs.getBytes("type_id_override"));
+          if (type_id != null) {
+            p.type_id = type_id;
+            p.type_name = bytes2UTF8(rs.getBytes("type_name_override"));
           }
           final String coll = bytes2UTF8(rs.getBytes("collection_override"));
           if (coll != null) {
@@ -221,7 +223,8 @@ public class DatabaseUtils {
     final SparseEntity ret = parseNameRoleACL(rs);
     ret.id = rs.getString("EntityID");
     ret.description = bytes2UTF8(rs.getBytes("EntityDesc"));
-    ret.datatype = bytes2UTF8(rs.getBytes("Datatype"));
+    ret.datatype_id = bytes2UTF8(rs.getBytes("DatatypeID"));
+    ret.datatype_name = bytes2UTF8(rs.getBytes("DatatypeName"));
     ret.collection = bytes2UTF8(rs.getBytes("Collection"));
 
     ret.filePath = bytes2UTF8(rs.getBytes("FilePath"));
@@ -300,6 +303,14 @@ public class DatabaseUtils {
       if (pp.status.equals(ReplacementStatus.REPLACEMENT.name())) {
         replacements.put(pp.value, null);
       }
+      if (pp.subProperties != null) {
+        for (ProtoProperty subP : pp.subProperties) {
+          if (subP.status.equals(ReplacementStatus.REPLACEMENT.name())) {
+
+            replacements.put(subP.value, null);
+          }
+        }
+      }
     }
     Iterator<ProtoProperty> iterator = properties.iterator();
     while (iterator.hasNext()) {
@@ -325,6 +336,13 @@ public class DatabaseUtils {
       if (pp.status.equals(ReplacementStatus.REPLACEMENT.name())) {
         replace(pp, replacements.get(pp.value), isHead);
       }
+      if (pp.subProperties != null) {
+        for (ProtoProperty subP : pp.subProperties) {
+          if (subP.status.equals(ReplacementStatus.REPLACEMENT.name())) {
+            replace(subP, replacements.get(subP.value), isHead);
+          }
+        }
+      }
       if (pp.collValues != null) {
         // sort
         pp.collValues.sort(
@@ -358,7 +376,8 @@ public class DatabaseUtils {
     }
     pp.desc = replacement.desc;
     pp.name = replacement.name;
-    pp.type = replacement.type;
+    pp.type_id = replacement.type_id;
+    pp.type_name = replacement.type_name;
     pp.collection = replacement.collection;
     pp.value = replacement.value;
     pp.collValues = replacement.collValues;