Merge branch 'f-query-version' of gitlab.com:caosdb/caosdb-mysqlbackend into f-query-version

doc/table_structure.md

@@ -101,7 +101,10 @@ There are a few extra tables for implementing versioning.  Also there is a speci
 `value_iversion` in the `reference_data` table which stores the version of the referenced entity at
 the time of the version.  If `value_iversion` is NULL, no specific version is stored.
 
-### transactions ###
+The `_iversion` is an incremental version counter, starting implicitly at 1.  Previous versions of
+entities are stored, along with their `_iversion`, in the `archive_*` tables.
+
+### `transactions` ###
 
 Replacement for `transaction_log`, holds ID, user and time of transaction:
 
@@ -123,15 +126,15 @@ Versioning info for entities:
 - `_ipparent` :: Primary parent ID (internal) for this version, i.e. predecessor
 - `srid` :: Server request / transaction ID which created this version
 
-### `archive_foo_data` ###
+### `archive_footable` ###
 
-Older (i.e. not current) data value, from previous versions of an entity.
-- `domain_id` :: Same as in `foo_data`.
-- `entity_id` :: Same as in `foo_data`.
-- `property_id` :: Same as in `foo_data`.
+Older (i.e. not current) data, from previous versions of an entity which were stored in `footable`.
+- `domain_id` :: Same as in `footable`.
+- `entity_id` :: Same as in `footable`.
+- `property_id` :: Same as in `footable`.
 - `value` :: The value at the given version.
 - `status` :: The value at the given version.
-- `pidx` :: Same as in `foo_data`.
+- `pidx` :: Same as in `footable`.
 - `_iversion` :: Version index of this version.
 - ... :: Data type specific columns may also be there.
 

patches/patch20201110-4.0.0-rc1/patch.sh

@@ -36,7 +36,11 @@ fi
 
 check_version $OLD_VERSION
 
-
+# * Add a `direct` column to `archive_isa` table
+# * Remove unused procedures:
+#    * `initNegationFilter`
+#    * `initConjunctionFilter`
+#    * `finishNegationFilter`
 mysql_execute_file $PATCH_DIR/update_archive_isa.sql
 
 

procedures/query/applyBackReference.sql

@@ -4,6 +4,9 @@
  *
  * Copyright (C) 2018 Research Group Biomedical Physics,
  * Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+ * Copyright (C) 2020 Indiscale GmbH <info@indiscale.com>
+ * Copyright (C) 2020 Timm Fitschen <t.fitschen@indiscale.com>
+ * Copyright (C) 2020 Daniel Hornung <d.hornung@indiscale.com>
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as
@@ -24,7 +27,44 @@
 DROP PROCEDURE IF EXISTS db_2_0.applyBackReference;
 DELIMITER //
 
-CREATE PROCEDURE db_2_0.applyBackReference(in sourceSet VARCHAR(255), targetSet VARCHAR(255), in propertiesTable VARCHAR(255), in entitiesTable VARCHAR(255), in subQuery BOOLEAN, in versioned BOOLEAN)
+/**
+ * Process a set of entities, keeping those which are referenced in a specific way.
+ *
+ * This procedure filters a source table and writes the result is written into a target table.  If
+ * the target table is not given, all non-matching entries are deleted from the source table
+ * instead.
+ *
+ * Candidate entities are allowed to pass if there is an entity from the entities table which
+ * references the candidate as one of the properties in the properties table.
+ *
+ * Parameters
+ * ----------
+ * sourceSet : table
+ * The name of the table which shall be filtered, must have column `id`.
+ *
+ * targetSet : table
+ * The name of the result table, must have column `id`.
+ *
+ * propertiesTable : table
+ * References as Properties in this table are counted.
+ *
+ * entitiesTable : table
+ * References by Entities in this table are counted.
+ *
+ * subQuery : boolean
+ * Create a temporary target table and select as `list`, instead of using the given targetSet.  The
+ * `versioned` parameter has no effect in this case.
+ *
+ * versioned : boolean
+ * If True, if a reference is versioned (references to specific versions of entities), the target
+ * candidate's version must match.  Therefore, the sourceSet and targetSet must have a `_iversion`
+ * column as well (unless sourceSet is the `entities` table).  If `sourceSet` is the `entities`
+ * table, unversioned references will result in all versions of a referenced candidate being returned.
+ *
+ */
+CREATE PROCEDURE db_2_0.applyBackReference(in sourceSet VARCHAR(255), targetSet VARCHAR(255),
+    in propertiesTable VARCHAR(255), in entitiesTable VARCHAR(255), in subQuery BOOLEAN,
+    in versioned BOOLEAN)
 BEGIN
     DECLARE newTableName VARCHAR(255) DEFAULT NULL;
 
@@ -38,7 +78,23 @@ BEGIN
         EXECUTE createBackRefSubQueryTable;
         DEALLOCATE PREPARE createBackRefSubQueryTable;
 
-        SET @backRefSubResultSetStmtStr = CONCAT('INSERT IGNORE INTO `',newTableName,'` (id,entity_id) SELECT entity_id AS id, value AS entity_id FROM `reference_data` AS data WHERE EXISTS (SELECT 1 FROM `',sourceSet,'` AS source WHERE source.id=data.value LIMIT 1)', IF(propertiesTable IS NULL,'',CONCAT(' AND EXISTS (SELECT 1 FROM `',propertiesTable,'` AS p WHERE p.id=data.property_id LIMIT 1)')), IF(entitiesTable IS NULL,'',CONCAT(' AND EXISTS (SELECT 1 FROM `',entitiesTable,'` AS e WHERE e.id=data.entity_id LIMIT 1)')));
+        SET @backRefSubResultSetStmtStr = CONCAT('INSERT IGNORE INTO `',
+            newTableName,
+            '` (id,entity_id) SELECT entity_id AS id, value AS entity_id FROM `reference_data` AS data ',
+            'WHERE EXISTS (SELECT 1 FROM `',
+                sourceSet,
+                '` AS source WHERE source.id=data.value LIMIT 1)',
+            IF(propertiesTable IS NULL,
+                '',
+                CONCAT(' AND EXISTS (SELECT 1 FROM `',
+                    propertiesTable,
+                    '` AS p WHERE p.id=data.property_id LIMIT 1)')),
+            IF(entitiesTable IS NULL,
+                '',
+                CONCAT(' AND EXISTS (SELECT 1 FROM `',
+                    entitiesTable,
+                    '` AS e WHERE e.id=data.entity_id LIMIT 1)'))
+        );
 
         PREPARE backRefSubResultSetStmt FROM @backRefSubResultSetStmtStr;
         EXECUTE backRefSubResultSetStmt;
@@ -48,9 +104,14 @@ BEGIN
     ELSE
         IF versioned THEN
             IF sourceSet = "entities" THEN
+                -- Find any referenced entity, current or archived
                 SET @stmtBackRefStr = CONCAT('INSERT IGNORE INTO `',
                     targetSet,
-                    '` (id, _iversion) SELECT source.id, _get_head_iversion(source.id) FROM entities AS source WHERE EXISTS (SELECT 1 FROM `reference_data` AS data WHERE data.value=source.id AND ( data.value_iversion IS NULL OR data.value_iversion=_get_head_iversion(source.id))',
+                    '` (id, _iversion) SELECT source.id, _get_head_iversion(source.id)',
+                    -- current entities
+                    ' FROM entities AS source WHERE EXISTS (',
+                        'SELECT 1 FROM `reference_data` AS data WHERE data.value=source.id AND (',
+                            'data.value_iversion IS NULL OR data.value_iversion=_get_head_iversion(source.id))',
                     IF(entitiesTable IS NULL,
                         '',
                         CONCAT(' AND EXISTS (SELECT 1 FROM `',
@@ -61,7 +122,11 @@ BEGIN
                         CONCAT(' AND EXISTS (SELECT 1 FROM `',
                             propertiesTable,
                             '` AS p WHERE p.id=data.property_id LIMIT 1)')),
-                    ') UNION ALL SELECT source.id, source._iversion FROM archive_entities AS source WHERE EXISTS (SELECT 1 FROM `reference_data` AS data WHERE data.value=source.id AND ( data.value_iversion IS NULL OR data.value_iversion=source._iversion)',
+                    ') UNION ALL ',
+                    -- and archived entities
+                    'SELECT source.id, source._iversion FROM archive_entities AS source WHERE EXISTS (',
+                        'SELECT 1 FROM `reference_data` AS data WHERE data.value=source.id AND ',
+                          '(data.value_iversion IS NULL OR data.value_iversion=source._iversion)',
                     IF(entitiesTable IS NULL,
                         '',
                         CONCAT(' AND EXISTS (SELECT 1 FROM `',
@@ -98,7 +163,9 @@ BEGIN
                     targetSet,
                     '` (id, _iversion) SELECT source.id, source._iversion FROM `',
                     sourceSet,
-                    '` AS source WHERE EXISTS (SELECT 1 FROM `reference_data` AS data WHERE data.value=source.id AND ( data.value_iversion IS NULL OR data.value_iversion=_get_head_iversion(source.id))',
+                    '` AS source WHERE EXISTS (',
+                    'SELECT 1 FROM `reference_data` AS data WHERE data.value=source.id AND',
+                    ' (data.value_iversion IS NULL OR data.value_iversion=source._iversion)',
                     IF(entitiesTable IS NULL,
                         '',
                         CONCAT(' AND EXISTS (SELECT 1 FROM `',
@@ -115,6 +182,7 @@ BEGIN
         ELSE
             -- unversioned queries
             IF targetSet IS NULL OR sourceSet = targetSet THEN
+                -- delete from sourceSet
                 SET @stmtBackRefStr = CONCAT('DELETE FROM `',
                     sourceSet,
                     '` WHERE NOT EXISTS (SELECT 1 FROM `reference_data` AS data WHERE data.value=`',
@@ -134,6 +202,7 @@ BEGIN
                             '` AS p WHERE p.id=data.property_id LIMIT 1)')),
                     ')');
             ELSE
+                -- write to targetSet
                 SET @stmtBackRefStr = CONCAT('INSERT IGNORE INTO `',
                     targetSet,
                     '` (id) SELECT id FROM `',