fix problems from merge of versioning

patches/patch20200903-4.0.0-rc1/patch.sh

+#!/bin/bash
+#
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2018 Research Group Biomedical Physics,
+# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+
+# Refactors the entities table and move all names to name_data
+# Update mysql schema to version v3.0.0-rc1
+NEW_VERSION="v4.0.0-rc1"
+OLD_VERSION="v3.0.0-rc2"
+
+if [ -z "$UTILSPATH" ]; then
+ UTILSPATH="../utils"
+fi
+
+. $UTILSPATH/patch_header.sh $*
+
+
+check_version $OLD_VERSION
+
+mysql_execute_file $PATCH_DIR/patch.sql
+
+update_version $NEW_VERSION
+
+success
+

patches/patch20200903-4.0.0-rc1/patch.sql

+-- USE _caosdb_schema_unit_tests;
+
+
+-- REMOVE SQLITE datatype
+DELETE FROM name_data WHERE entity_id = 50;
+DELETE FROM entities WHERE id = 50;
+
+ALTER TABLE files ADD UNIQUE (`path`);
+
+
+ALTER TABLE files ADD COLUMN IF NOT EXISTS (
+    mimetype VARBINARY(255) DEFAULT NULL,
+    fileStorage VARBINARY(255) NOT NULL DEFAULT "DEFAULT",
+    storageId VARBINARY(255) DEFAULT NULL
+);
+
+ALTER TABLE archive_files ADD COLUMN IF NOT EXISTS (
+    mimetype VARBINARY(255) DEFAULT NULL,
+    fileStorage VARBINARY(255) NOT NULL DEFAULT "DEFAULT",
+    storageId VARBINARY(255) DEFAULT NULL,
+    checked_timestamp BIGINT DEFAULT NULL
+);
+
+
+ALTER TABLE entities MODIFY COLUMN `role` enum('RECORDTYPE','RECORD','FILE','DOMAIN','PROPERTY','DATATYPE','ROLE','QUERYTEMPLATE', 'DIRECTORY') COLLATE utf8_unicode_ci NOT NULL;
+
+INSERT INTO entities (id, description, role, acl) VALUES (9, "The directory role.", "ROLE", 0);
+INSERT INTO name_data (domain_id, entity_id, property_id, value, status, pidx) VALUES (0, 9, 20, "DIRECTORY", "FIX", 0);
+
+UPDATE files SET storageId=path;
+
+ALTER TABLE files MODIFY COLUMN storageId VARBINARY(255) NOT NULL;
+
+DROP PROCEDURE IF EXISTS _create_dirs;
+DELIMITER //
+CREATE PROCEDURE _create_dirs()
+BEGIN
+    DECLARE done BOOLEAN DEFAULT FALSE;
+    DECLARE _file_id INT UNSIGNED DEFAULT NULL;
+    DECLARE _path VARCHAR(255) DEFAULT NULL;
+    DECLARE dir_id INT UNSIGNED DEFAULT NULL;
+    DECLARE dir_path VARCHAR(255) DEFAULT "";
+    DECLARE dir_exists BOOLEAN DEFAULT FALSE;
+    DECLARE cur CURSOR FOR SELECT file_id, path FROM files;
+    DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = TRUE;
+    OPEN cur;
+
+    loop1: LOOP
+        FETCH cur INTO _file_id, _path;
+        IF done THEN
+            LEAVE loop1;
+        END IF;
+
+        -- TODO
+        loop2: LOOP
+            SELECT REGEXP_SUBSTR(_path, CONCAT("(?U)^", dir_path, ".*/" )) INTO dir_path;
+            IF dir_path = "" THEN
+                SET dir_exists = FALSE;
+                SET done = FALSE;
+                LEAVE loop2;
+            END IF;
+            SELECT TRUE INTO dir_exists FROM files WHERE path = LEFT(dir_path, CHAR_LENGTH(dir_path)-1);
+            IF dir_exists IS TRUE THEN
+                SET dir_exists = FALSE;
+                ITERATE loop2;
+            END IF;
+            INSERT INTO entities (description, role, acl) VALUES (NULL, "DIRECTORY", 0);
+            SET dir_id = LAST_INSERT_ID();
+            INSERT INTO files (file_id, path, size, hash, checked_timestamp,
+                    mimetype, fileStorage, storageId)
+                VALUES (dir_id, LEFT(dir_path, CHAR_LENGTH(dir_path)-1), 0,
+                    NULL, 0, "inode/directory", "DEFAULT", LEFT(dir_path,
+                        CHAR_LENGTH(dir_path)-1));
+        END LOOP;
+
+    END LOOP;
+
+    CLOSE cur;
+
+END //
+
+DELIMITER ;
+
+CALL _create_dirs();

procedures/deleteIsaCache.sql

@@ -55,7 +55,7 @@ BEGIN
             SELECT e.child, IVersion AS child_iversion, e.parent
             FROM isa_cache AS e
             WHERE e.child = EntityID
-            AND e.rpath = CAST(EntityID AS CHAR);
+            AND e.rpath = CAST(EntityID AS CHAR) COLLATE utf8_unicode_ci;
     END IF;
 
     DELETE FROM isa_cache

procedures/entityVersioning.sql

@@ -399,7 +399,11 @@ CREATE PROCEDURE setFileProperties (
     in EntityID INT UNSIGNED,
     in FilePath TEXT,
     in FileSize BIGINT UNSIGNED,
-    in FileHash VARCHAR(255)
+    in FileHash VARCHAR(255),
+    in FileChecked BIGINT,
+    in FileMimetype VARBINARY(255),
+    in FileStorage VARBINARY(255),
+    in FileStorageId VARBINARY(65525)
 )
 BEGIN
     DECLARE IVersion INT UNSIGNED DEFAULT NULL;
@@ -409,8 +413,10 @@ BEGIN
             WHERE e.entity_id = EntityID;
 
         INSERT INTO archive_files (file_id, path, size, hash,
+                checked_timestamp, mimetype, fileStorage, storageId,
                 _iversion)
-            SELECT file_id, path, size, hash, IVersion AS _iversion
+            SELECT file_id, path, size, hash, checked_timestamp, mimetype,
+                fileStorage, storageId, IVersion AS _iversion
             FROM files
             WHERE file_id = EntityID;
     END IF;
@@ -418,8 +424,10 @@ BEGIN
     DELETE FROM files WHERE file_id = EntityID;
 
     IF FilePath IS NOT NULL THEN
-        INSERT INTO files (file_id, path, size, hash)
-            VALUES (EntityID, FilePath, FileSize, unhex(FileHash));
+        INSERT INTO files (file_id, path, size, hash, checked_timestamp,
+                mimetype, fileStorage, storageId)
+            VALUES (EntityID, FilePath, FileSize, unhex(FileHash), FileChecked,
+                FileMimetype, FileStorage, FileStorageId);
     END IF;
 
 END //

procedures/insertEntity.sql

@@ -46,7 +46,12 @@ Select
 
 A tuple (EntityID, Version)
 */
-CREATE PROCEDURE db_2_0.insertEntity(in EntityName VARCHAR(255), in EntityDesc TEXT, in EntityRole VARCHAR(255), in ACL VARBINARY(65525))
+CREATE PROCEDURE db_2_0.insertEntity(
+    in EntityName VARCHAR(255),
+    in EntityDesc TEXT,
+    in EntityRole VARCHAR(255),
+    in ACL VARBINARY(65525)
+)
 BEGIN
     DECLARE NewEntityID INT UNSIGNED DEFAULT NULL;
     DECLARE NewACLID INT UNSIGNED DEFAULT NULL;

tests/test_reference_values.sql

@@ -31,18 +31,20 @@ CALL tap.no_plan();
 -- #####################################################################
 
 -- SETUP
+-- switch off versioning
+DELETE FROM feature_config WHERE _key = "ENTITY_VERSIONING";
+
 DELETE FROM entity_version;
 DELETE FROM transactions;
 INSERT INTO transactions (srid,seconds,nanos,username,realm) VALUES
 ("SRIDbla", 1234, 2345, "me", "home"),
 ("SRIDblub", 2345, 3465, "me", "home"),
 ("SRIDblieb", 3456, 4576, "you", "home");
+SET @SRID = "SRIDbla";
+CALL insertEntity("EntityName", "EntityDescription", "RECORDTYPE", "{}");
 SET @EntityID=99;
 SET @PropertyID=11;
-SET @Value=50;
-
--- switch off versioning
-DELETE FROM feature_config WHERE _key = "ENTITY_VERSIONING";
+SELECT id INTO @Value FROM entities WHERE description = "EntityDescription";
 
 -- TEST insertEntityProperty without Versioning
 CALL insertEntityProperty(0, @EntityID, @PropertyID, "reference_data", @Value,
@@ -52,7 +54,7 @@ CALL insertEntityProperty(0, @EntityID, @PropertyID, "reference_data", @Value,
 SELECT tap.eq(0, domain_id, "domain ok") FROM reference_data;
 SELECT tap.eq(99, entity_id, "entity ok") FROM reference_data;
 SELECT tap.eq(11, property_id, "property ok") FROM reference_data;
-SELECT tap.eq(50, value, "value ok") FROM reference_data;
+SELECT tap.eq(@VALUE, value, "value ok") FROM reference_data;
 SELECT tap.eq("FIX", status, "status ok") FROM reference_data;
 SELECT tap.eq("0", pidx, "pidx ok") FROM reference_data;
 SELECT tap.eq(NULL, value_iversion, "value_iversion ok") FROM reference_data;
@@ -69,7 +71,7 @@ INSERT INTO feature_config (_key, _value) VALUES ("ENTITY_VERSIONING", "ENABLED"
 
 
 -- TEST insertEntityProperty with Versioning - REFERENCE HEAD
-SET @VALUE="50";
+SELECT id INTO @Value FROM entities WHERE description = "EntityDescription";
 CALL insertEntityProperty(0, @EntityID, @PropertyID, "reference_data", @Value,
     NULL, "FIX", NULL, NULL, NULL, NULL, 0);
 
@@ -77,7 +79,7 @@ CALL insertEntityProperty(0, @EntityID, @PropertyID, "reference_data", @Value,
 SELECT tap.eq(0, domain_id, "domain ok") FROM reference_data;
 SELECT tap.eq(99, entity_id, "entity ok") FROM reference_data;
 SELECT tap.eq(11, property_id, "property ok") FROM reference_data;
-SELECT tap.eq(50, value, "value ok") FROM reference_data;
+SELECT tap.eq(@Value, value, "value ok") FROM reference_data;
 SELECT tap.eq("FIX", status, "status ok") FROM reference_data;
 SELECT tap.eq("0", pidx, "pidx ok") FROM reference_data;
 SELECT tap.eq(value_iversion, NULL, "value_iversion ok") FROM reference_data;
@@ -86,47 +88,51 @@ DELETE FROM reference_data WHERE domain_id=0 AND entity_id=99;
 
 
 -- TEST insertEntityProperty with Versioning - Reference version
-CALL insert_single_child_version(50, "hashbla", "versionbla", NULL, "SRIDbla");
-CALL insert_single_child_version(50, "hashblub", "versionblub", "versionbla", "SRIDblub");
+SELECT id INTO @Value FROM entities WHERE description = "EntityDescription";
+CALL insert_single_child_version(@Value, "hashbla", "versionbla", NULL, "SRIDbla");
+CALL insert_single_child_version(@Value, "hashblub", "versionblub", "versionbla", "SRIDblub");
 
-SET @VALUE="50@versionbla";
-CALL insertEntityProperty(0, @EntityID, @PropertyID, "reference_data", @Value,
-    NULL, "FIX", NULL, NULL, NULL, NULL, 0);
+
+SET @VersionedValue=CONCAT(@Value, "@versionbla");
+CALL insertEntityProperty(0, @EntityID, @PropertyID, "reference_data",
+    @VersionedValue, NULL, "FIX", NULL, NULL, NULL, NULL, 0);
 
 SELECT tap.eq(0, domain_id, "domain ok") FROM reference_data;
 SELECT tap.eq(99, entity_id, "entity ok") FROM reference_data;
 SELECT tap.eq(11, property_id, "property ok") FROM reference_data;
-SELECT tap.eq(50, value, "value ok") FROM reference_data;
+SELECT tap.eq(@Value, value, "value ok") FROM reference_data;
 SELECT tap.eq("FIX", status, "status ok") FROM reference_data;
 SELECT tap.eq("0", pidx, "pidx ok") FROM reference_data;
 SELECT tap.eq(value_iversion, "1", "value_iversion ok") FROM reference_data;
 
 DELETE FROM reference_data WHERE domain_id=0 AND entity_id=99;
 
-SET @VALUE="50@versionblub";
-CALL insertEntityProperty(0, @EntityID, @PropertyID, "reference_data", @Value,
-    NULL, "FIX", NULL, NULL, NULL, NULL, 0);
+SET @VersionedValue=CONCAT(@Value, "@versionblub");
+CALL insertEntityProperty(0, @EntityID, @PropertyID, "reference_data",
+    @VersionedValue, NULL, "FIX", NULL, NULL, NULL, NULL, 0);
 
 -- TODO switch expected/actual
 SELECT tap.eq(0, domain_id, "domain ok") FROM reference_data;
 SELECT tap.eq(99, entity_id, "entity ok") FROM reference_data;
 SELECT tap.eq(11, property_id, "property ok") FROM reference_data;
-SELECT tap.eq(50, value, "value ok") FROM reference_data;
+SELECT tap.eq(@Value, value, "value ok") FROM reference_data;
 SELECT tap.eq("FIX", status, "status ok") FROM reference_data;
 SELECT tap.eq("0", pidx, "pidx ok") FROM reference_data;
 SELECT tap.eq(value_iversion, "2", "value_iversion ok") FROM reference_data;
 
 
 -- invalid values throw errors
-SET @VALUE="50@";
+SET @VersionedValue=CONCAT(@Value, "@");
 CALL tap._assert_throws('
-    CALL insertEntityProperty(0, @EntityID, @PropertyID, "reference_data", @Value,
-        NULL, "FIX", NULL, NULL, NULL, NULL, 0)', "@ but no version id");
+    CALL insertEntityProperty(0, @EntityID, @PropertyID, "reference_data",
+        @VersionedValue, NULL, "FIX", NULL, NULL, NULL, NULL, 0)', "@ but no
+    version id");
 
-SET @VALUE="50@non-existing-version";
+SET @VersionedValue=CONCAT(@Value, "@non-existing-version");
 CALL tap._assert_throws('
-    CALL insertEntityProperty(0, @EntityID, @PropertyID, "reference_data", @Value,
-        NULL, "FIX", NULL, NULL, NULL, NULL, 0)', "non-existing-version id");
+    CALL insertEntityProperty(0, @EntityID, @PropertyID, "reference_data",
+        @VersionedValue, NULL, "FIX", NULL, NULL, NULL, NULL, 0)',
+    "non-existing-version id");
 
 
 -- #####################################################################