WIP: file system refactoring

make_db

@@ -51,12 +51,13 @@ function _execute_tests () {
     for tfile in $TESTS ; do
         echo "Running $tfile"
         echo "----- $tfile -----" >> .TEST_RESULTS
-        cat $tfile | $SQL --disable-pager --batch --raw --skip-column-names --unbuffered >> .TEST_RESULTS
+        cat $tfile | $SQL --disable-pager --batch --raw --skip-column-names --unbuffered >> .TEST_RESULTS 2>&1
     done;
 
     popd
     cat tests/.TEST_RESULTS
     grep -c -i "failed" tests/.TEST_RESULTS > /dev/null && return 1
+    grep -c -i "ERROR" tests/.TEST_RESULTS > /dev/null && return 1
     return 0
 }
 

procedures/retrieveEntity.sql

@@ -29,13 +29,16 @@ drop procedure if exists db_2_0.retrieveEntity //
 
 create procedure db_2_0.retrieveEntity(in EntityID INT UNSIGNED)
 BEGIN
-	DECLARE FilePath VARCHAR(255) DEFAULT NULL;
-	DECLARE FileSize VARCHAR(255) DEFAULT NULL;
+	DECLARE FilePath VARBINARY(65525) DEFAULT NULL;
+	DECLARE FileSize BIGINT UNSIGNED DEFAULT NULL;
 	DECLARE FileHash VARCHAR(255) DEFAULT NULL;
+    DECLARE FileMimetype VARBINARY(255) DEFAULT NULL;
+    DECLARE FileStorage VARBINARY(255) DEFAULT NULL;
+    DECLARE FileStorageId VARBINARY(65525) DEFAULT NULL;
 	DECLARE DatatypeID INT UNSIGNED DEFAULT NULL;
     DECLARE CollectionName VARCHAR(255) DEFAULT NULL;
 	
-	Select path, size, hex(hash) into FilePath, FileSize, FileHash from files where file_id = EntityID LIMIT 1;
+	Select path, size, hex(hash), mimetype, fileStorage, storageId into FilePath, FileSize, FileHash, FileMimetype, FileStorage, FileStorageId FROM files WHERE file_id = EntityID LIMIT 1;
 	Select datatype into DatatypeID from data_type where domain_id=0 and entity_id=0 and property_id=EntityID LIMIT 1;
 
 	SELECT collection into CollectionName from collection_type where domain_id=0 and entity_id=0 and property_id=EntityID LIMIT 1;
@@ -50,6 +53,9 @@ BEGIN
 		FileSize as FileSize, 
 		FilePath as FilePath, 
 		FileHash as FileHash,
+        FileMimetype as FileMimetype,
+        FileStorage as FileStorage,
+        FileStorageId as FileStorageId,
 		(SELECT acl FROM entity_acl as a WHERE a.id = e.acl) as ACL
 	from entities e where id = EntityID LIMIT 1;
 END;

rules.sql

@@ -149,8 +149,3 @@ INSERT INTO rules (domain_id, entity_id, transaction, criterion, modus) VALUES (
 INSERT INTO rules (domain_id, entity_id, transaction, criterion, modus) VALUES (0,17,'UPDATE','CheckRefidIsaParRefid','MUST');
 #-- INSERT INTO rules (domain_id, entity_id, transaction, criterion, modus) VALUES (0,17,'UPDATE','CheckDescPresent','MUST');
 
-#-- SQLite files
-INSERT INTO rules (domain_id, entity_id, transaction, criterion, modus) VALUES (0,50,'UPDATE','CheckRefidValid','MUST');
-INSERT INTO rules (domain_id, entity_id, transaction, criterion, modus) VALUES (0,50,'UPDATE','CheckRefidIsaParRefid','SHOULD');
-#-- INSERT INTO rules (domain_id, entity_id, transaction, criterion, modus) VALUES (0,50,'UPDATE','CheckDescPresent','SHOULD');
-INSERT INTO rules (domain_id, entity_id, transaction, criterion, modus) VALUES (0,50,'UPDATE','SQLiteTransaction','MUST');

tests/test_0_next_patch.sql

+-- USE _caosdb_schema_unit_tests;
+
+
+-- REMOVE SQLITE datatype
+DELETE FROM entities WHERE id = 50;
+
+ALTER TABLE files ADD UNIQUE (`path`);
+
+ALTER TABLE files ADD COLUMN IF NOT EXISTS (
+    mimetype VARBINARY(255) DEFAULT NULL,
+    fileStorage VARBINARY(255) NOT NULL DEFAULT "DEFAULT",
+    storageId VARBINARY(255) DEFAULT NULL
+);
+
+
+ALTER TABLE entities MODIFY COLUMN `role` enum('RECORDTYPE','RECORD','FILE','DOMAIN','PROPERTY','DATATYPE','ROLE','QUERYTEMPLATE', 'DIRECTORY') COLLATE utf8_unicode_ci NOT NULL;
+
+INSERT IGNORE INTO entities (id, name, description, role, acl) VALUES (9, "DIRECTORY", "The directory role.", "ROLE", 0);
+
+UPDATE files SET storageId=path;
+
+ALTER TABLE files MODIFY COLUMN storageId VARBINARY(255) NOT NULL;
+
+DROP PROCEDURE IF EXISTS _create_dirs;
+DELIMITER //
+CREATE PROCEDURE _create_dirs()
+BEGIN
+    DECLARE done BOOLEAN DEFAULT FALSE;
+    DECLARE _file_id INT UNSIGNED DEFAULT NULL;
+    DECLARE _path VARCHAR(255) DEFAULT NULL;
+    DECLARE dir_id INT UNSIGNED DEFAULT NULL;
+    DECLARE dir_path VARCHAR(255) DEFAULT "";
+    DECLARE dir_exists BOOLEAN DEFAULT FALSE;
+    DECLARE cur CURSOR FOR SELECT file_id, path FROM files;
+    DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = TRUE;
+    OPEN cur;
+
+    loop1: LOOP
+        FETCH cur INTO _file_id, _path;
+        IF done THEN
+            LEAVE loop1;
+        END IF;
+
+        -- TODO
+        loop2: LOOP
+            SELECT REGEXP_SUBSTR(_path, CONCAT("(?U)^", dir_path, ".*/" )) INTO dir_path;
+            IF dir_path = "" THEN
+                SET dir_exists = FALSE;
+                SET done = FALSE;
+                LEAVE loop2;
+            END IF;
+            SELECT TRUE INTO dir_exists FROM files WHERE path = LEFT(dir_path, CHAR_LENGTH(dir_path)-1);
+            IF dir_exists IS TRUE THEN
+                SET dir_exists = FALSE;
+                ITERATE loop2;
+            END IF;
+            INSERT INTO entities (name, description, role, acl) VALUES (NULL, NULL, "DIRECTORY", 0);
+            SET dir_id = LAST_INSERT_ID();
+            INSERT INTO files (file_id, path, size, hash, checked_timestamp, mimetype, fileStorage, storageId) VALUES (dir_id, LEFT(dir_path, CHAR_LENGTH(dir_path)-1), 0, NULL, 0, "inode/directory", "DEFAULT", LEFT(dir_path, CHAR_LENGTH(dir_path)-1));
+        END LOOP;
+
+    END LOOP;
+
+    CLOSE cur;
+
+END //
+
+DELIMITER ;
+
+CALL _create_dirs();

tests/test_directory_patch.sql

+
+USE _caosdb_schema_unit_tests;
+
+BEGIN;
+CALL tap.no_plan();
+
+-- SETUP
+INSERT INTO entities (id, role, acl) VALUES
+(51, "FILE", 0),
+(52, "FILE", 0),
+(53, "FILE", 0),
+(54, "FILE", 0),
+(55, "FILE", 0),
+(56, "FILE", 0);
+
+
+INSERT INTO files (file_id, path, size, storageId) VALUES
+(51, "file_at_root_a", 256, "file_at_root_a"),
+(52, "file_at_root_b", 256, "file_at_root_b"),
+(53, "level1/file_at_level1_a", 256, "level1/file_at_level1_a"),
+(54, "level1/file_at_level1_b", 256, "level1/file_at_level1_b"),
+(55, "level1/level2/file_at_level2_a", 256, "level1/level2/file_at_level2_a"),
+(56, "level1/level2/file_at_level2_b", 256, "level1/level2/file_at_level2_b");
+
+SELECT tap.eq(COUNT(*), 6, "six rows in test_files") FROM files;
+
+-- TEST _create_dirs
+
+CALL _create_dirs();
+
+SELECT * FROM files;
+SELECT tap.eq(COUNT(*), 8, "8 rows in test_files") FROM files;
+SELECT tap.eq(COUNT(*), 1, "one dir level1") FROM files WHERE path="level1";
+SELECT tap.eq(COUNT(*), 1, "one dir level1/level2") FROM files WHERE path="level1/level2";
+
+
+
+CALL tap.finish();
+ROLLBACK;

update_sql_procedures.sh

@@ -33,7 +33,11 @@ mysql_execute "$(sed s/db_2_0/$DATABASE_NAME/g rules.sql)"
 echo "[OK]"
 
 echo -n "updating procedures ... "
-mysql_execute "$(sed s/db_2_0/$DATABASE_NAME/g procedures/*.sql procedures/query/*.sql)" 
+PROCEDURES="procedures/*.sql procedures/query/*.sql"
+for p in $PROCEDURES
+do
+    mysql_execute "$(sed s/db_2_0/$DATABASE_NAME/g $p)"
+done
 success