From 79bcbedcfaa5e8d6b8e23231bcfcba0a724d18eb Mon Sep 17 00:00:00 2001
From: Timm Fitschen <t.fitschen@indiscale.com>
Date: Mon, 30 Mar 2020 13:04:25 +0200
Subject: [PATCH] WIP: file system refactoring
---
make_db | 3 +-
procedures/retrieveEntity.sql | 12 ++++--
rules.sql | 5 ---
tests/test_0_next_patch.sql | 70 ++++++++++++++++++++++++++++++++++
tests/test_directory_patch.sql | 39 +++++++++++++++++++
update_sql_procedures.sh | 6 ++-
6 files changed, 125 insertions(+), 10 deletions(-)
create mode 100644 tests/test_0_next_patch.sql
create mode 100644 tests/test_directory_patch.sql
diff --git a/make_db b/make_db
index ccf3cd4..6f23fac 100755
--- a/make_db
+++ b/make_db
@@ -51,12 +51,13 @@ function _execute_tests () {
for tfile in $TESTS ; do
echo "Running $tfile"
echo "----- $tfile -----" >> .TEST_RESULTS
- cat $tfile | $SQL --disable-pager --batch --raw --skip-column-names --unbuffered >> .TEST_RESULTS
+ cat $tfile | $SQL --disable-pager --batch --raw --skip-column-names --unbuffered >> .TEST_RESULTS 2>&1
done;
popd
cat tests/.TEST_RESULTS
grep -c -i "failed" tests/.TEST_RESULTS > /dev/null && return 1
+ grep -c -i "ERROR" tests/.TEST_RESULTS > /dev/null && return 1
return 0
}
diff --git a/procedures/retrieveEntity.sql b/procedures/retrieveEntity.sql
index 1e2621b..d4c0cf7 100644
--- a/procedures/retrieveEntity.sql
+++ b/procedures/retrieveEntity.sql
@@ -29,13 +29,16 @@ drop procedure if exists db_2_0.retrieveEntity //
create procedure db_2_0.retrieveEntity(in EntityID INT UNSIGNED)
BEGIN
- DECLARE FilePath VARCHAR(255) DEFAULT NULL;
- DECLARE FileSize VARCHAR(255) DEFAULT NULL;
+ DECLARE FilePath VARBINARY(65525) DEFAULT NULL;
+ DECLARE FileSize BIGINT UNSIGNED DEFAULT NULL;
DECLARE FileHash VARCHAR(255) DEFAULT NULL;
+ DECLARE FileMimetype VARBINARY(255) DEFAULT NULL;
+ DECLARE FileStorage VARBINARY(255) DEFAULT NULL;
+ DECLARE FileStorageId VARBINARY(65525) DEFAULT NULL;
DECLARE DatatypeID INT UNSIGNED DEFAULT NULL;
DECLARE CollectionName VARCHAR(255) DEFAULT NULL;
- Select path, size, hex(hash) into FilePath, FileSize, FileHash from files where file_id = EntityID LIMIT 1;
+ Select path, size, hex(hash), mimetype, fileStorage, storageId into FilePath, FileSize, FileHash, FileMimetype, FileStorage, FileStorageId FROM files WHERE file_id = EntityID LIMIT 1;
Select datatype into DatatypeID from data_type where domain_id=0 and entity_id=0 and property_id=EntityID LIMIT 1;
SELECT collection into CollectionName from collection_type where domain_id=0 and entity_id=0 and property_id=EntityID LIMIT 1;
@@ -50,6 +53,9 @@ BEGIN
FileSize as FileSize,
FilePath as FilePath,
FileHash as FileHash,
+ FileMimetype as FileMimetype,
+ FileStorage as FileStorage,
+ FileStorageId as FileStorageId,
(SELECT acl FROM entity_acl as a WHERE a.id = e.acl) as ACL
from entities e where id = EntityID LIMIT 1;
END;
diff --git a/rules.sql b/rules.sql
index 017fe16..9c82c7f 100644
--- a/rules.sql
+++ b/rules.sql
@@ -149,8 +149,3 @@ INSERT INTO rules (domain_id, entity_id, transaction, criterion, modus) VALUES (
INSERT INTO rules (domain_id, entity_id, transaction, criterion, modus) VALUES (0,17,'UPDATE','CheckRefidIsaParRefid','MUST');
#-- INSERT INTO rules (domain_id, entity_id, transaction, criterion, modus) VALUES (0,17,'UPDATE','CheckDescPresent','MUST');
-#-- SQLite files
-INSERT INTO rules (domain_id, entity_id, transaction, criterion, modus) VALUES (0,50,'UPDATE','CheckRefidValid','MUST');
-INSERT INTO rules (domain_id, entity_id, transaction, criterion, modus) VALUES (0,50,'UPDATE','CheckRefidIsaParRefid','SHOULD');
-#-- INSERT INTO rules (domain_id, entity_id, transaction, criterion, modus) VALUES (0,50,'UPDATE','CheckDescPresent','SHOULD');
-INSERT INTO rules (domain_id, entity_id, transaction, criterion, modus) VALUES (0,50,'UPDATE','SQLiteTransaction','MUST');
diff --git a/tests/test_0_next_patch.sql b/tests/test_0_next_patch.sql
new file mode 100644
index 0000000..e55e488
--- /dev/null
+++ b/tests/test_0_next_patch.sql
@@ -0,0 +1,70 @@
+-- USE _caosdb_schema_unit_tests;
+
+
+-- REMOVE SQLITE datatype
+DELETE FROM entities WHERE id = 50;
+
+ALTER TABLE files ADD UNIQUE (`path`);
+
+ALTER TABLE files ADD COLUMN IF NOT EXISTS (
+ mimetype VARBINARY(255) DEFAULT NULL,
+ fileStorage VARBINARY(255) NOT NULL DEFAULT "DEFAULT",
+ storageId VARBINARY(255) DEFAULT NULL
+);
+
+
+ALTER TABLE entities MODIFY COLUMN `role` enum('RECORDTYPE','RECORD','FILE','DOMAIN','PROPERTY','DATATYPE','ROLE','QUERYTEMPLATE', 'DIRECTORY') COLLATE utf8_unicode_ci NOT NULL;
+
+INSERT IGNORE INTO entities (id, name, description, role, acl) VALUES (9, "DIRECTORY", "The directory role.", "ROLE", 0);
+
+UPDATE files SET storageId=path;
+
+ALTER TABLE files MODIFY COLUMN storageId VARBINARY(255) NOT NULL;
+
+DROP PROCEDURE IF EXISTS _create_dirs;
+DELIMITER //
+CREATE PROCEDURE _create_dirs()
+BEGIN
+ DECLARE done BOOLEAN DEFAULT FALSE;
+ DECLARE _file_id INT UNSIGNED DEFAULT NULL;
+ DECLARE _path VARCHAR(255) DEFAULT NULL;
+ DECLARE dir_id INT UNSIGNED DEFAULT NULL;
+ DECLARE dir_path VARCHAR(255) DEFAULT "";
+ DECLARE dir_exists BOOLEAN DEFAULT FALSE;
+ DECLARE cur CURSOR FOR SELECT file_id, path FROM files;
+ DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = TRUE;
+ OPEN cur;
+
+ loop1: LOOP
+ FETCH cur INTO _file_id, _path;
+ IF done THEN
+ LEAVE loop1;
+ END IF;
+
+ -- TODO
+ loop2: LOOP
+ SELECT REGEXP_SUBSTR(_path, CONCAT("(?U)^", dir_path, ".*/" )) INTO dir_path;
+ IF dir_path = "" THEN
+ SET dir_exists = FALSE;
+ SET done = FALSE;
+ LEAVE loop2;
+ END IF;
+ SELECT TRUE INTO dir_exists FROM files WHERE path = LEFT(dir_path, CHAR_LENGTH(dir_path)-1);
+ IF dir_exists IS TRUE THEN
+ SET dir_exists = FALSE;
+ ITERATE loop2;
+ END IF;
+ INSERT INTO entities (name, description, role, acl) VALUES (NULL, NULL, "DIRECTORY", 0);
+ SET dir_id = LAST_INSERT_ID();
+ INSERT INTO files (file_id, path, size, hash, checked_timestamp, mimetype, fileStorage, storageId) VALUES (dir_id, LEFT(dir_path, CHAR_LENGTH(dir_path)-1), 0, NULL, 0, "inode/directory", "DEFAULT", LEFT(dir_path, CHAR_LENGTH(dir_path)-1));
+ END LOOP;
+
+ END LOOP;
+
+ CLOSE cur;
+
+END //
+
+DELIMITER ;
+
+CALL _create_dirs();
diff --git a/tests/test_directory_patch.sql b/tests/test_directory_patch.sql
new file mode 100644
index 0000000..a774704
--- /dev/null
+++ b/tests/test_directory_patch.sql
@@ -0,0 +1,39 @@
+
+USE _caosdb_schema_unit_tests;
+
+BEGIN;
+CALL tap.no_plan();
+
+-- SETUP
+INSERT INTO entities (id, role, acl) VALUES
+(51, "FILE", 0),
+(52, "FILE", 0),
+(53, "FILE", 0),
+(54, "FILE", 0),
+(55, "FILE", 0),
+(56, "FILE", 0);
+
+
+INSERT INTO files (file_id, path, size, storageId) VALUES
+(51, "file_at_root_a", 256, "file_at_root_a"),
+(52, "file_at_root_b", 256, "file_at_root_b"),
+(53, "level1/file_at_level1_a", 256, "level1/file_at_level1_a"),
+(54, "level1/file_at_level1_b", 256, "level1/file_at_level1_b"),
+(55, "level1/level2/file_at_level2_a", 256, "level1/level2/file_at_level2_a"),
+(56, "level1/level2/file_at_level2_b", 256, "level1/level2/file_at_level2_b");
+
+SELECT tap.eq(COUNT(*), 6, "six rows in test_files") FROM files;
+
+-- TEST _create_dirs
+
+CALL _create_dirs();
+
+SELECT * FROM files;
+SELECT tap.eq(COUNT(*), 8, "8 rows in test_files") FROM files;
+SELECT tap.eq(COUNT(*), 1, "one dir level1") FROM files WHERE path="level1";
+SELECT tap.eq(COUNT(*), 1, "one dir level1/level2") FROM files WHERE path="level1/level2";
+
+
+
+CALL tap.finish();
+ROLLBACK;
diff --git a/update_sql_procedures.sh b/update_sql_procedures.sh
index 7b0b457..d708006 100755
--- a/update_sql_procedures.sh
+++ b/update_sql_procedures.sh
@@ -33,7 +33,11 @@ mysql_execute "$(sed s/db_2_0/$DATABASE_NAME/g rules.sql)"
echo "[OK]"
echo -n "updating procedures ... "
-mysql_execute "$(sed s/db_2_0/$DATABASE_NAME/g procedures/*.sql procedures/query/*.sql)"
+PROCEDURES="procedures/*.sql procedures/query/*.sql"
+for p in $PROCEDURES
+do
+ mysql_execute "$(sed s/db_2_0/$DATABASE_NAME/g $p)"
+done
success
--
GitLab