From 16ed23be423359430029f513a58959f03aa35506 Mon Sep 17 00:00:00 2001
From: Timm Fitschen <t.fitschen@indiscale.com>
Date: Tue, 13 Dec 2022 16:58:21 +0100
Subject: [PATCH] WIP: file storage refactoring: directories
---
patches/patch20220110-6.0-SNAPSHOT/patch.sql | 34 ++++++++++----------
procedures/deleteEntity.sql | 2 +-
procedures/getDependentEntities.sql | 14 +++++---
procedures/insertFSODescriptor.sql | 9 ++++--
procedures/retrieveEntity.sql | 12 +++----
procedures/retrieveFSODescriptorByPath.sql | 9 +++---
6 files changed, 44 insertions(+), 36 deletions(-)
diff --git a/patches/patch20220110-6.0-SNAPSHOT/patch.sql b/patches/patch20220110-6.0-SNAPSHOT/patch.sql
index 7ad3403..18f1b3e 100644
--- a/patches/patch20220110-6.0-SNAPSHOT/patch.sql
+++ b/patches/patch20220110-6.0-SNAPSHOT/patch.sql
@@ -40,8 +40,8 @@ ALTER TABLE archive_files DROP COLUMN hash;
-- Add new fields to files table...
ALTER TABLE files ADD COLUMN IF NOT EXISTS (
- hash_algorithm VARBINARY(255) NULL DEFAULT "SHA-512",
- -- parent_directory INT UNSIGNED DEFAULT NULL,
+ hash_algorithm VARBINARY(255) NULL DEFAULT NULL,
+ parent_directory INT UNSIGNED DEFAULT NULL,
mimetype VARBINARY(255) DEFAULT NULL,
file_storage_id VARBINARY(255) NOT NULL DEFAULT "DEFAULT",
file_key VARBINARY(16000) DEFAULT NULL
@@ -50,8 +50,8 @@ ALTER TABLE files ADD COLUMN IF NOT EXISTS (
-- ... and to the corresponding archive_files table.
ALTER TABLE archive_files ADD COLUMN IF NOT EXISTS (
checked_timestamp BIGINT NOT NULL DEFAULT 0,
- hash_algorithm VARBINARY(255) NULL DEFAULT "SHA-512",
- -- parent_directory INT UNSIGNED DEFAULT NULL,
+ hash_algorithm VARBINARY(255) NULL DEFAULT NULL,
+ parent_directory INT UNSIGNED DEFAULT NULL,
mimetype VARBINARY(255) DEFAULT NULL,
file_storage_id VARBINARY(255) NOT NULL DEFAULT "DEFAULT",
file_key VARBINARY(16000) DEFAULT NULL
@@ -60,15 +60,15 @@ ALTER TABLE archive_files ADD COLUMN IF NOT EXISTS (
-- no two files at the same path are allowed.
ALTER TABLE files ADD UNIQUE (`path`);
-- directories cannot be deleted without deleting all the content first.
--- ALTER TABLE files ADD FOREIGN KEY (parent_directory) REFERENCES entities (id);
--- ALTER TABLE archive_files ADD FOREIGN KEY (parent_directory) REFERENCES entities (id) ON DELETE CASCADE;
+ALTER TABLE files ADD FOREIGN KEY (parent_directory) REFERENCES entities (id);
+ALTER TABLE archive_files ADD FOREIGN KEY (parent_directory) REFERENCES entities (id) ON DELETE CASCADE;
--- ALTER TABLE entities MODIFY COLUMN `role` enum('RECORDTYPE','RECORD','FILE','DOMAIN','PROPERTY','DATATYPE','ROLE','QUERYTEMPLATE', 'DIRECTORY') COLLATE utf8_unicode_ci NOT NULL;
+ALTER TABLE entities MODIFY COLUMN `role` enum('RECORDTYPE','RECORD','FILE','DOMAIN','PROPERTY','DATATYPE','ROLE','QUERYTEMPLATE', 'DIRECTORY') COLLATE utf8_unicode_ci NOT NULL;
-- Add a new special role "DIRECTORY"
--- INSERT IGNORE INTO entities (id, description, role, acl) VALUES (9, "The directory role.", "ROLE", 0);
--- INSERT IGNORE INTO name_data (domain_id, entity_id, property_id, value, status, pidx) VALUES (0, 9, 20, "DIRECTORY", "FIX", 0);
+INSERT IGNORE INTO entities (id, description, role, acl) VALUES (9, "The directory role.", "ROLE", 0);
+INSERT IGNORE INTO name_data (domain_id, entity_id, property_id, value, status, pidx) VALUES (0, 9, 20, "DIRECTORY", "FIX", 0);
-- In the default file storage back-end the key is just the path.
UPDATE files SET file_key=path;
@@ -81,8 +81,8 @@ ALTER TABLE files MODIFY COLUMN file_key VARBINARY(16000) NOT NULL;
ALTER TABLE archive_files MODIFY COLUMN file_key VARBINARY(16000) NOT NULL;
-- Size may be NULL for directories
--- ALTER TABLE files MODIFY COLUMN size BIGINT UNSIGNED NULL DEFAULT NULL;
--- ALTER TABLE archive_files MODIFY COLUMN size BIGINT UNSIGNED NULL DEFAULT NULL;
+ALTER TABLE files MODIFY COLUMN size BIGINT UNSIGNED NULL DEFAULT NULL;
+ALTER TABLE archive_files MODIFY COLUMN size BIGINT UNSIGNED NULL DEFAULT NULL;
-- More characters for long names
ALTER TABLE files MODIFY COLUMN path VARCHAR(16000) NOT NULL;
@@ -90,11 +90,11 @@ ALTER TABLE archive_files MODIFY COLUMN path VARCHAR(16000) NOT NULL;
-- Add a special nameless directory which is used as the root directory of the
-- internal file system.
--- INSERT IGNORE INTO entities (id, description, role, acl) VALUES (51, "The root directory of the internal files system", "DIRECTORY", 0);
---INSERT IGNORE INTO files
--- (file_id, path, checked_timestamp, mimetype, file_storage_id,
--- file_key, parent_directory)
--- VALUES (51, "", 0, "inode/directory", "DEFAULT", "", NULL);
+INSERT IGNORE INTO entities (id, description, role, acl) VALUES (51, "The root directory of the internal files system", "DIRECTORY", 0);
+INSERT IGNORE INTO files
+ (file_id, path, checked_timestamp, mimetype, file_storage_id,
+ file_key, parent_directory)
+ VALUES (51, "", 0, "inode/directory", "DEFAULT", "", NULL);
DROP PROCEDURE IF EXISTS _create_dirs;
DELIMITER //
@@ -188,7 +188,7 @@ END //
DELIMITER ;
--- CALL _create_dirs();
+CALL _create_dirs();
-- not to be used after this patch has been applied.
-- DROP PROCEDURE _create_dirs;
diff --git a/procedures/deleteEntity.sql b/procedures/deleteEntity.sql
index d64061f..859f43c 100644
--- a/procedures/deleteEntity.sql
+++ b/procedures/deleteEntity.sql
@@ -43,7 +43,7 @@ BEGIN
-- detele file properties
DELETE FROM files where file_id=EntityID;
- /*DELETE FROM files where parent_directory=EntityID;*/
+ DELETE FROM files where parent_directory=EntityID;
-- delete datatype stuff
DELETE FROM data_type
diff --git a/procedures/getDependentEntities.sql b/procedures/getDependentEntities.sql
index 708503d..fecf96e 100644
--- a/procedures/getDependentEntities.sql
+++ b/procedures/getDependentEntities.sql
@@ -4,6 +4,8 @@
*
* Copyright (C) 2018 Research Group Biomedical Physics,
* Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+ * Copyright (C) 2022 Timm Fitschen <t.fitschen@indiscale.com>
+ * Copyright (C) 2022 IndiScale GmbH <info@indiscale.com>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
@@ -29,9 +31,9 @@ delimiter //
CREATE PROCEDURE db_5_0.getDependentEntities(in EntityID INT UNSIGNED)
BEGIN
-DROP TEMPORARY TABLE IF EXISTS refering;
+DROP TEMPORARY TABLE IF EXISTS refering;
CREATE TEMPORARY TABLE refering (
-id INT UNSIGNED UNIQUE
+ id INT UNSIGNED UNIQUE
);
INSERT IGNORE INTO refering (id) SELECT entity_id FROM reference_data WHERE (value=EntityID OR property_id=EntityID) AND domain_id=0 AND entity_id!=EntityID;
@@ -61,11 +63,13 @@ INSERT IGNORE INTO refering (id) SELECT domain_id FROM date_data WHERE property_
INSERT IGNORE INTO refering (id) SELECT entity_id FROM null_data WHERE property_id=EntityID AND domain_id=0 AND entity_id!=EntityID;
INSERT IGNORE INTO refering (id) SELECT domain_id FROM null_data WHERE property_id=EntityID AND domain_id!=EntityID AND entity_id!=EntityID AND domain_id!=0;
-INSERT IGNORE INTO refering (id) SELECT entity_id from data_type WHERE datatype=EntityID AND domain_id=0 AND entity_id!=EntityID;
-INSERT IGNORE INTO refering (id) SELECT domain_id from data_type WHERE datatype=EntityID;
+INSERT IGNORE INTO refering (id) SELECT entity_id FROM data_type WHERE datatype=EntityID AND domain_id=0 AND entity_id!=EntityID;
+INSERT IGNORE INTO refering (id) SELECT domain_id FROM data_type WHERE datatype=EntityID;
+INSERT IGNORE INTO refering (id) SELECT file_id FROM files WHERE parent_directory=EntityID;
-Select id from refering WHERE id!=0 and id!=EntityID;
+
+SELECT id FROM refering WHERE id!=0 AND id!=EntityID;
DROP TEMPORARY TABLE refering;
diff --git a/procedures/insertFSODescriptor.sql b/procedures/insertFSODescriptor.sql
index cdbd6bd..8e24a3c 100644
--- a/procedures/insertFSODescriptor.sql
+++ b/procedures/insertFSODescriptor.sql
@@ -42,6 +42,8 @@ DROP PROCEDURE IF EXISTS db_5_0.insertFSODescriptor //
* Byte size of the file.
* FilePath VARCHAR(16000),
* The path of the FSO in the virtual file system.
+ * FileParentID INT UNSIGNED,
+ * The parent directory.
* FileMimeType VARCHAR(255),
* The files mime type.
* FileStorageId VARCHAR(255),
@@ -57,6 +59,7 @@ CREATE PROCEDURE db_5_0.insertFSODescriptor(
IN FileCheckedTimestamp BIGINT,
IN FileSize BIGINT UNSIGNED,
IN FilePath VARCHAR(16000),
+ IN FileParentID INT UNSIGNED,
IN FileMimeType VARCHAR(255),
IN FileStorageId VARCHAR(255),
IN FileKey VARCHAR(16000))
@@ -71,9 +74,9 @@ insertFSODescriptorBody: BEGIN
path,
mimetype,
file_storage_id,
- file_key
- /*, parent_directory*/
- ) VALUES (EntityID, unhex(FileHash), FileHashAlgo, FileCheckedTimestamp, FileSize, FilePath, FileMimeType, FileStorageId, FileKey);
+ file_key,
+ parent_directory
+ ) VALUES (EntityID, unhex(FileHash), FileHashAlgo, FileCheckedTimestamp, FileSize, FilePath, FileMimeType, FileStorageId, FileKey, FileParentID);
END;
//
diff --git a/procedures/retrieveEntity.sql b/procedures/retrieveEntity.sql
index 8ab9996..f7eb1e9 100644
--- a/procedures/retrieveEntity.sql
+++ b/procedures/retrieveEntity.sql
@@ -55,7 +55,7 @@ create procedure db_5_0.retrieveEntity(
in Version VARBINARY(255))
retrieveEntityBody: BEGIN
DECLARE FilePath VARCHAR(15920) DEFAULT NULL;
- /*DECLARE FileParentID INT UNSIGNED DEFAULT NULL;*/
+ DECLARE FileParentID INT UNSIGNED DEFAULT NULL;
DECLARE FileSize BIGINT UNSIGNED DEFAULT NULL;
DECLARE FileHash VARCHAR(255) DEFAULT NULL;
DECLARE FileHashAlgo VARCHAR(255) DEFAULT NULL;
@@ -89,8 +89,8 @@ retrieveEntityBody: BEGIN
LEAVE retrieveEntityBody;
END IF;
- SELECT path, size, hex(hash), hash_algorithm, checked_timestamp, mimetype, file_storage_id, file_key
- INTO FilePath, FileSize, FileHash, FileHashAlgo, FileHashChecked, FileMimeType, FileStorageID, FileKey
+ SELECT path, parent_directory, size, hex(hash), hash_algorithm, checked_timestamp, mimetype, file_storage_id, file_key
+ INTO FilePath, FileParentID, FileSize, FileHash, FileHashAlgo, FileHashChecked, FileMimeType, FileStorageID, FileKey
FROM archive_files
WHERE file_id = EntityID
AND _iversion = IVersion
@@ -153,8 +153,8 @@ retrieveEntityBody: BEGIN
END IF;
END IF;
- SELECT path, /*parent_directory,*/ size, hex(hash), hash_algorithm, checked_timestamp, mimetype, file_storage_id, file_key
- INTO FilePath, /*FileParentID,*/ FileSize, FileHash, FileHashAlgo, FileHashChecked, FileMimeType, FileStorageID, FileKey
+ SELECT path, parent_directory, size, hex(hash), hash_algorithm, checked_timestamp, mimetype, file_storage_id, file_key
+ INTO FilePath, FileParentID, FileSize, FileHash, FileHashAlgo, FileHashChecked, FileMimeType, FileStorageID, FileKey
FROM files
WHERE file_id = EntityID
LIMIT 1;
@@ -194,7 +194,7 @@ retrieveEntityBody: BEGIN
FileMimeType as FileMimeType,
FileStorageID as FileStorageID,
FileKey as FileKey,
- /*FileParentID as FileParentID,*/
+ FileParentID as FileParentID,
(SELECT acl FROM entity_acl AS a WHERE a.id = e.acl) AS ACL,
Version AS Version
FROM entities e WHERE id = EntityID LIMIT 1;
diff --git a/procedures/retrieveFSODescriptorByPath.sql b/procedures/retrieveFSODescriptorByPath.sql
index 7a93a0a..821cf9d 100644
--- a/procedures/retrieveFSODescriptorByPath.sql
+++ b/procedures/retrieveFSODescriptorByPath.sql
@@ -40,7 +40,7 @@ DROP PROCEDURE IF EXISTS db_5_0.retrieveFSODescriptorByPath;
*
* ResultSet
* ---------
- * Tuples of (FileHashAlgo, FileId, FilePath, FileSize, FileHash,
+ * Tuples of (FileHashAlgo, FileID, FilePath, FileSize, FileHash, FileParentID,
* FileHashChecked, FileMimeType, FileStorageID, FileKey)
*/
CREATE PROCEDURE db_5_0.retrieveFSODescriptorByPath(
@@ -48,15 +48,16 @@ CREATE PROCEDURE db_5_0.retrieveFSODescriptorByPath(
retrieveFSODescriptorByPathBody: BEGIN
SELECT hash_algorithm AS FileHashAlgo,
- byPath as Param,
- file_id AS FileId,
+ byPath as ParamByPath,
+ file_id AS FileID,
path AS FilePath,
size AS FileSize,
hex(hash) AS FileHash,
checked_timestamp AS FileHashChecked,
mimetype AS FileMimeType,
file_storage_id AS FileStorageID,
- file_key AS FileKey
+ file_key AS FileKey,
+ parent_directory AS FileParentID
FROM files
WHERE path LIKE byPath;
--
GitLab