Skip to content
Snippets Groups Projects
Verified Commit 16ed23be authored by Timm Fitschen's avatar Timm Fitschen
Browse files

WIP: file storage refactoring: directories

parent addd3e39
No related branches found
No related tags found
1 merge request!10Draft: ENH: file system: directory
Pipeline #31742 failed
Stage: test
Stage: deploy
Hide whitespace changes
Inline Side-by-side
patches/patch20220110-6.0-SNAPSHOT/patch.sql
+ 17
− 17
View file @ 16ed23be
......@@ -40,8 +40,8 @@ ALTER TABLE archive_files DROP COLUMN hash;
-- Add new fields to files table...
ALTER TABLE files ADD COLUMN IF NOT EXISTS (
hash_algorithm VARBINARY(255) NULL DEFAULT "SHA-512",
-- parent_directory INT UNSIGNED DEFAULT NULL,
hash_algorithm VARBINARY(255) NULL DEFAULT NULL,
parent_directory INT UNSIGNED DEFAULT NULL,
mimetype VARBINARY(255) DEFAULT NULL,
file_storage_id VARBINARY(255) NOT NULL DEFAULT "DEFAULT",
file_key VARBINARY(16000) DEFAULT NULL
......@@ -50,8 +50,8 @@ ALTER TABLE files ADD COLUMN IF NOT EXISTS (
-- ... and to the corresponding archive_files table.
ALTER TABLE archive_files ADD COLUMN IF NOT EXISTS (
checked_timestamp BIGINT NOT NULL DEFAULT 0,
hash_algorithm VARBINARY(255) NULL DEFAULT "SHA-512",
-- parent_directory INT UNSIGNED DEFAULT NULL,
hash_algorithm VARBINARY(255) NULL DEFAULT NULL,
parent_directory INT UNSIGNED DEFAULT NULL,
mimetype VARBINARY(255) DEFAULT NULL,
file_storage_id VARBINARY(255) NOT NULL DEFAULT "DEFAULT",
file_key VARBINARY(16000) DEFAULT NULL
......@@ -60,15 +60,15 @@ ALTER TABLE archive_files ADD COLUMN IF NOT EXISTS (
-- no two files at the same path are allowed.
ALTER TABLE files ADD UNIQUE (`path`);
-- directories cannot be deleted without deleting all the content first.
-- ALTER TABLE files ADD FOREIGN KEY (parent_directory) REFERENCES entities (id);
-- ALTER TABLE archive_files ADD FOREIGN KEY (parent_directory) REFERENCES entities (id) ON DELETE CASCADE;
ALTER TABLE files ADD FOREIGN KEY (parent_directory) REFERENCES entities (id);
ALTER TABLE archive_files ADD FOREIGN KEY (parent_directory) REFERENCES entities (id) ON DELETE CASCADE;
-- ALTER TABLE entities MODIFY COLUMN `role` enum('RECORDTYPE','RECORD','FILE','DOMAIN','PROPERTY','DATATYPE','ROLE','QUERYTEMPLATE', 'DIRECTORY') COLLATE utf8_unicode_ci NOT NULL;
ALTER TABLE entities MODIFY COLUMN `role` enum('RECORDTYPE','RECORD','FILE','DOMAIN','PROPERTY','DATATYPE','ROLE','QUERYTEMPLATE', 'DIRECTORY') COLLATE utf8_unicode_ci NOT NULL;
-- Add a new special role "DIRECTORY"
-- INSERT IGNORE INTO entities (id, description, role, acl) VALUES (9, "The directory role.", "ROLE", 0);
-- INSERT IGNORE INTO name_data (domain_id, entity_id, property_id, value, status, pidx) VALUES (0, 9, 20, "DIRECTORY", "FIX", 0);
INSERT IGNORE INTO entities (id, description, role, acl) VALUES (9, "The directory role.", "ROLE", 0);
INSERT IGNORE INTO name_data (domain_id, entity_id, property_id, value, status, pidx) VALUES (0, 9, 20, "DIRECTORY", "FIX", 0);
-- In the default file storage back-end the key is just the path.
UPDATE files SET file_key=path;
......@@ -81,8 +81,8 @@ ALTER TABLE files MODIFY COLUMN file_key VARBINARY(16000) NOT NULL;
ALTER TABLE archive_files MODIFY COLUMN file_key VARBINARY(16000) NOT NULL;
-- Size may be NULL for directories
-- ALTER TABLE files MODIFY COLUMN size BIGINT UNSIGNED NULL DEFAULT NULL;
-- ALTER TABLE archive_files MODIFY COLUMN size BIGINT UNSIGNED NULL DEFAULT NULL;
ALTER TABLE files MODIFY COLUMN size BIGINT UNSIGNED NULL DEFAULT NULL;
ALTER TABLE archive_files MODIFY COLUMN size BIGINT UNSIGNED NULL DEFAULT NULL;
-- More characters for long names
ALTER TABLE files MODIFY COLUMN path VARCHAR(16000) NOT NULL;
......@@ -90,11 +90,11 @@ ALTER TABLE archive_files MODIFY COLUMN path VARCHAR(16000) NOT NULL;
-- Add a special nameless directory which is used as the root directory of the
-- internal file system.
-- INSERT IGNORE INTO entities (id, description, role, acl) VALUES (51, "The root directory of the internal files system", "DIRECTORY", 0);
--INSERT IGNORE INTO files
-- (file_id, path, checked_timestamp, mimetype, file_storage_id,
-- file_key, parent_directory)
-- VALUES (51, "", 0, "inode/directory", "DEFAULT", "", NULL);
INSERT IGNORE INTO entities (id, description, role, acl) VALUES (51, "The root directory of the internal files system", "DIRECTORY", 0);
INSERT IGNORE INTO files
(file_id, path, checked_timestamp, mimetype, file_storage_id,
file_key, parent_directory)
VALUES (51, "", 0, "inode/directory", "DEFAULT", "", NULL);
DROP PROCEDURE IF EXISTS _create_dirs;
DELIMITER //
......@@ -188,7 +188,7 @@ END //
DELIMITER ;
-- CALL _create_dirs();
CALL _create_dirs();
-- not to be used after this patch has been applied.
-- DROP PROCEDURE _create_dirs;
procedures/deleteEntity.sql
+ 1
− 1
View file @ 16ed23be
......@@ -43,7 +43,7 @@ BEGIN
-- detele file properties
DELETE FROM files where file_id=EntityID;
/*DELETE FROM files where parent_directory=EntityID;*/
DELETE FROM files where parent_directory=EntityID;
-- delete datatype stuff
DELETE FROM data_type
......
procedures/getDependentEntities.sql
+ 9
− 5
View file @ 16ed23be
......@@ -4,6 +4,8 @@
*
* Copyright (C) 2018 Research Group Biomedical Physics,
* Max-Planck-Institute for Dynamics and Self-Organization Göttingen
* Copyright (C) 2022 Timm Fitschen <t.fitschen@indiscale.com>
* Copyright (C) 2022 IndiScale GmbH <info@indiscale.com>
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
......@@ -29,9 +31,9 @@ delimiter //
CREATE PROCEDURE db_5_0.getDependentEntities(in EntityID INT UNSIGNED)
BEGIN
DROP TEMPORARY TABLE IF EXISTS refering;
DROP TEMPORARY TABLE IF EXISTS refering;
CREATE TEMPORARY TABLE refering (
id INT UNSIGNED UNIQUE
id INT UNSIGNED UNIQUE
);
INSERT IGNORE INTO refering (id) SELECT entity_id FROM reference_data WHERE (value=EntityID OR property_id=EntityID) AND domain_id=0 AND entity_id!=EntityID;
......@@ -61,11 +63,13 @@ INSERT IGNORE INTO refering (id) SELECT domain_id FROM date_data WHERE property_
INSERT IGNORE INTO refering (id) SELECT entity_id FROM null_data WHERE property_id=EntityID AND domain_id=0 AND entity_id!=EntityID;
INSERT IGNORE INTO refering (id) SELECT domain_id FROM null_data WHERE property_id=EntityID AND domain_id!=EntityID AND entity_id!=EntityID AND domain_id!=0;
INSERT IGNORE INTO refering (id) SELECT entity_id from data_type WHERE datatype=EntityID AND domain_id=0 AND entity_id!=EntityID;
INSERT IGNORE INTO refering (id) SELECT domain_id from data_type WHERE datatype=EntityID;
INSERT IGNORE INTO refering (id) SELECT entity_id FROM data_type WHERE datatype=EntityID AND domain_id=0 AND entity_id!=EntityID;
INSERT IGNORE INTO refering (id) SELECT domain_id FROM data_type WHERE datatype=EntityID;
INSERT IGNORE INTO refering (id) SELECT file_id FROM files WHERE parent_directory=EntityID;
Select id from refering WHERE id!=0 and id!=EntityID;
SELECT id FROM refering WHERE id!=0 AND id!=EntityID;
DROP TEMPORARY TABLE refering;
......
procedures/insertFSODescriptor.sql
+ 6
− 3
View file @ 16ed23be
......@@ -42,6 +42,8 @@ DROP PROCEDURE IF EXISTS db_5_0.insertFSODescriptor //
* Byte size of the file.
* FilePath VARCHAR(16000),
* The path of the FSO in the virtual file system.
* FileParentID INT UNSIGNED,
* The parent directory.
* FileMimeType VARCHAR(255),
* The files mime type.
* FileStorageId VARCHAR(255),
......@@ -57,6 +59,7 @@ CREATE PROCEDURE db_5_0.insertFSODescriptor(
IN FileCheckedTimestamp BIGINT,
IN FileSize BIGINT UNSIGNED,
IN FilePath VARCHAR(16000),
IN FileParentID INT UNSIGNED,
IN FileMimeType VARCHAR(255),
IN FileStorageId VARCHAR(255),
IN FileKey VARCHAR(16000))
......@@ -71,9 +74,9 @@ insertFSODescriptorBody: BEGIN
path,
mimetype,
file_storage_id,
file_key
/*, parent_directory*/
) VALUES (EntityID, unhex(FileHash), FileHashAlgo, FileCheckedTimestamp, FileSize, FilePath, FileMimeType, FileStorageId, FileKey);
file_key,
parent_directory
) VALUES (EntityID, unhex(FileHash), FileHashAlgo, FileCheckedTimestamp, FileSize, FilePath, FileMimeType, FileStorageId, FileKey, FileParentID);
END;
//
......
procedures/retrieveEntity.sql
+ 6
− 6
View file @ 16ed23be
......@@ -55,7 +55,7 @@ create procedure db_5_0.retrieveEntity(
in Version VARBINARY(255))
retrieveEntityBody: BEGIN
DECLARE FilePath VARCHAR(15920) DEFAULT NULL;
/*DECLARE FileParentID INT UNSIGNED DEFAULT NULL;*/
DECLARE FileParentID INT UNSIGNED DEFAULT NULL;
DECLARE FileSize BIGINT UNSIGNED DEFAULT NULL;
DECLARE FileHash VARCHAR(255) DEFAULT NULL;
DECLARE FileHashAlgo VARCHAR(255) DEFAULT NULL;
......@@ -89,8 +89,8 @@ retrieveEntityBody: BEGIN
LEAVE retrieveEntityBody;
END IF;
SELECT path, size, hex(hash), hash_algorithm, checked_timestamp, mimetype, file_storage_id, file_key
INTO FilePath, FileSize, FileHash, FileHashAlgo, FileHashChecked, FileMimeType, FileStorageID, FileKey
SELECT path, parent_directory, size, hex(hash), hash_algorithm, checked_timestamp, mimetype, file_storage_id, file_key
INTO FilePath, FileParentID, FileSize, FileHash, FileHashAlgo, FileHashChecked, FileMimeType, FileStorageID, FileKey
FROM archive_files
WHERE file_id = EntityID
AND _iversion = IVersion
......@@ -153,8 +153,8 @@ retrieveEntityBody: BEGIN
END IF;
END IF;
SELECT path, /*parent_directory,*/ size, hex(hash), hash_algorithm, checked_timestamp, mimetype, file_storage_id, file_key
INTO FilePath, /*FileParentID,*/ FileSize, FileHash, FileHashAlgo, FileHashChecked, FileMimeType, FileStorageID, FileKey
SELECT path, parent_directory, size, hex(hash), hash_algorithm, checked_timestamp, mimetype, file_storage_id, file_key
INTO FilePath, FileParentID, FileSize, FileHash, FileHashAlgo, FileHashChecked, FileMimeType, FileStorageID, FileKey
FROM files
WHERE file_id = EntityID
LIMIT 1;
......@@ -194,7 +194,7 @@ retrieveEntityBody: BEGIN
FileMimeType as FileMimeType,
FileStorageID as FileStorageID,
FileKey as FileKey,
/*FileParentID as FileParentID,*/
FileParentID as FileParentID,
(SELECT acl FROM entity_acl AS a WHERE a.id = e.acl) AS ACL,
Version AS Version
FROM entities e WHERE id = EntityID LIMIT 1;
......
procedures/retrieveFSODescriptorByPath.sql
+ 5
− 4
View file @ 16ed23be
......@@ -40,7 +40,7 @@ DROP PROCEDURE IF EXISTS db_5_0.retrieveFSODescriptorByPath;
*
* ResultSet
* ---------
* Tuples of (FileHashAlgo, FileId, FilePath, FileSize, FileHash,
* Tuples of (FileHashAlgo, FileID, FilePath, FileSize, FileHash, FileParentID,
* FileHashChecked, FileMimeType, FileStorageID, FileKey)
*/
CREATE PROCEDURE db_5_0.retrieveFSODescriptorByPath(
......@@ -48,15 +48,16 @@ CREATE PROCEDURE db_5_0.retrieveFSODescriptorByPath(
retrieveFSODescriptorByPathBody: BEGIN
SELECT hash_algorithm AS FileHashAlgo,
byPath as Param,
file_id AS FileId,
byPath as ParamByPath,
file_id AS FileID,
path AS FilePath,
size AS FileSize,
hex(hash) AS FileHash,
checked_timestamp AS FileHashChecked,
mimetype AS FileMimeType,
file_storage_id AS FileStorageID,
file_key AS FileKey
file_key AS FileKey,
parent_directory AS FileParentID
FROM files
WHERE path LIKE byPath;
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment