From 409361775fcbc14ab04562ab352f62cc29dc1436 Mon Sep 17 00:00:00 2001
From: Timm Fitschen <t.fitschen@indiscale.com>
Date: Tue, 13 Dec 2022 17:05:55 +0100
Subject: [PATCH] WIP: file storage refactoring: directory
---
.../java/org/caosdb/server/entity/Entity.java | 15 ++--
.../org/caosdb/server/entity/MagicTypes.java | 8 +-
.../java/org/caosdb/server/entity/Role.java | 5 +-
.../server/filesystem/FSODescriptor.java | 8 +-
.../filesystem/FileStorageInterface.java | 3 +-
.../caosdb/server/filesystem/FileSystem.java | 10 +--
.../filesystem/FileSystemInterface.java | 10 ---
.../server/filesystem/LocalFileStorage.java | 24 +++---
.../VirtualFSODescriptorInterface.java | 2 +
.../server/jobs/core/AutoCreateDirs.java | 82 +++++++++++++++++++
.../core/CheckFileStorageConsistency.java | 4 +-
.../java/org/caosdb/server/query/CQLLexer.g4 | 4 +
.../java/org/caosdb/server/query/CQLParser.g4 | 1 +
.../java/org/caosdb/server/query/Query.java | 1 +
14 files changed, 129 insertions(+), 48 deletions(-)
create mode 100644 src/main/java/org/caosdb/server/jobs/core/AutoCreateDirs.java
diff --git a/src/main/java/org/caosdb/server/entity/Entity.java b/src/main/java/org/caosdb/server/entity/Entity.java
index 48a92fd0..317aea4d 100644
--- a/src/main/java/org/caosdb/server/entity/Entity.java
+++ b/src/main/java/org/caosdb/server/entity/Entity.java
@@ -809,14 +809,13 @@ public abstract class Entity extends AbstractObservable implements EntityInterfa
if (tmpIdentifier != null || checksum != null || path != null || size != null) {
// legacy clients (which use this api) always use sha512.
Hash hash = Hash.create(checksum, 0, Hasher.Default);
- setFSODescriptor(
- new FSODescriptor(
- FileSystem.DEFAULT_BACKEND,
- new Path(path).toString(),
- hash,
- new Path(path),
- size,
- tmpIdentifier));
+ if (getRole() == Role.Directory) {
+ setFSODescriptor(FSODescriptor.createDir(FileSystem.DEFAULT_BACKEND, new Path(path)));
+ } else {
+ setFSODescriptor(
+ new FSODescriptor(
+ FileSystem.DEFAULT_BACKEND, null, hash, new Path(path), size, tmpIdentifier));
+ }
}
// Parse flags
diff --git a/src/main/java/org/caosdb/server/entity/MagicTypes.java b/src/main/java/org/caosdb/server/entity/MagicTypes.java
index 39556385..289fae6e 100644
--- a/src/main/java/org/caosdb/server/entity/MagicTypes.java
+++ b/src/main/java/org/caosdb/server/entity/MagicTypes.java
@@ -31,11 +31,13 @@ import org.caosdb.server.transaction.Retrieve;
public enum MagicTypes {
UNIT,
NAME,
- DESCRIPTION;
+ DESCRIPTION,
+ ROOT_DIRECTORY;
private static final EntityID UNIT_ID = new EntityID(21);
private static final EntityID DESCRIPTION_ID = new EntityID(24);
private static final EntityID NAME_ID = new EntityID(20);
+ private static final EntityID ROOT_DIRECTORY_ID = new EntityID(51);
public EntityID getId() {
switch (this) {
@@ -45,6 +47,8 @@ public enum MagicTypes {
return NAME_ID;
case DESCRIPTION:
return DESCRIPTION_ID;
+ case ROOT_DIRECTORY:
+ return ROOT_DIRECTORY_ID;
default:
return null;
}
@@ -58,6 +62,8 @@ public enum MagicTypes {
return NAME;
case 24:
return DESCRIPTION;
+ case 51:
+ return ROOT_DIRECTORY;
default:
return null;
}
diff --git a/src/main/java/org/caosdb/server/entity/Role.java b/src/main/java/org/caosdb/server/entity/Role.java
index 1ce6329d..53bc3a47 100644
--- a/src/main/java/org/caosdb/server/entity/Role.java
+++ b/src/main/java/org/caosdb/server/entity/Role.java
@@ -35,7 +35,6 @@ public enum Role {
Domain,
File,
Directory,
- Link,
Property,
DataType,
QueryTemplate;
@@ -75,14 +74,12 @@ public enum Role {
return new FileToElementStrategy(toString());
case Directory:
return new FileToElementStrategy(toString());
- case Link:
- return new FileToElementStrategy(toString());
default:
return new EntityToElementStrategy(toString());
}
}
public boolean hasFSODescriptor() {
- return this == File; // || this == Directory;
+ return this == File || this == Directory;
}
}
diff --git a/src/main/java/org/caosdb/server/filesystem/FSODescriptor.java b/src/main/java/org/caosdb/server/filesystem/FSODescriptor.java
index d1b59ca8..5290f869 100644
--- a/src/main/java/org/caosdb/server/filesystem/FSODescriptor.java
+++ b/src/main/java/org/caosdb/server/filesystem/FSODescriptor.java
@@ -187,8 +187,9 @@ public class FSODescriptor implements VirtualFSODescriptorInterface {
this.children = children;
}
- public static FSODescriptor createDir(final Path dirPath) {
+ public static FSODescriptor createDir(String fileStorageId, final Path dirPath) {
final FSODescriptor ret = new FSODescriptor(dirPath);
+ ret.fileStorageId = fileStorageId;
ret.type = ObjectType.DIRECTORY;
return ret;
}
@@ -252,4 +253,9 @@ public class FSODescriptor implements VirtualFSODescriptorInterface {
public void setTwin(RealFSODescriptorInterface twin) {
this.twin = twin;
}
+
+ @Override
+ public void setKey(String key) {
+ this.key = key;
+ }
}
diff --git a/src/main/java/org/caosdb/server/filesystem/FileStorageInterface.java b/src/main/java/org/caosdb/server/filesystem/FileStorageInterface.java
index 19420a85..e2bca5af 100644
--- a/src/main/java/org/caosdb/server/filesystem/FileStorageInterface.java
+++ b/src/main/java/org/caosdb/server/filesystem/FileStorageInterface.java
@@ -39,7 +39,8 @@ public interface FileStorageInterface {
*/
public abstract RealFSODescriptorInterface resolve(String fileKey);
- public abstract Undoable move(RealFSODescriptorInterface file, String fileKey) throws Message;
+ public abstract Undoable move(
+ RealFSODescriptorInterface file, VirtualFSODescriptorInterface target) throws Message;
public abstract Undoable delete(String fileKey) throws Message;
diff --git a/src/main/java/org/caosdb/server/filesystem/FileSystem.java b/src/main/java/org/caosdb/server/filesystem/FileSystem.java
index e016d795..800a003b 100644
--- a/src/main/java/org/caosdb/server/filesystem/FileSystem.java
+++ b/src/main/java/org/caosdb/server/filesystem/FileSystem.java
@@ -46,12 +46,6 @@ public class FileSystem implements FileSystemInterface {
return fs.delete(file.getKey());
}
- @Override
- public Undoable store(final RealFSODescriptorInterface file) throws Message {
- final FileStorageInterface fs = getFileStorage(file);
- return fs.move(file, file.getKey());
- }
-
public static FileSystem getInstance() {
return instance;
}
@@ -98,10 +92,10 @@ public class FileSystem implements FileSystemInterface {
return null;
}
- public Undoable move(RealFSODescriptorInterface fso, FSODescriptorInterface target)
+ public Undoable move(RealFSODescriptorInterface fso, VirtualFSODescriptorInterface target)
throws Message {
FileStorageInterface fileStorage = getFileStorage(target);
- return fileStorage.move(fso, target.getKey());
+ return fileStorage.move(fso, target);
}
// public static boolean resolve(FSODescriptorInterface fso) {
diff --git a/src/main/java/org/caosdb/server/filesystem/FileSystemInterface.java b/src/main/java/org/caosdb/server/filesystem/FileSystemInterface.java
index 33cd2204..a28cff1d 100644
--- a/src/main/java/org/caosdb/server/filesystem/FileSystemInterface.java
+++ b/src/main/java/org/caosdb/server/filesystem/FileSystemInterface.java
@@ -10,16 +10,6 @@ import org.caosdb.server.utils.Undoable;
*/
public interface FileSystemInterface {
- /**
- * Store the file described by the FileDescriptorInterface.
- *
- * @param file
- * @return an {@link Undoable} object, which reverts the storage procedure and puts the file back
- * to where it was. To be called when failures occured and the transaction needs to roll back.
- * @throws Message
- */
- public abstract Undoable store(RealFSODescriptorInterface file) throws Message;
-
/**
* Delete the file described by the FileDescriptorInterface.
*
diff --git a/src/main/java/org/caosdb/server/filesystem/LocalFileStorage.java b/src/main/java/org/caosdb/server/filesystem/LocalFileStorage.java
index e981071c..54fb6980 100644
--- a/src/main/java/org/caosdb/server/filesystem/LocalFileStorage.java
+++ b/src/main/java/org/caosdb/server/filesystem/LocalFileStorage.java
@@ -285,10 +285,17 @@ public abstract class LocalFileStorage implements FileStorageInterface {
}
@Override
- public Undoable move(final RealFSODescriptorInterface file, String key) throws Message {
+ public Undoable move(final RealFSODescriptorInterface file, VirtualFSODescriptorInterface target)
+ throws Message {
+ String key = target.getKey();
+ if (key == null) {
+ key = target.getPath().toString();
+ target.setKey(key);
+ }
+
final Path targetPath = getRoot().resolve(key);
- if (file.getObjectType() == ObjectType.DIRECTORY) {
+ if (target.getObjectType() == ObjectType.DIRECTORY) {
final File newDirectory = createDirs(targetPath.toFile());
return new Undoable() {
@@ -319,7 +326,7 @@ public abstract class LocalFileStorage implements FileStorageInterface {
public Undoable delete(final String fileKey) throws Message {
final Path targetPath = getRoot().resolve(fileKey);
- return deleteWithEmptyParentDir(targetPath.toFile(), true);
+ return delete(targetPath.toFile(), true);
}
public Path getRoot() {
@@ -613,17 +620,6 @@ public abstract class LocalFileStorage implements FileStorageInterface {
};
}
- Undoable deleteWithEmptyParentDir(final File f, final boolean recursive) throws Message {
- final File parent = f.getParentFile();
- if (!getRoot().toFile().equals(parent)) {
- final File[] siblings = parent.listFiles();
- if (siblings == null || siblings.length == 1 && siblings[0].equals(f)) {
- return deleteWithEmptyParentDir(parent, recursive);
- }
- }
- return delete(f, recursive);
- }
-
public Long getSize(String key) {
File file = getFile(key);
if (file == null) {
diff --git a/src/main/java/org/caosdb/server/filesystem/VirtualFSODescriptorInterface.java b/src/main/java/org/caosdb/server/filesystem/VirtualFSODescriptorInterface.java
index 6b1e8e1e..fa5cf556 100644
--- a/src/main/java/org/caosdb/server/filesystem/VirtualFSODescriptorInterface.java
+++ b/src/main/java/org/caosdb/server/filesystem/VirtualFSODescriptorInterface.java
@@ -120,4 +120,6 @@ public interface VirtualFSODescriptorInterface extends FSODescriptorInterface {
@Override
public abstract List<? extends VirtualFSODescriptorInterface> listChildren();
+
+ public abstract void setKey(String key);
}
diff --git a/src/main/java/org/caosdb/server/jobs/core/AutoCreateDirs.java b/src/main/java/org/caosdb/server/jobs/core/AutoCreateDirs.java
new file mode 100644
index 00000000..afbadcbe
--- /dev/null
+++ b/src/main/java/org/caosdb/server/jobs/core/AutoCreateDirs.java
@@ -0,0 +1,82 @@
+package org.caosdb.server.jobs.core;
+
+import org.caosdb.server.database.backend.transaction.GetFileEntityByPath;
+import org.caosdb.server.database.exceptions.EntityDoesNotExistException;
+import org.caosdb.server.entity.EntityInterface;
+import org.caosdb.server.entity.InsertEntity;
+import org.caosdb.server.entity.MagicTypes;
+import org.caosdb.server.entity.Role;
+import org.caosdb.server.entity.UpdateEntity;
+import org.caosdb.server.filesystem.FSODescriptor;
+import org.caosdb.server.filesystem.Path;
+import org.caosdb.server.filesystem.VirtualFSODescriptorInterface;
+import org.caosdb.server.jobs.EntityFlagJob;
+import org.caosdb.server.jobs.JobAnnotation;
+import org.caosdb.server.permissions.EntityACL;
+
+@JobAnnotation(flag = "autoCreateDirs", defaultValue = "true", loadAlways = true)
+public class AutoCreateDirs extends EntityFlagJob {
+
+ @Override
+ protected void job(final String value) {
+ final boolean createDirs = "true".equalsIgnoreCase(value);
+
+ if (createDirs
+ && (getEntity() instanceof InsertEntity || getEntity() instanceof UpdateEntity)
+ && getEntity().hasFSODescriptor()) {
+ createDirs(getEntity().getFSODescriptor());
+ }
+ }
+
+ private void createDirs(final VirtualFSODescriptorInterface fso) {
+ VirtualFSODescriptorInterface nextFSO = fso;
+ String fileStorageId = nextFSO.getFileStorageId();
+ Path parentDir = nextFSO.getPath().getParent();
+ while (parentDir != null) {
+ final VirtualFSODescriptorInterface parentFSO = exists(parentDir);
+ if (parentFSO != null) {
+ // the parent does exist.
+ fso.setParentDirectory(parentFSO.getEntityId());
+ break;
+ }
+ nextFSO = createDir(fileStorageId, parentDir, nextFSO);
+ parentDir = parentDir.getParent();
+ }
+ if (parentDir == null) { // root of file system
+ nextFSO.setParentDirectory(MagicTypes.ROOT_DIRECTORY.getId());
+ }
+ }
+
+ private VirtualFSODescriptorInterface exists(final Path dirPath) {
+ for (final EntityInterface e : getContainer()) {
+ if (e.getRole() == Role.Directory
+ && (e instanceof InsertEntity || e instanceof UpdateEntity)
+ && e.getFSODescriptor().getPath().equals(dirPath)) {
+ // Found the directory in the container
+ return e.getFSODescriptor();
+ }
+ }
+ final GetFileEntityByPath t = new GetFileEntityByPath(dirPath);
+ try {
+ execute(t);
+ return t.getEntity().getFSODescriptor();
+ } catch (final EntityDoesNotExistException e) {
+ return null;
+ }
+ }
+
+ private VirtualFSODescriptorInterface createDir(
+ final String fileStorageId, final Path dirPath, final VirtualFSODescriptorInterface child) {
+ final String name = dirPath.getLastSegment();
+ final VirtualFSODescriptorInterface newFD = FSODescriptor.createDir(fileStorageId, dirPath);
+
+ child.setParentDirectory(newFD.getEntityId());
+
+ final EntityInterface newDir = new InsertEntity(name, Role.Directory);
+ newDir.setEntityACL(EntityACL.getOwnerACLFor(getUser()));
+ newDir.setFSODescriptor(newFD);
+ getContainer().add(newDir);
+
+ return newFD;
+ }
+}
diff --git a/src/main/java/org/caosdb/server/jobs/core/CheckFileStorageConsistency.java b/src/main/java/org/caosdb/server/jobs/core/CheckFileStorageConsistency.java
index bd6b42d2..8cc5d26e 100644
--- a/src/main/java/org/caosdb/server/jobs/core/CheckFileStorageConsistency.java
+++ b/src/main/java/org/caosdb/server/jobs/core/CheckFileStorageConsistency.java
@@ -38,6 +38,7 @@ import org.caosdb.server.database.exceptions.TransactionException;
import org.caosdb.server.entity.Message;
import org.caosdb.server.entity.Message.MessageType;
import org.caosdb.server.entity.xml.ToElementable;
+import org.caosdb.server.filesystem.FSODescriptor;
import org.caosdb.server.filesystem.FileSystem;
import org.caosdb.server.filesystem.Path;
import org.caosdb.server.filesystem.RealFSODescriptorInterface;
@@ -187,7 +188,8 @@ public class CheckFileStorageConsistency extends FlagJob {
if (f1 == null) {
return;
}
- Undoable undo1 = FileSystem.getInstance().getFileStorage(f1).move(f1, "somewhere.else");
+ FSODescriptor target = new FSODescriptor(new Path("debug/somewhere.else"));
+ Undoable undo1 = FileSystem.getInstance().getFileStorage(f1).move(f1, target);
getTransaction()
.acceptObserver(
diff --git a/src/main/java/org/caosdb/server/query/CQLLexer.g4 b/src/main/java/org/caosdb/server/query/CQLLexer.g4
index 99c9879d..1b113a27 100644
--- a/src/main/java/org/caosdb/server/query/CQLLexer.g4
+++ b/src/main/java/org/caosdb/server/query/CQLLexer.g4
@@ -343,6 +343,10 @@ FILE:
[Ff][Ii][Ll][Ee]([Ss])? WHITE_SPACE_f?
;
+DIRECTORY:
+ [Dd][Ii][Rr][Ee][Cc][Tt][Oo][Rr]([Yy]|[Ii][Ee][Ss]) WHITE_SPACE_f?
+;
+
ENTITY:
[Ee][Nn][Tt][Ii][Tt]([Yy]|[Ii][Ee][Ss]) WHITE_SPACE_f?
;
diff --git a/src/main/java/org/caosdb/server/query/CQLParser.g4 b/src/main/java/org/caosdb/server/query/CQLParser.g4
index d44674b2..a9656ca1 100644
--- a/src/main/java/org/caosdb/server/query/CQLParser.g4
+++ b/src/main/java/org/caosdb/server/query/CQLParser.g4
@@ -99,6 +99,7 @@ role returns [Query.Role r]:
| RECORD {$r = Query.Role.RECORD;}
| PROPERTY {$r = Query.Role.PROPERTY;}
| FILE {$r = Query.Role.FILE;}
+ | DIRECTORY {$r = Query.Role.DIRECTORY;}
| QUERYTEMPLATE {$r = Query.Role.QUERYTEMPLATE;}
| ENTITY {$r = Query.Role.ENTITY;}
;
diff --git a/src/main/java/org/caosdb/server/query/Query.java b/src/main/java/org/caosdb/server/query/Query.java
index 88da39bc..ac1a41a2 100644
--- a/src/main/java/org/caosdb/server/query/Query.java
+++ b/src/main/java/org/caosdb/server/query/Query.java
@@ -160,6 +160,7 @@ public class Query implements QueryInterface, ToElementable, TransactionInterfac
PROPERTY,
ENTITY,
FILE,
+ DIRECTORY,
QUERYTEMPLATE
}
--
GitLab