WIP: file storage refactoring: directory

src/main/java/org/caosdb/server/entity/Entity.java

@@ -809,14 +809,13 @@ public abstract class Entity extends AbstractObservable implements EntityInterfa
     if (tmpIdentifier != null || checksum != null || path != null || size != null) {
       // legacy clients (which use this api) always use sha512.
       Hash hash = Hash.create(checksum, 0, Hasher.Default);
+      if (getRole() == Role.Directory) {
+        setFSODescriptor(FSODescriptor.createDir(FileSystem.DEFAULT_BACKEND, new Path(path)));
+      } else {
         setFSODescriptor(
             new FSODescriptor(
-              FileSystem.DEFAULT_BACKEND,
-              new Path(path).toString(),
-              hash,
-              new Path(path),
-              size,
-              tmpIdentifier));
+                FileSystem.DEFAULT_BACKEND, null, hash, new Path(path), size, tmpIdentifier));
+      }
     }
 
     // Parse flags

src/main/java/org/caosdb/server/entity/MagicTypes.java

@@ -31,11 +31,13 @@ import org.caosdb.server.transaction.Retrieve;
 public enum MagicTypes {
   UNIT,
   NAME,
-  DESCRIPTION;
+  DESCRIPTION,
+  ROOT_DIRECTORY;
 
   private static final EntityID UNIT_ID = new EntityID(21);
   private static final EntityID DESCRIPTION_ID = new EntityID(24);
   private static final EntityID NAME_ID = new EntityID(20);
+  private static final EntityID ROOT_DIRECTORY_ID = new EntityID(51);
 
   public EntityID getId() {
     switch (this) {
@@ -45,6 +47,8 @@ public enum MagicTypes {
         return NAME_ID;
       case DESCRIPTION:
         return DESCRIPTION_ID;
+      case ROOT_DIRECTORY:
+        return ROOT_DIRECTORY_ID;
       default:
         return null;
     }
@@ -58,6 +62,8 @@ public enum MagicTypes {
         return NAME;
       case 24:
         return DESCRIPTION;
+      case 51:
+        return ROOT_DIRECTORY;
       default:
         return null;
     }

src/main/java/org/caosdb/server/entity/Role.java

@@ -35,7 +35,6 @@ public enum Role {
   Domain,
   File,
   Directory,
-  Link,
   Property,
   DataType,
   QueryTemplate;
@@ -75,14 +74,12 @@ public enum Role {
         return new FileToElementStrategy(toString());
       case Directory:
         return new FileToElementStrategy(toString());
-      case Link:
-        return new FileToElementStrategy(toString());
       default:
         return new EntityToElementStrategy(toString());
     }
   }
 
   public boolean hasFSODescriptor() {
-    return this == File; // || this == Directory;
+    return this == File || this == Directory;
   }
 }

src/main/java/org/caosdb/server/filesystem/FSODescriptor.java

@@ -187,8 +187,9 @@ public class FSODescriptor implements VirtualFSODescriptorInterface {
     this.children = children;
   }
 
-  public static FSODescriptor createDir(final Path dirPath) {
+  public static FSODescriptor createDir(String fileStorageId, final Path dirPath) {
     final FSODescriptor ret = new FSODescriptor(dirPath);
+    ret.fileStorageId = fileStorageId;
     ret.type = ObjectType.DIRECTORY;
     return ret;
   }
@@ -252,4 +253,9 @@ public class FSODescriptor implements VirtualFSODescriptorInterface {
   public void setTwin(RealFSODescriptorInterface twin) {
     this.twin = twin;
   }
+
+  @Override
+  public void setKey(String key) {
+    this.key = key;
+  }
 }

src/main/java/org/caosdb/server/filesystem/FileStorageInterface.java

@@ -39,7 +39,8 @@ public interface FileStorageInterface {
    */
   public abstract RealFSODescriptorInterface resolve(String fileKey);
 
-  public abstract Undoable move(RealFSODescriptorInterface file, String fileKey) throws Message;
+  public abstract Undoable move(
+      RealFSODescriptorInterface file, VirtualFSODescriptorInterface target) throws Message;
 
   public abstract Undoable delete(String fileKey) throws Message;
 

src/main/java/org/caosdb/server/filesystem/FileSystem.java

@@ -46,12 +46,6 @@ public class FileSystem implements FileSystemInterface {
     return fs.delete(file.getKey());
   }
 
-  @Override
-  public Undoable store(final RealFSODescriptorInterface file) throws Message {
-    final FileStorageInterface fs = getFileStorage(file);
-    return fs.move(file, file.getKey());
-  }
-
   public static FileSystem getInstance() {
     return instance;
   }
@@ -98,10 +92,10 @@ public class FileSystem implements FileSystemInterface {
     return null;
   }
 
-  public Undoable move(RealFSODescriptorInterface fso, FSODescriptorInterface target)
+  public Undoable move(RealFSODescriptorInterface fso, VirtualFSODescriptorInterface target)
       throws Message {
     FileStorageInterface fileStorage = getFileStorage(target);
-    return fileStorage.move(fso, target.getKey());
+    return fileStorage.move(fso, target);
   }
 
   //  public static boolean resolve(FSODescriptorInterface fso) {

src/main/java/org/caosdb/server/filesystem/FileSystemInterface.java

@@ -10,16 +10,6 @@ import org.caosdb.server.utils.Undoable;
  */
 public interface FileSystemInterface {
 
-  /**
-   * Store the file described by the FileDescriptorInterface.
-   *
-   * @param file
-   * @return an {@link Undoable} object, which reverts the storage procedure and puts the file back
-   *     to where it was. To be called when failures occured and the transaction needs to roll back.
-   * @throws Message
-   */
-  public abstract Undoable store(RealFSODescriptorInterface file) throws Message;
-
   /**
    * Delete the file described by the FileDescriptorInterface.
    *

src/main/java/org/caosdb/server/filesystem/LocalFileStorage.java

@@ -285,10 +285,17 @@ public abstract class LocalFileStorage implements FileStorageInterface {
   }
 
   @Override
-  public Undoable move(final RealFSODescriptorInterface file, String key) throws Message {
+  public Undoable move(final RealFSODescriptorInterface file, VirtualFSODescriptorInterface target)
+      throws Message {
+    String key = target.getKey();
+    if (key == null) {
+      key = target.getPath().toString();
+      target.setKey(key);
+    }
+
     final Path targetPath = getRoot().resolve(key);
 
-    if (file.getObjectType() == ObjectType.DIRECTORY) {
+    if (target.getObjectType() == ObjectType.DIRECTORY) {
       final File newDirectory = createDirs(targetPath.toFile());
       return new Undoable() {
 
@@ -319,7 +326,7 @@ public abstract class LocalFileStorage implements FileStorageInterface {
   public Undoable delete(final String fileKey) throws Message {
     final Path targetPath = getRoot().resolve(fileKey);
 
-    return deleteWithEmptyParentDir(targetPath.toFile(), true);
+    return delete(targetPath.toFile(), true);
   }
 
   public Path getRoot() {
@@ -613,17 +620,6 @@ public abstract class LocalFileStorage implements FileStorageInterface {
     };
   }
 
-  Undoable deleteWithEmptyParentDir(final File f, final boolean recursive) throws Message {
-    final File parent = f.getParentFile();
-    if (!getRoot().toFile().equals(parent)) {
-      final File[] siblings = parent.listFiles();
-      if (siblings == null || siblings.length == 1 && siblings[0].equals(f)) {
-        return deleteWithEmptyParentDir(parent, recursive);
-      }
-    }
-    return delete(f, recursive);
-  }
-
   public Long getSize(String key) {
     File file = getFile(key);
     if (file == null) {

src/main/java/org/caosdb/server/filesystem/VirtualFSODescriptorInterface.java

@@ -120,4 +120,6 @@ public interface VirtualFSODescriptorInterface extends FSODescriptorInterface {
 
   @Override
   public abstract List<? extends VirtualFSODescriptorInterface> listChildren();
+
+  public abstract void setKey(String key);
 }

src/main/java/org/caosdb/server/jobs/core/AutoCreateDirs.java

+package org.caosdb.server.jobs.core;
+
+import org.caosdb.server.database.backend.transaction.GetFileEntityByPath;
+import org.caosdb.server.database.exceptions.EntityDoesNotExistException;
+import org.caosdb.server.entity.EntityInterface;
+import org.caosdb.server.entity.InsertEntity;
+import org.caosdb.server.entity.MagicTypes;
+import org.caosdb.server.entity.Role;
+import org.caosdb.server.entity.UpdateEntity;
+import org.caosdb.server.filesystem.FSODescriptor;
+import org.caosdb.server.filesystem.Path;
+import org.caosdb.server.filesystem.VirtualFSODescriptorInterface;
+import org.caosdb.server.jobs.EntityFlagJob;
+import org.caosdb.server.jobs.JobAnnotation;
+import org.caosdb.server.permissions.EntityACL;
+
+@JobAnnotation(flag = "autoCreateDirs", defaultValue = "true", loadAlways = true)
+public class AutoCreateDirs extends EntityFlagJob {
+
+  @Override
+  protected void job(final String value) {
+    final boolean createDirs = "true".equalsIgnoreCase(value);
+
+    if (createDirs
+        && (getEntity() instanceof InsertEntity || getEntity() instanceof UpdateEntity)
+        && getEntity().hasFSODescriptor()) {
+      createDirs(getEntity().getFSODescriptor());
+    }
+  }
+
+  private void createDirs(final VirtualFSODescriptorInterface fso) {
+    VirtualFSODescriptorInterface nextFSO = fso;
+    String fileStorageId = nextFSO.getFileStorageId();
+    Path parentDir = nextFSO.getPath().getParent();
+    while (parentDir != null) {
+      final VirtualFSODescriptorInterface parentFSO = exists(parentDir);
+      if (parentFSO != null) {
+        // the parent does exist.
+        fso.setParentDirectory(parentFSO.getEntityId());
+        break;
+      }
+      nextFSO = createDir(fileStorageId, parentDir, nextFSO);
+      parentDir = parentDir.getParent();
+    }
+    if (parentDir == null) { // root of file system
+      nextFSO.setParentDirectory(MagicTypes.ROOT_DIRECTORY.getId());
+    }
+  }
+
+  private VirtualFSODescriptorInterface exists(final Path dirPath) {
+    for (final EntityInterface e : getContainer()) {
+      if (e.getRole() == Role.Directory
+          && (e instanceof InsertEntity || e instanceof UpdateEntity)
+          && e.getFSODescriptor().getPath().equals(dirPath)) {
+        // Found the directory in the container
+        return e.getFSODescriptor();
+      }
+    }
+    final GetFileEntityByPath t = new GetFileEntityByPath(dirPath);
+    try {
+      execute(t);
+      return t.getEntity().getFSODescriptor();
+    } catch (final EntityDoesNotExistException e) {
+      return null;
+    }
+  }
+
+  private VirtualFSODescriptorInterface createDir(
+      final String fileStorageId, final Path dirPath, final VirtualFSODescriptorInterface child) {
+    final String name = dirPath.getLastSegment();
+    final VirtualFSODescriptorInterface newFD = FSODescriptor.createDir(fileStorageId, dirPath);
+
+    child.setParentDirectory(newFD.getEntityId());
+
+    final EntityInterface newDir = new InsertEntity(name, Role.Directory);
+    newDir.setEntityACL(EntityACL.getOwnerACLFor(getUser()));
+    newDir.setFSODescriptor(newFD);
+    getContainer().add(newDir);
+
+    return newFD;
+  }
+}

src/main/java/org/caosdb/server/jobs/core/CheckFileStorageConsistency.java

@@ -38,6 +38,7 @@ import org.caosdb.server.database.exceptions.TransactionException;
 import org.caosdb.server.entity.Message;
 import org.caosdb.server.entity.Message.MessageType;
 import org.caosdb.server.entity.xml.ToElementable;
+import org.caosdb.server.filesystem.FSODescriptor;
 import org.caosdb.server.filesystem.FileSystem;
 import org.caosdb.server.filesystem.Path;
 import org.caosdb.server.filesystem.RealFSODescriptorInterface;
@@ -187,7 +188,8 @@ public class CheckFileStorageConsistency extends FlagJob {
         if (f1 == null) {
           return;
         }
-        Undoable undo1 = FileSystem.getInstance().getFileStorage(f1).move(f1, "somewhere.else");
+        FSODescriptor target = new FSODescriptor(new Path("debug/somewhere.else"));
+        Undoable undo1 = FileSystem.getInstance().getFileStorage(f1).move(f1, target);
 
         getTransaction()
             .acceptObserver(

src/main/java/org/caosdb/server/query/CQLLexer.g4

@@ -343,6 +343,10 @@ FILE:
     [Ff][Ii][Ll][Ee]([Ss])? WHITE_SPACE_f?
 ;
 
+DIRECTORY:
+	[Dd][Ii][Rr][Ee][Cc][Tt][Oo][Rr]([Yy]|[Ii][Ee][Ss]) WHITE_SPACE_f?
+;
+
 ENTITY:
     [Ee][Nn][Tt][Ii][Tt]([Yy]|[Ii][Ee][Ss]) WHITE_SPACE_f?
 ;

src/main/java/org/caosdb/server/query/CQLParser.g4

@@ -99,6 +99,7 @@ role returns [Query.Role r]:
     | RECORD {$r = Query.Role.RECORD;}
     | PROPERTY {$r = Query.Role.PROPERTY;}
     | FILE {$r = Query.Role.FILE;}
+    | DIRECTORY {$r = Query.Role.DIRECTORY;}
     | QUERYTEMPLATE {$r = Query.Role.QUERYTEMPLATE;}
     | ENTITY {$r = Query.Role.ENTITY;}
 ;

src/main/java/org/caosdb/server/query/Query.java

@@ -160,6 +160,7 @@ public class Query implements QueryInterface, ToElementable, TransactionInterfac
     PROPERTY,
     ENTITY,
     FILE,
+    DIRECTORY,
     QUERYTEMPLATE
   }