From f541d5f7e655471f5c4fafc7f9c0ad2003eac2c6 Mon Sep 17 00:00:00 2001
From: Timm Fitschen <t.fitschen@indiscale.com>
Date: Tue, 7 May 2024 22:07:13 +0200
Subject: [PATCH] WIP: import
---
Makefile | 4 +-
conf/core/jobs.csv | 3 +
mvn_install_jar.sh | 29 -----
.../java/org/caosdb/server/entity/Entity.java | 8 +-
.../caosdb/server/entity/InsertEntity.java | 4 +
.../server/filesystem/FSODescriptor.java | 21 ++++
.../VirtualFSODescriptorInterface.java | 2 +
.../consistency/ConsistencyCheck.java | 3 -
.../java/org/caosdb/server/jobs/FilesJob.java | 4 +-
.../jobs/core/CheckTargetPathValid.java | 10 --
.../caosdb/server/jobs/core/ImportFSO.java | 103 ++++++++++++++++++
.../server/transaction/WriteTransaction.java | 40 +++++++
.../caosdb/server/utils/ServerMessages.java | 6 +
13 files changed, 191 insertions(+), 46 deletions(-)
delete mode 100755 mvn_install_jar.sh
create mode 100644 src/main/java/org/caosdb/server/jobs/core/ImportFSO.java
diff --git a/Makefile b/Makefile
index 29b7d5a5..caa14420 100644
--- a/Makefile
+++ b/Makefile
@@ -26,8 +26,8 @@
CAOSDB_SERVER_VERSION ?= $(shell mvn org.apache.maven.plugins:maven-help-plugin:3.1.0:evaluate -Dexpression=project.version -q -DforceStdout)
CAOSDB_COMMAND_LINE_OPTIONS ?=
SHELL:=/bin/bash
-JPDA_PORT ?= 9000
-JMX_PORT ?= 9090
+JPDA_PORT ?= 9001
+JMX_PORT ?= 9091
compile: print-version easy-units
mvn compile
diff --git a/conf/core/jobs.csv b/conf/core/jobs.csv
index 6f6be872..013f4984 100644
--- a/conf/core/jobs.csv
+++ b/conf/core/jobs.csv
@@ -56,6 +56,7 @@
0,2,UPDATE,SetImpToFix,ERROR
## file rules
+0,3,INSERT,ImportFSO,ERROR
0,3,INSERT,CheckNamePresent,WARN
0,3,INSERT,MatchFileProp,ERROR
0,3,INSERT,CheckTargetPathValid,ERROR
@@ -67,6 +68,7 @@
0,3,UPDATE,SetImpToFix,ERROR
## directory rules
+0,9,INSERT,ImportFSO,ERROR
0,9,INSERT,CheckNamePresent,WARN
0,9,INSERT,MatchFileProp,ERROR
0,9,INSERT,CheckTargetPathValid,ERROR
@@ -78,6 +80,7 @@
0,9,UPDATE,SetImpToFix,ERROR
## link rules
+0,10,INSERT,ImportFSO,ERROR
0,10,INSERT,CheckNamePresent,WARN
0,10,INSERT,MatchFileProp,ERROR
0,10,INSERT,CheckTargetPathValid,ERROR
diff --git a/mvn_install_jar.sh b/mvn_install_jar.sh
deleted file mode 100755
index b7bdb6cf..00000000
--- a/mvn_install_jar.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-#
-# ** header v3.0
-# This file is a part of the CaosDB Project.
-#
-# Copyright (C) 2018 Research Group Biomedical Physics,
-# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <https://www.gnu.org/licenses/>.
-#
-# ** end header
-#
-
-# Alexander Schlemmer, 01/2013
-# Installiert die jar-Datei des Servers in das lokale mvn-Repository.
-
-mvn install:install-file -DgroupId=ds.mpg.de -DartifactId=caosdb-server \
- -Dversion=1.0 -Dpackaging=jar -Dfile=dist/CaosDBServer.jar
diff --git a/src/main/java/org/caosdb/server/entity/Entity.java b/src/main/java/org/caosdb/server/entity/Entity.java
index 7ea937ef..4c4aed5c 100644
--- a/src/main/java/org/caosdb/server/entity/Entity.java
+++ b/src/main/java/org/caosdb/server/entity/Entity.java
@@ -837,7 +837,12 @@ public abstract class Entity extends AbstractObservable implements EntityInterfa
// Parse IMPORT
boolean isImport = false;
- if (element.getAttribute("import") != null
+ boolean isRecursiveImport = false;
+ if (element.getAttribute("recursive_import") != null
+ && element.getAttributeValue("recursive_import").equals("true")) {
+ isImport = true;
+ isRecursiveImport = true;
+ } else if (element.getAttribute("import") != null
&& element.getAttributeValue("import").equals("true")) {
isImport = true;
}
@@ -859,6 +864,7 @@ public abstract class Entity extends AbstractObservable implements EntityInterfa
FileSystem.DEFAULT_BACKEND, null, hash, new Path(path), size, tmpIdentifier);
}
fso.setImport(isImport);
+ fso.setRecursiveImport(isRecursiveImport);
setFSODescriptor(fso);
}
diff --git a/src/main/java/org/caosdb/server/entity/InsertEntity.java b/src/main/java/org/caosdb/server/entity/InsertEntity.java
index b88b319f..2992beac 100644
--- a/src/main/java/org/caosdb/server/entity/InsertEntity.java
+++ b/src/main/java/org/caosdb/server/entity/InsertEntity.java
@@ -34,6 +34,10 @@ public class InsertEntity extends WritableEntity {
super(name, role);
}
+ public InsertEntity(Role role) {
+ super((String) null, role);
+ }
+
public InsertEntity(Element e) {
super(e);
}
diff --git a/src/main/java/org/caosdb/server/filesystem/FSODescriptor.java b/src/main/java/org/caosdb/server/filesystem/FSODescriptor.java
index 9c4d62bd..008e5703 100644
--- a/src/main/java/org/caosdb/server/filesystem/FSODescriptor.java
+++ b/src/main/java/org/caosdb/server/filesystem/FSODescriptor.java
@@ -44,6 +44,7 @@ public class FSODescriptor implements VirtualFSODescriptorInterface {
private RealFSODescriptorInterface twin;
private EntityID linkTarget = null;
private boolean isImport = false;
+ private boolean isRecursiveImport;
public FSODescriptor(
final String fileStorageId,
@@ -91,6 +92,17 @@ public class FSODescriptor implements VirtualFSODescriptorInterface {
this.size = size;
}
+ public FSODescriptor(RealFSODescriptorInterface twin, Path path) {
+ this.path = path;
+ this.type = twin.getObjectType();
+ this.hash = twin.getHash();
+ this.size = twin.getSize();
+ this.fileStorageId = twin.getFileStorageId();
+ this.key = twin.getKey();
+ this.mimeType = twin.getMimeType();
+ this.twin = twin;
+ }
+
@Override
public EntityStatus getEntityStatus() {
return status;
@@ -291,7 +303,16 @@ public class FSODescriptor implements VirtualFSODescriptorInterface {
return isImport;
}
+ @Override
+ public boolean isRecursiveImport() {
+ return isRecursiveImport;
+ }
+
public void setImport(boolean isImport) {
this.isImport = isImport;
}
+
+ public void setRecursiveImport(boolean isRecursiveImport) {
+ this.isRecursiveImport = isRecursiveImport;
+ }
}
diff --git a/src/main/java/org/caosdb/server/filesystem/VirtualFSODescriptorInterface.java b/src/main/java/org/caosdb/server/filesystem/VirtualFSODescriptorInterface.java
index 4e65b45b..e907dac8 100644
--- a/src/main/java/org/caosdb/server/filesystem/VirtualFSODescriptorInterface.java
+++ b/src/main/java/org/caosdb/server/filesystem/VirtualFSODescriptorInterface.java
@@ -128,4 +128,6 @@ public interface VirtualFSODescriptorInterface extends FSODescriptorInterface {
public abstract void setLinkTargetPath(Path target);
public abstract boolean isImport();
+
+ boolean isRecursiveImport();
}
diff --git a/src/main/java/org/caosdb/server/filesystem/consistency/ConsistencyCheck.java b/src/main/java/org/caosdb/server/filesystem/consistency/ConsistencyCheck.java
index 71e1a2cd..af33421d 100644
--- a/src/main/java/org/caosdb/server/filesystem/consistency/ConsistencyCheck.java
+++ b/src/main/java/org/caosdb/server/filesystem/consistency/ConsistencyCheck.java
@@ -83,7 +83,6 @@ class RunThroughInternalFileSystem extends ConsistencyCheckStrategy {
while (iterator != null && iterator.hasNext()) {
long timestamp = System.currentTimeMillis();
- System.err.println("here");
if (DatabaseAccessManager.whoHasReservedWriteAccess() != null) {
// there is a thread waiting to write. pause this one and
@@ -92,11 +91,9 @@ class RunThroughInternalFileSystem extends ConsistencyCheckStrategy {
this.getAccess().release();
this.setAccess(DatabaseAccessManager.getInstance().acquireReadAccess(this));
iterator = execute(it, this.getAccess()).getIterator();
- System.err.println("here2");
}
final VirtualFSODescriptorInterface howItShouldBe = iterator.next();
- System.err.println("check: " + howItShouldBe.toString());
check(howItShouldBe);
diff --git a/src/main/java/org/caosdb/server/jobs/FilesJob.java b/src/main/java/org/caosdb/server/jobs/FilesJob.java
index 70b099ea..f746e7f7 100644
--- a/src/main/java/org/caosdb/server/jobs/FilesJob.java
+++ b/src/main/java/org/caosdb/server/jobs/FilesJob.java
@@ -52,6 +52,8 @@ public abstract class FilesJob extends EntityJob {
throw ServerMessages.FILE_IMPORT_FAILED_FILE_DOES_NOT_EXIST;
}
}
- return getContainer().getFiles().get(fso.getTmpIdentifier());
+ RealFSODescriptorInterface twin = getContainer().getFiles().get(fso.getTmpIdentifier());
+ fso.setTwin(twin);
+ return twin;
}
}
diff --git a/src/main/java/org/caosdb/server/jobs/core/CheckTargetPathValid.java b/src/main/java/org/caosdb/server/jobs/core/CheckTargetPathValid.java
index 60e5ddd5..6d381553 100644
--- a/src/main/java/org/caosdb/server/jobs/core/CheckTargetPathValid.java
+++ b/src/main/java/org/caosdb/server/jobs/core/CheckTargetPathValid.java
@@ -23,7 +23,6 @@
package org.caosdb.server.jobs.core;
import org.caosdb.server.database.backend.transaction.CheckTargetPath;
-import org.caosdb.server.filesystem.FileSystem;
import org.caosdb.server.filesystem.VirtualFSODescriptorInterface;
import org.caosdb.server.jobs.FilesJob;
import org.caosdb.server.utils.EntityStatus;
@@ -52,15 +51,6 @@ public class CheckTargetPathValid extends FilesJob {
// check that target path is not owned by another entity.
final CheckTargetPath t = new CheckTargetPath(getEntity(), getContainer());
execute(t);
-
- // check if the fso exists in the file storage
- if (FileSystem.exists(getEntity().getFSODescriptor())) {
- if (getEntity().getFSODescriptor().isImport()) {
- // ok?
- } else {
- // is update?
- }
- }
}
}
}
diff --git a/src/main/java/org/caosdb/server/jobs/core/ImportFSO.java b/src/main/java/org/caosdb/server/jobs/core/ImportFSO.java
new file mode 100644
index 00000000..a86da810
--- /dev/null
+++ b/src/main/java/org/caosdb/server/jobs/core/ImportFSO.java
@@ -0,0 +1,103 @@
+package org.caosdb.server.jobs.core;
+
+import java.util.LinkedList;
+import java.util.List;
+import org.caosdb.server.entity.EntityInterface;
+import org.caosdb.server.entity.InsertEntity;
+import org.caosdb.server.entity.Message;
+import org.caosdb.server.entity.Role;
+import org.caosdb.server.filesystem.FSODescriptor;
+import org.caosdb.server.filesystem.FileSystem.ObjectType;
+import org.caosdb.server.filesystem.Path;
+import org.caosdb.server.filesystem.RealFSODescriptorInterface;
+import org.caosdb.server.filesystem.VirtualFSODescriptorInterface;
+import org.caosdb.server.jobs.FilesJob;
+import org.caosdb.server.jobs.Job;
+import org.caosdb.server.jobs.JobAnnotation;
+import org.caosdb.server.jobs.TransactionStage;
+import org.caosdb.server.utils.ServerMessages;
+
+@JobAnnotation(stage = TransactionStage.PRE_CHECK)
+public class ImportFSO extends FilesJob {
+
+ @Override
+ protected void run() throws Message {
+ if (getEntity().hasFSODescriptor() && getEntity().getFSODescriptor().isImport()) {
+ VirtualFSODescriptorInterface fso = getEntity().getFSODescriptor();
+ RealFSODescriptorInterface realFSO = getRealFSODescriptor(fso);
+
+ if (realFSO == null) {
+ // handle
+ }
+
+ if (getEntity().getFSODescriptor().isRecursiveImport()
+ && fso.getObjectType() == ObjectType.DIRECTORY) {
+ assertIsDirectory(realFSO);
+ fso.setChildren(importRecursively(getEntity(), realFSO, fso.getPath()));
+ }
+ // else warn not a Directory?
+ }
+ }
+
+ private List<VirtualFSODescriptorInterface> importRecursively(
+ EntityInterface parentDirectory, RealFSODescriptorInterface directory, Path path) {
+ List<? extends RealFSODescriptorInterface> children = directory.listChildren();
+ List<VirtualFSODescriptorInterface> results = new LinkedList<>();
+ for (RealFSODescriptorInterface child : children) {
+ Path childPath = path.append(child.getName());
+ switch (child.getObjectType()) {
+ case DIRECTORY:
+ EntityInterface newDirectory = importDirectory(parentDirectory, child, childPath);
+ results.addAll(importRecursively(newDirectory, child, childPath));
+ break;
+ case LINK:
+ results.add(importLink(parentDirectory, child, childPath));
+ break;
+ default:
+ results.add(importFile(parentDirectory, child, childPath));
+ break;
+ }
+ }
+ return results;
+ }
+
+ private EntityInterface importDirectory(
+ EntityInterface parentDirectory, RealFSODescriptorInterface toBeImported, Path path) {
+ InsertEntity directoryEntity = new InsertEntity(Role.Directory);
+ importFSO(directoryEntity, parentDirectory, toBeImported, path);
+ return directoryEntity;
+ }
+
+ private VirtualFSODescriptorInterface importLink(
+ EntityInterface parentDirectory, RealFSODescriptorInterface toBeImported, Path path) {
+ throw new UnsupportedOperationException();
+ }
+
+ private VirtualFSODescriptorInterface importFile(
+ EntityInterface parentDirectory, RealFSODescriptorInterface toBeImported, Path path) {
+ InsertEntity fileEntity = new InsertEntity(Role.File);
+ return importFSO(fileEntity, parentDirectory, toBeImported, path);
+ }
+
+ private VirtualFSODescriptorInterface importFSO(
+ InsertEntity entity,
+ EntityInterface parentDirectory,
+ RealFSODescriptorInterface toBeImported,
+ Path path) {
+ entity.setEntityACL(getEntity().getEntityACL());
+ VirtualFSODescriptorInterface fso = new FSODescriptor(toBeImported, path);
+ fso.setParentDirectory(parentDirectory.getId());
+ entity.setFSODescriptor(fso);
+ getContainer().add(entity);
+
+ final List<Job> loadJobs = this.loadJobs(entity, getTransaction());
+ getTransaction().getSchedule().addAll(loadJobs);
+ return fso;
+ }
+
+ private void assertIsDirectory(RealFSODescriptorInterface realFSO) throws Message {
+ if (realFSO.getObjectType() != ObjectType.DIRECTORY) {
+ throw ServerMessages.CANNOT_PERFORM_RECURSIVE_IMPORT_ON_NON_DIRECTORIES;
+ }
+ }
+}
diff --git a/src/main/java/org/caosdb/server/transaction/WriteTransaction.java b/src/main/java/org/caosdb/server/transaction/WriteTransaction.java
index a35b7a59..4193a0df 100644
--- a/src/main/java/org/caosdb/server/transaction/WriteTransaction.java
+++ b/src/main/java/org/caosdb/server/transaction/WriteTransaction.java
@@ -52,10 +52,12 @@ import org.caosdb.server.entity.wrapper.Parent;
import org.caosdb.server.entity.wrapper.Property;
import org.caosdb.server.filesystem.FSODescriptorInterface;
import org.caosdb.server.filesystem.FileSystem;
+import org.caosdb.server.filesystem.FileSystem.ObjectType;
import org.caosdb.server.filesystem.RealFSODescriptorInterface;
import org.caosdb.server.filesystem.VirtualFSODescriptorInterface;
import org.caosdb.server.jobs.Job;
import org.caosdb.server.jobs.Schedule;
+import org.caosdb.server.jobs.core.ImportFSO;
import org.caosdb.server.permissions.EntityACL;
import org.caosdb.server.permissions.EntityPermission;
import org.caosdb.server.permissions.Permission;
@@ -478,6 +480,18 @@ public class WriteTransaction extends Transaction<WritableContainer>
needPermissions.add(EntityPermission.UPDATE_REMOVE_FILE);
needPermissions.add(EntityPermission.UPDATE_ADD_FILE);
updateFile = true;
+ } else if (newFile.isImport()) {
+ RealFSODescriptorInterface realFso = FileSystem.getInstance().resolve(oldFile);
+ if (realFso.getSize() != oldFile.getSize()) {
+ updateFile = true;
+ } else if (!oldFile.getHash().equals(realFso.getHash())) {
+ updateFile = true;
+ } else if (newFile.isRecursiveImport()
+ && realFso.getObjectType() == ObjectType.DIRECTORY) {
+ // run import job now
+ getSchedule().runJob(newEntity, ImportFSO.class);
+ updateFile = matchFso(newEntity.getFSODescriptor(), oldEntity.getFSODescriptor());
+ }
}
}
@@ -549,12 +563,38 @@ public class WriteTransaction extends Transaction<WritableContainer>
// nothing to be updated
if (!updatetable) {
newEntity.setEntityStatus(EntityStatus.VALID);
+ newEntity.setVersion(oldEntity.getVersion());
newEntity.addInfo("Nothing to be updated.");
}
return needPermissions;
}
+ private boolean matchFso(
+ VirtualFSODescriptorInterface newFsoDescriptor,
+ VirtualFSODescriptorInterface oldFsoDescriptor) {
+ boolean updateFso = false;
+
+ if(oldFsoDescriptor.getObjectType() != newFsoDescriptor.getObjectType()) {
+ throw new UnsupportedOperationException("Cannot update the file system object type.");
+ }
+
+ // now it is clear that both have the same object type.
+ switch(newFsoDescriptor.getObjectType()) {
+ case DIRECTORY:
+ break;
+ case LINK:
+ break;
+ default:
+ boolean pathEquals = Objects.equals(newFsoDescriptor.getPath(), oldFsoDescriptor.getPath());
+ boolean
+
+ break;
+ }
+
+ return updateFso;
+ }
+
/**
* Attempt to find a (sparse) entity among a list of entities.
*
diff --git a/src/main/java/org/caosdb/server/utils/ServerMessages.java b/src/main/java/org/caosdb/server/utils/ServerMessages.java
index cc963ccf..5ea9aa55 100644
--- a/src/main/java/org/caosdb/server/utils/ServerMessages.java
+++ b/src/main/java/org/caosdb/server/utils/ServerMessages.java
@@ -607,6 +607,12 @@ public class ServerMessages {
MessageCode.MESSAGE_CODE_INTEGRITY_VIOLATION,
"This entity caused an unexpected integrity violation. This is a strong indicator for a server bug. Please report.");
+ public static final Message CANNOT_PERFORM_RECURSIVE_IMPORT_ON_NON_DIRECTORIES =
+ new Message(
+ MessageType.Error,
+ MessageCode.MESSAGE_CODE_UNKNOWN,
+ "Cannot perform recursive import on non-directory objects.");
+
public static final Message INVALID_USER_NAME(String policy) {
return new Message(
MessageType.Error,
--
GitLab