diff --git a/src/main/java/org/caosdb/server/database/BackendTransaction.java b/src/main/java/org/caosdb/server/database/BackendTransaction.java
index 4f41f7a3d4e5f4b077abbf7a2a6aa9e196112bc2..c4af8f462c3ece4b06e623bebc2d049433a36322 100644
--- a/src/main/java/org/caosdb/server/database/BackendTransaction.java
+++ b/src/main/java/org/caosdb/server/database/BackendTransaction.java
@@ -36,6 +36,7 @@ import org.caosdb.server.database.backend.implementation.MySQL.MySQLGetFilesInDi
import org.caosdb.server.database.backend.implementation.MySQL.MySQLGetIDByName;
import org.caosdb.server.database.backend.implementation.MySQL.MySQLGetInfo;
import org.caosdb.server.database.backend.implementation.MySQL.MySQLGetUpdateableChecksums;
+import org.caosdb.server.database.backend.implementation.MySQL.MySQLGetVirtualFSO;
import org.caosdb.server.database.backend.implementation.MySQL.MySQLInsertEntityDatatype;
import org.caosdb.server.database.backend.implementation.MySQL.MySQLInsertEntityProperties;
import org.caosdb.server.database.backend.implementation.MySQL.MySQLInsertFSODescriptor;
@@ -52,7 +53,6 @@ import org.caosdb.server.database.backend.implementation.MySQL.MySQLListUsers;
import org.caosdb.server.database.backend.implementation.MySQL.MySQLLogUserVisit;
import org.caosdb.server.database.backend.implementation.MySQL.MySQLRegisterSubDomain;
import org.caosdb.server.database.backend.implementation.MySQL.MySQLRetrieveAll;
-import org.caosdb.server.database.backend.implementation.MySQL.MySQLRetrieveAllUncheckedFiles;
import org.caosdb.server.database.backend.implementation.MySQL.MySQLRetrieveDatatypes;
import org.caosdb.server.database.backend.implementation.MySQL.MySQLRetrieveEntityACL;
import org.caosdb.server.database.backend.implementation.MySQL.MySQLRetrieveLogRecord;
@@ -88,6 +88,7 @@ import org.caosdb.server.database.backend.interfaces.GetFilesInDirectoryImpl;
import org.caosdb.server.database.backend.interfaces.GetIDByNameImpl;
import org.caosdb.server.database.backend.interfaces.GetInfoImpl;
import org.caosdb.server.database.backend.interfaces.GetUpdateableChecksumsImpl;
+import org.caosdb.server.database.backend.interfaces.GetVirtualFSOImpl;
import org.caosdb.server.database.backend.interfaces.InsertEntityDatatypeImpl;
import org.caosdb.server.database.backend.interfaces.InsertEntityPropertiesImpl;
import org.caosdb.server.database.backend.interfaces.InsertFSODescriptorImpl;
@@ -104,7 +105,6 @@ import org.caosdb.server.database.backend.interfaces.ListUsersImpl;
import org.caosdb.server.database.backend.interfaces.LogUserVisitImpl;
import org.caosdb.server.database.backend.interfaces.RegisterSubDomainImpl;
import org.caosdb.server.database.backend.interfaces.RetrieveAllImpl;
-import org.caosdb.server.database.backend.interfaces.RetrieveAllUncheckedFilesImpl;
import org.caosdb.server.database.backend.interfaces.RetrieveDatatypesImpl;
import org.caosdb.server.database.backend.interfaces.RetrieveEntityACLImpl;
import org.caosdb.server.database.backend.interfaces.RetrieveLogRecordImpl;
@@ -184,7 +184,6 @@ public abstract class BackendTransaction implements Undoable {
setImpl(RetrieveSparseEntityImpl.class, MySQLRetrieveSparseEntity.class);
setImpl(SyncStatsImpl.class, MySQLSyncStats.class);
setImpl(SetFileCheckedTimestampImpl.class, MySQLSetFileCheckedTimestamp.class);
- setImpl(RetrieveAllUncheckedFilesImpl.class, MySQLRetrieveAllUncheckedFiles.class);
setImpl(UpdateUserImpl.class, MySQLUpdateUser.class);
setImpl(DeleteUserImpl.class, MySQLDeleteUser.class);
setImpl(SetPasswordImpl.class, MySQLSetPassword.class);
@@ -212,6 +211,7 @@ public abstract class BackendTransaction implements Undoable {
setImpl(ListUsersImpl.class, MySQLListUsers.class);
setImpl(LogUserVisitImpl.class, MySQLLogUserVisit.class);
setImpl(RetrieveEntityACLImpl.class, MySQLRetrieveEntityACL.class);
+ setImpl(GetVirtualFSOImpl.class, MySQLGetVirtualFSO.class);
}
}
diff --git a/src/main/java/org/caosdb/server/database/backend/implementation/MySQL/MySQLGetVirtualFSO.java b/src/main/java/org/caosdb/server/database/backend/implementation/MySQL/MySQLGetVirtualFSO.java
new file mode 100644
index 0000000000000000000000000000000000000000..71b0ba1e99789e9fd08d7d1ca5abd5639baf2939
--- /dev/null
+++ b/src/main/java/org/caosdb/server/database/backend/implementation/MySQL/MySQLGetVirtualFSO.java
@@ -0,0 +1,42 @@
+package org.caosdb.server.database.backend.implementation.MySQL;
+
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import org.caosdb.server.database.DatabaseUtils;
+import org.caosdb.server.database.access.Access;
+import org.caosdb.server.database.backend.interfaces.GetVirtualFSOImpl;
+import org.caosdb.server.database.exceptions.TransactionException;
+import org.caosdb.server.database.proto.SparseEntity;
+import org.caosdb.server.filesystem.FSODescriptor;
+import org.caosdb.server.filesystem.VirtualFSODescriptorInterface;
+
+public class MySQLGetVirtualFSO extends MySQLTransaction implements GetVirtualFSOImpl {
+
+ public MySQLGetVirtualFSO(Access access) {
+ super(access);
+ }
+
+ public static final String STMT_GET_FSO =
+ "SELECT 'SHA-512' AS FileHashAlgo, file_id AS FileId, "
+ + /*parent_directory AS FileParentID,*/ " path AS FilePath, size AS FileSize, hex(hash) AS FileHash, checked_timestamp AS FileHashChecked, mimetype AS FileMimeType, file_storage_id AS FileStorageID, file_key AS FileKey FROM files WHERE file_storage_id = ? AND file_key = ?";
+
+ @Override
+ public VirtualFSODescriptorInterface execute(String fileStorageId, String key) {
+ try {
+ PreparedStatement stmt = prepareStatement(STMT_GET_FSO);
+ stmt.setString(1, fileStorageId);
+ stmt.setString(2, key);
+ ResultSet resultSet = stmt.executeQuery();
+ if (resultSet.next()) {
+ SparseEntity e = new SparseEntity();
+ e.id = resultSet.getInt("FileId");
+ DatabaseUtils.parseFSODescriptorFields(resultSet, e);
+ return new FSODescriptor(e);
+ }
+ } catch (SQLException | ConnectionException e) {
+ throw new TransactionException(e);
+ }
+ return null;
+ }
+}
diff --git a/src/main/java/org/caosdb/server/database/backend/implementation/MySQL/MySQLListFiles.java b/src/main/java/org/caosdb/server/database/backend/implementation/MySQL/MySQLListFiles.java
index dfeed05b36d4001ad46e24aa82dab2b968d14db0..504f992a8fc26cccea4c93eb6970c6e7655ab2d7 100644
--- a/src/main/java/org/caosdb/server/database/backend/implementation/MySQL/MySQLListFiles.java
+++ b/src/main/java/org/caosdb/server/database/backend/implementation/MySQL/MySQLListFiles.java
@@ -10,8 +10,9 @@ import org.caosdb.server.database.access.Access;
import org.caosdb.server.database.backend.interfaces.ListFilesImpl;
import org.caosdb.server.database.exceptions.TransactionException;
import org.caosdb.server.database.proto.SparseEntity;
-import org.caosdb.server.filesystem.FSODescriptorInterface;
+import org.caosdb.server.filesystem.FSODescriptor;
import org.caosdb.server.filesystem.Path;
+import org.caosdb.server.filesystem.VirtualFSODescriptorInterface;
public class MySQLListFiles extends MySQLTransaction implements ListFilesImpl {
@@ -20,7 +21,8 @@ public class MySQLListFiles extends MySQLTransaction implements ListFilesImpl {
}
public static final String LIST_FILES_STMT =
- "SELECT 'SHA-512' AS FileHashAlgo, file_id AS FileId, parent_directory AS FileParentID, path AS FilePath, size AS FileSize, hex(hash) AS FileHash, checked_timestamp AS FileHashChecked, mimetype AS FileMimeType, file_storage_id AS FileStorageID, file_key AS FileKey FROM files";
+ "SELECT 'SHA-512' AS FileHashAlgo, file_id AS FileId, "
+ + /*parent_directory AS FileParentID,*/ " path AS FilePath, size AS FileSize, hex(hash) AS FileHash, checked_timestamp AS FileHashChecked, mimetype AS FileMimeType, file_storage_id AS FileStorageID, file_key AS FileKey FROM files";
public static final String FILE_STORAGE_CLAUSE = " file_storage_id = ?";
public static final String PATH_LIKE_CLAUSE = " path LIKE ?";
public static final String LAST_CHECKED_BEFORE_CLAUSE = " checked_timestamp < ?";
@@ -29,18 +31,18 @@ public class MySQLListFiles extends MySQLTransaction implements ListFilesImpl {
public static final String LIMIT = " LIMIT ";
@Override
- public Iterable<FSODescriptorInterface> execute(
+ public List<VirtualFSODescriptorInterface> execute(
String fileStorageId, Path path, Long lastChecked, Integer limit) {
+ final List<VirtualFSODescriptorInterface> result = new LinkedList<>();
try {
PreparedStatement stmt = prepareStatement(fileStorageId, path, lastChecked, limit);
- final List<SparseEntity> result = new LinkedList<>();
try (ResultSet rs = stmt.executeQuery()) {
while (rs.next()) {
final SparseEntity entity = new SparseEntity();
entity.id = rs.getInt("FileId");
DatabaseUtils.parseFSODescriptorFields(rs, entity);
- result.add(entity);
+ result.add(new FSODescriptor(entity));
}
}
} catch (SQLException e) {
@@ -48,8 +50,9 @@ public class MySQLListFiles extends MySQLTransaction implements ListFilesImpl {
} catch (ConnectionException e) {
throw new TransactionException(e);
}
+ System.err.println("MySQLListFiles: " + result.size());
- return null;
+ return result;
}
private PreparedStatement prepareStatement(
diff --git a/src/main/java/org/caosdb/server/database/backend/implementation/MySQL/MySQLRetrieveAllUncheckedFiles.java b/src/main/java/org/caosdb/server/database/backend/implementation/MySQL/MySQLRetrieveAllUncheckedFiles.java
deleted file mode 100644
index 9fc50c821f59325f3ac320d1d3693b6bad3510bc..0000000000000000000000000000000000000000
--- a/src/main/java/org/caosdb/server/database/backend/implementation/MySQL/MySQLRetrieveAllUncheckedFiles.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * ** header v3.0
- * This file is a part of the CaosDB Project.
- *
- * Copyright (C) 2018 Research Group Biomedical Physics,
- * Max-Planck-Institute for Dynamics and Self-Organization Göttingen
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
- * ** end header
- */
-package org.caosdb.server.database.backend.implementation.MySQL;
-
-import java.sql.PreparedStatement;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.util.Iterator;
-import org.caosdb.server.database.DatabaseUtils;
-import org.caosdb.server.database.access.Access;
-import org.caosdb.server.database.backend.interfaces.RetrieveAllUncheckedFilesImpl;
-import org.caosdb.server.database.exceptions.TransactionException;
-import org.caosdb.server.database.proto.SparseEntity;
-
-/**
- * Implements {@link RetrieveAllUncheckedFilesImpl} for a MySQL/MariaDB back-end.
- *
- * @author Timm Fitschen <t.fitschen@indiscale.com>
- */
-public class MySQLRetrieveAllUncheckedFiles extends MySQLTransaction
- implements RetrieveAllUncheckedFilesImpl {
-
- private static final String STMT_RETRIEVE_ALL =
- "SELECT hash_algorithm AS FileHashAlgo, file_id AS FileId, " /*parent_directory AS FileParentId,*/
- + "path AS FilePath, size AS FileSize, hex(hash) AS FileHash, checked_timestamp AS FileHashChecked, mimetype AS FileMimeType, file_storage_id AS FileStorageId, file_key AS FileKey FROM files WHERE checked_timestamp<? AND path LIKE ? AND (mimetype IS NULL OR mimetype != 'inode/directory')";
-
- public MySQLRetrieveAllUncheckedFiles(final Access access) {
- super(access);
- }
-
- @Override
- public Iterator<SparseEntity> execute(final long ts, final String directory)
- throws TransactionException {
- try {
- final PreparedStatement stmt = getMySQLHelper().prepareStatement(STMT_RETRIEVE_ALL);
- stmt.setLong(1, ts);
- stmt.setString(2, directory + "%");
- final ResultSet rs = stmt.executeQuery();
- if (rs.next()) {
- final SparseEntity first = parseResultSet(rs);
- return new Iterator<SparseEntity>() {
-
- SparseEntity next = first;
-
- @Override
- public boolean hasNext() {
- return this.next != null;
- }
-
- @Override
- public SparseEntity next() {
- final SparseEntity ret = this.next;
- try {
- if (rs.next()) {
- this.next = parseResultSet(rs);
- } else {
- this.next = null;
- rs.close();
- }
- } catch (final SQLException e) {
- e.printStackTrace();
- }
- return ret;
- }
-
- @Override
- public void remove() {}
-
- @Override
- protected void finalize() throws Throwable {
- super.finalize();
- rs.close();
- }
- };
- }
-
- } catch (final SQLException e) {
- throw new TransactionException(e);
- } catch (final ConnectionException e) {
- throw new TransactionException(e);
- }
- return null;
- }
-
- private SparseEntity parseResultSet(final ResultSet rs) throws SQLException {
- final SparseEntity ret = new SparseEntity();
- ret.id = rs.getInt("FileId");
- DatabaseUtils.parseFSODescriptorFields(rs, ret);
- return ret;
- }
-}
diff --git a/src/main/java/org/caosdb/server/database/backend/implementation/MySQL/MySQLSetFileCheckedTimestamp.java b/src/main/java/org/caosdb/server/database/backend/implementation/MySQL/MySQLSetFileCheckedTimestamp.java
index 514ed5ea6548a43d00ae98ba6ebd183e65b85283..1e58fc9058bc2006ec8e782420add85b61aaffca 100644
--- a/src/main/java/org/caosdb/server/database/backend/implementation/MySQL/MySQLSetFileCheckedTimestamp.java
+++ b/src/main/java/org/caosdb/server/database/backend/implementation/MySQL/MySQLSetFileCheckedTimestamp.java
@@ -44,6 +44,7 @@ public class MySQLSetFileCheckedTimestamp extends MySQLTransaction
final PreparedStatement stmt = getMySQLHelper().prepareStatement(STMT_SET_TS);
stmt.setLong(1, ts);
stmt.setInt(2, id.toInteger());
+ stmt.execute();
} catch (final SQLException e) {
throw new TransactionException(e);
} catch (final ConnectionException e) {
diff --git a/src/main/java/org/caosdb/server/database/backend/interfaces/GetVirtualFSOImpl.java b/src/main/java/org/caosdb/server/database/backend/interfaces/GetVirtualFSOImpl.java
new file mode 100644
index 0000000000000000000000000000000000000000..1c86822d4f0c0e920ab68a01c236c628a3b678e5
--- /dev/null
+++ b/src/main/java/org/caosdb/server/database/backend/interfaces/GetVirtualFSOImpl.java
@@ -0,0 +1,8 @@
+package org.caosdb.server.database.backend.interfaces;
+
+import org.caosdb.server.filesystem.VirtualFSODescriptorInterface;
+
+public interface GetVirtualFSOImpl extends BackendTransactionImpl {
+
+ VirtualFSODescriptorInterface execute(String fileStorageId, String key);
+}
diff --git a/src/main/java/org/caosdb/server/database/backend/interfaces/ListFilesImpl.java b/src/main/java/org/caosdb/server/database/backend/interfaces/ListFilesImpl.java
index d77a282e5e65dad69ebcb7687ab34e25599dd36d..8ca82f9581897f26b293928d1e00c19f26fdf2cb 100644
--- a/src/main/java/org/caosdb/server/database/backend/interfaces/ListFilesImpl.java
+++ b/src/main/java/org/caosdb/server/database/backend/interfaces/ListFilesImpl.java
@@ -1,10 +1,11 @@
package org.caosdb.server.database.backend.interfaces;
-import org.caosdb.server.filesystem.FSODescriptorInterface;
+import java.util.List;
import org.caosdb.server.filesystem.Path;
+import org.caosdb.server.filesystem.VirtualFSODescriptorInterface;
public interface ListFilesImpl extends BackendTransactionImpl {
- Iterable<FSODescriptorInterface> execute(
+ List<VirtualFSODescriptorInterface> execute(
String fileStorageId, Path path, Long lastChecked, Integer limit);
}
diff --git a/src/main/java/org/caosdb/server/database/backend/interfaces/RetrieveAllUncheckedFilesImpl.java b/src/main/java/org/caosdb/server/database/backend/interfaces/RetrieveAllUncheckedFilesImpl.java
deleted file mode 100644
index dc247bf6b2975f96185245813647662e8e9217f3..0000000000000000000000000000000000000000
--- a/src/main/java/org/caosdb/server/database/backend/interfaces/RetrieveAllUncheckedFilesImpl.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * ** header v3.0
- * This file is a part of the CaosDB Project.
- *
- * Copyright (C) 2018 Research Group Biomedical Physics,
- * Max-Planck-Institute for Dynamics and Self-Organization Göttingen
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
- * ** end header
- */
-package org.caosdb.server.database.backend.interfaces;
-
-import java.util.Iterator;
-import org.caosdb.server.database.exceptions.TransactionException;
-import org.caosdb.server.database.proto.SparseEntity;
-
-/**
- * Retrieve an iterator which iterates over all File entities which need to be consistency-checked.
- *
- * @author Timm Fitschen (t.fitschen@indiscale.com)
- */
-public interface RetrieveAllUncheckedFilesImpl extends BackendTransactionImpl {
-
- /**
- * Return an iterator over all (non-directory) file entities which have not been
- * consistency-checked since the (unix) timestamp `ts` and which are located below the directory
- * `directory`.
- *
- * @param ts
- * @param directory
- * @return
- * @throws TransactionException
- */
- public Iterator<SparseEntity> execute(long ts, String directory) throws TransactionException;
-}
diff --git a/src/main/java/org/caosdb/server/database/backend/transaction/GetFileIterator.java b/src/main/java/org/caosdb/server/database/backend/transaction/GetFileIterator.java
index 4a150c428444b710aa25d80a18f8948d72abf509..47657eb79a621cddfc49b6927149c51aefdd227f 100644
--- a/src/main/java/org/caosdb/server/database/backend/transaction/GetFileIterator.java
+++ b/src/main/java/org/caosdb/server/database/backend/transaction/GetFileIterator.java
@@ -24,7 +24,10 @@ package org.caosdb.server.database.backend.transaction;
import java.util.Iterator;
import org.caosdb.server.database.BackendTransaction;
+import org.caosdb.server.database.backend.interfaces.ListFilesImpl;
import org.caosdb.server.database.exceptions.TransactionException;
+import org.caosdb.server.filesystem.Path;
+import org.caosdb.server.filesystem.VirtualFSODescriptorInterface;
/**
* Generate an Iterator over all file paths in a file storage (below a particular directory).
@@ -33,16 +36,47 @@ import org.caosdb.server.database.exceptions.TransactionException;
*/
public class GetFileIterator extends BackendTransaction {
- private Iterator<String> iterator;
+ public static class FSOIterator implements Iterator<VirtualFSODescriptorInterface> {
- public GetFileIterator(final String fileStorageId, final String directory) {}
+ private GetFileIterator getFileIterator;
+
+ public FSOIterator(GetFileIterator getFileIterator) {
+ this.getFileIterator = getFileIterator;
+ }
+
+ @Override
+ public boolean hasNext() {
+ if (this.getFileIterator.chunk == null || !this.getFileIterator.chunk.hasNext()) {
+ this.getFileIterator.execute();
+ }
+ return this.getFileIterator.chunk.hasNext();
+ }
+
+ @Override
+ public VirtualFSODescriptorInterface next() {
+ return this.getFileIterator.chunk.next();
+ }
+ }
+
+ private Iterator<VirtualFSODescriptorInterface> chunk = null;
+ private Iterator<VirtualFSODescriptorInterface> iterator = new FSOIterator(this);
+ private String fileStorageId;
+ private Path belowPath;
+ private Long lastCheckedTimestamp;
+
+ public GetFileIterator(String fileStorageId, Path belowPath, Long lastCheckedTimestamp) {
+ this.fileStorageId = fileStorageId;
+ this.belowPath = belowPath;
+ this.lastCheckedTimestamp = lastCheckedTimestamp;
+ }
@Override
protected void execute() throws TransactionException {
- // TODO
+ ListFilesImpl t = getImplementation(ListFilesImpl.class);
+ this.chunk = t.execute(fileStorageId, belowPath, lastCheckedTimestamp, 100).iterator();
}
- public Iterator<String> getIterator() {
+ public Iterator<VirtualFSODescriptorInterface> getIterator() {
return this.iterator;
}
}
diff --git a/src/main/java/org/caosdb/server/database/backend/transaction/GetVirtualFSO.java b/src/main/java/org/caosdb/server/database/backend/transaction/GetVirtualFSO.java
new file mode 100644
index 0000000000000000000000000000000000000000..6d3e082d58b2bc47296b531f7c91d65d421b0f42
--- /dev/null
+++ b/src/main/java/org/caosdb/server/database/backend/transaction/GetVirtualFSO.java
@@ -0,0 +1,27 @@
+package org.caosdb.server.database.backend.transaction;
+
+import org.caosdb.server.database.BackendTransaction;
+import org.caosdb.server.database.backend.interfaces.GetVirtualFSOImpl;
+import org.caosdb.server.filesystem.VirtualFSODescriptorInterface;
+
+public class GetVirtualFSO extends BackendTransaction {
+
+ private String fileStorageId;
+ private String key;
+ private VirtualFSODescriptorInterface fso;
+
+ public GetVirtualFSO(String fileStorageId, String key) {
+ this.fileStorageId = fileStorageId;
+ this.key = key;
+ }
+
+ @Override
+ protected void execute() {
+ GetVirtualFSOImpl t = getImplementation(GetVirtualFSOImpl.class);
+ fso = t.execute(fileStorageId, key);
+ }
+
+ public boolean isKnown() {
+ return fso != null;
+ }
+}
diff --git a/src/main/java/org/caosdb/server/database/backend/transaction/ListFilesTransaction.java b/src/main/java/org/caosdb/server/database/backend/transaction/ListFilesTransaction.java
index c26573d6c19bf8802b31cd6b0d64a51ceabf1472..59db67289b9fc45086315ef711689936f8350002 100644
--- a/src/main/java/org/caosdb/server/database/backend/transaction/ListFilesTransaction.java
+++ b/src/main/java/org/caosdb/server/database/backend/transaction/ListFilesTransaction.java
@@ -2,16 +2,17 @@ package org.caosdb.server.database.backend.transaction;
import org.caosdb.server.database.BackendTransaction;
import org.caosdb.server.database.backend.interfaces.ListFilesImpl;
-import org.caosdb.server.filesystem.FSODescriptorInterface;
import org.caosdb.server.filesystem.FileStorageInterface;
import org.caosdb.server.filesystem.Path;
+import org.caosdb.server.filesystem.VirtualFSODescriptorInterface;
+@Deprecated
public class ListFilesTransaction extends BackendTransaction {
private Path path;
private Long lastChecked;
private String fileStorageId;
- private Iterable<FSODescriptorInterface> files;
+ private Iterable<VirtualFSODescriptorInterface> files;
private Integer limit;
public ListFilesTransaction(
@@ -26,7 +27,7 @@ public class ListFilesTransaction extends BackendTransaction {
this.limit = limit;
}
- public Iterable<FSODescriptorInterface> getFiles() {
+ public Iterable<VirtualFSODescriptorInterface> getFiles() {
return files;
}
diff --git a/src/main/java/org/caosdb/server/database/backend/transaction/RetrieveAllUncheckedFiles.java b/src/main/java/org/caosdb/server/database/backend/transaction/RetrieveAllUncheckedFiles.java
deleted file mode 100644
index 655e47b8312c3a9772ae3e9055bc73494236a0ec..0000000000000000000000000000000000000000
--- a/src/main/java/org/caosdb/server/database/backend/transaction/RetrieveAllUncheckedFiles.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * ** header v3.0
- * This file is a part of the CaosDB Project.
- *
- * Copyright (C) 2018 Research Group Biomedical Physics,
- * Max-Planck-Institute for Dynamics and Self-Organization Göttingen
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
- * ** end header
- */
-package org.caosdb.server.database.backend.transaction;
-
-import java.util.Iterator;
-import org.caosdb.server.database.BackendTransaction;
-import org.caosdb.server.database.backend.interfaces.RetrieveAllUncheckedFilesImpl;
-import org.caosdb.server.database.exceptions.TransactionException;
-import org.caosdb.server.database.proto.SparseEntity;
-
-public class RetrieveAllUncheckedFiles extends BackendTransaction {
-
- private Iterator<SparseEntity> iterator;
- private final long ts;
- private final String location;
-
- public RetrieveAllUncheckedFiles(final long ts, final String location) {
- this.ts = ts;
- this.location = location;
- }
-
- @Override
- protected void execute() throws TransactionException {
- final RetrieveAllUncheckedFilesImpl t = getImplementation(RetrieveAllUncheckedFilesImpl.class);
- this.iterator = t.execute(this.ts, this.location);
- }
-
- public Iterator<SparseEntity> getIterator() {
- return this.iterator;
- }
-}
diff --git a/src/main/java/org/caosdb/server/filesystem/FSODescriptorInterface.java b/src/main/java/org/caosdb/server/filesystem/FSODescriptorInterface.java
index 5387dce0df7bbd1ad34de287027e52dadbbde6d5..34d0279fadc36d5ca5e330fa247a4edc5ff154c2 100644
--- a/src/main/java/org/caosdb/server/filesystem/FSODescriptorInterface.java
+++ b/src/main/java/org/caosdb/server/filesystem/FSODescriptorInterface.java
@@ -13,7 +13,7 @@ import org.caosdb.server.filesystem.FileSystem.ObjectType;
*
* @author Timm Fitschen (t.fitschen@indiscale.com)
*/
-public interface FSODescriptorInterface {
+public interface FSODescriptorInterface extends Comparable<FSODescriptorInterface> {
/**
* An identifier of the file. The key is guaranteed to be unique across the file storage where
@@ -55,4 +55,13 @@ public interface FSODescriptorInterface {
public abstract String getMimeType();
public abstract ObjectType getObjectType();
+
+ @Override
+ default int compareTo(FSODescriptorInterface other) {
+ int fileStorageCompared = other.getFileStorageId().compareTo(getFileStorageId());
+ if (fileStorageCompared == 0) {
+ return other.getKey().compareTo(getKey());
+ }
+ return fileStorageCompared;
+ }
}
diff --git a/src/main/java/org/caosdb/server/filesystem/FileStorageInterface.java b/src/main/java/org/caosdb/server/filesystem/FileStorageInterface.java
index f0b8be54c72e20eaf3b75164a94ce5d3a85bfce1..19420a8527576c35ba29dc8eba39f1484840b9c9 100644
--- a/src/main/java/org/caosdb/server/filesystem/FileStorageInterface.java
+++ b/src/main/java/org/caosdb/server/filesystem/FileStorageInterface.java
@@ -50,7 +50,5 @@ public interface FileStorageInterface {
/** Return true iff an object with this key exists. */
public abstract boolean exists(String key);
- /*
- * fileIterator(String keyPrefix, Long lastModifiedAfter,
- */
+ public abstract Iterable<? extends RealFSODescriptorInterface> list(String prefix);
}
diff --git a/src/main/java/org/caosdb/server/filesystem/FileSystem.java b/src/main/java/org/caosdb/server/filesystem/FileSystem.java
index 5212573e03f2a1eb74626b77bb08252a53dd7baf..e016d7952f96e0a43e5a8f878e41e5f1f44f9223 100644
--- a/src/main/java/org/caosdb/server/filesystem/FileSystem.java
+++ b/src/main/java/org/caosdb/server/filesystem/FileSystem.java
@@ -72,22 +72,6 @@ public class FileSystem implements FileSystemInterface {
return result;
}
- public static Hash getHash(FSODescriptorInterface fso, String algorithm) {
- FileStorageInterface fileStorage = getInstance().getFileStorage(fso);
- if (fileStorage.getCapabilities().getHash(algorithm)) {
- return fileStorage.getHash(fso, algorithm);
- }
- return null;
- }
-
- public static Long getSize(FSODescriptorInterface fso) {
- FileStorageInterface fileStorage = getInstance().getFileStorage(fso);
- if (fileStorage.getCapabilities().getSize) {
- return fileStorage.getSize(fso);
- }
- return null;
- }
-
// public static FSODescriptorInterface resolveLinkTarget(FSODescriptorInterface fso) {
// return resolveLinkTarget(fso.getFileStorageId(), fso.getKey());
// }
diff --git a/src/main/java/org/caosdb/server/filesystem/LocalFileStorage.java b/src/main/java/org/caosdb/server/filesystem/LocalFileStorage.java
index b2666d26e3b7f0fe0b29f45ecfbfa1fd9b0973c7..237b9428ce89d4c573793480853bcf41b2363b4c 100644
--- a/src/main/java/org/caosdb/server/filesystem/LocalFileStorage.java
+++ b/src/main/java/org/caosdb/server/filesystem/LocalFileStorage.java
@@ -8,6 +8,8 @@ import java.io.Reader;
import java.nio.file.Files;
import java.nio.file.LinkOption;
import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import org.apache.commons.io.FileUtils;
@@ -58,6 +60,12 @@ public abstract class LocalFileStorage implements FileStorageInterface {
this.key = key;
}
+ private LocalFSODescriptor(File file) {
+ this.key = null;
+ this.fileStorage = null;
+ this.file = file;
+ }
+
private ObjectType _getObjectType() {
if (type == null) {
type = fileStorage.getObjectType(key);
@@ -133,7 +141,35 @@ public abstract class LocalFileStorage implements FileStorageInterface {
@Override
public Boolean exists() {
- return fileStorage.exists(key);
+ return getFile().exists();
+ }
+
+ @Override
+ public RealFSODescriptorInterface resolveLinkTarget() {
+ try {
+ Path resolvedPath = getFile().toPath().toRealPath();
+ if (fileStorage.belongsToUs(resolvedPath)) {
+ return new LocalFSODescriptor(fileStorage, fileStorage.getKey(resolvedPath));
+ }
+ for (String otherFileStorageId : FileSystem.getInstance().getFileStorages()) {
+ if (!otherFileStorageId.equals(fileStorage.getId())) {
+ FileStorageInterface otherFileStorage =
+ FileSystem.getInstance().getFileStorage(otherFileStorageId);
+ if (otherFileStorage instanceof LocalFileStorage) {
+ if (((LocalFileStorage) otherFileStorage).belongsToUs(resolvedPath)) {
+ // this file belongs to another file storage.
+ return new LocalFSODescriptor(
+ (LocalFileStorage) otherFileStorage,
+ ((LocalFileStorage) otherFileStorage).getKey(resolvedPath));
+ }
+ }
+ }
+ }
+ // We don't know this file.
+ return new LocalFSODescriptor(resolvedPath.toFile());
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
}
}
@@ -153,6 +189,14 @@ public abstract class LocalFileStorage implements FileStorageInterface {
this.root = root.toAbsolutePath();
}
+ private boolean belongsToUs(Path path) {
+ return path.startsWith(root);
+ }
+
+ private String getKey(Path path) {
+ return root.relativize(path).toString();
+ }
+
public ObjectType getObjectType(String key) {
File file = getFile(key);
return getObjectType(file);
@@ -675,66 +719,67 @@ public abstract class LocalFileStorage implements FileStorageInterface {
}
}
- // @Override
- // public Iterator<String> execute(final String storage, final String location)
- // throws TransactionException {
- // File base;
- // try {
- // // TODO: move everything to GetFileIterator and refactor. Needs a listFiles
- // // Transaction
- // base = getFile(storage, location).getFile();
- // } catch (final Message e) {
- // throw new TransactionException(e);
- // }
- // return new FileNameIterator(base, location);
- // }
- //
- // public static class FileNameIterator implements Iterator<String> {
- //
- // private final File base;
- // private File[] files = null;
- // private FileNameIterator subfiles = null;
- // private int i = 0;
- // private final String rootPath;
- //
- // public FileNameIterator(final File base, final String rootPath) {
- // this.rootPath = rootPath.length() == 0 || rootPath.endsWith("/") ? rootPath : rootPath +
- // "/";
- // this.base = base;
- // this.files = this.base.listFiles();
- // Arrays.sort(this.files);
- // }
- //
- // @Override
- // public boolean hasNext() {
- // return this.files.length > this.i;
- // }
- //
- // @Override
- // public String next() {
- // if (this.subfiles != null) {
- // if (this.subfiles.hasNext()) {
- // return this.subfiles.next();
- // } else {
- // this.subfiles = null;
- // }
- // }
- // final File ret = this.files[this.i++];
- // if (ret.isDirectory()) {
- // this.subfiles = new FileNameIterator(ret, this.rootPath + ret.getName() + "/");
- // if (this.subfiles.hasNext()) {
- // return next();
- // } else {
- // return this.rootPath + ret.getName() + "/";
- // }
- // }
- // return this.rootPath + ret.getName();
- // }
- //
- // @Override
- // public void remove() {
- // throw new UnsupportedOperationException();
- // }
- // }
- // }
+ public static class LocalFSOIterator implements Iterator<LocalFSODescriptor> {
+
+ private final File base;
+ private File[] files = null;
+ private LocalFSOIterator subfiles = null;
+ private int i = 0;
+ private final org.caosdb.server.filesystem.Path rootPath;
+ private LocalFileStorage fileStorage;
+
+ public LocalFSOIterator(
+ LocalFileStorage fileStorage, File base, final org.caosdb.server.filesystem.Path rootPath) {
+ this.fileStorage = fileStorage;
+ this.rootPath = rootPath;
+ this.base = base;
+ this.files = this.base.listFiles();
+ Arrays.sort(this.files);
+ }
+
+ @Override
+ public boolean hasNext() {
+ return this.files.length > this.i;
+ }
+
+ @Override
+ public LocalFSODescriptor next() {
+ if (this.subfiles != null) {
+ if (this.subfiles.hasNext()) {
+ return this.subfiles.next();
+ } else {
+ this.subfiles = null;
+ }
+ }
+ final File ret = this.files[this.i++];
+ if (ret.isDirectory()) {
+ this.subfiles = new LocalFSOIterator(fileStorage, ret, this.rootPath.append(ret.getName()));
+ if (this.subfiles.hasNext()) {
+ return next();
+ } else {
+ return new LocalFSODescriptor(fileStorage, rootPath.append(ret.getName()).toString());
+ }
+ }
+ return new LocalFSODescriptor(fileStorage, rootPath.append(ret.getName()).toString());
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ @Override
+ public Iterable<LocalFSODescriptor> list(String prefix) {
+ final LocalFileStorage fileStorage = this;
+ final File base = prefix == null ? this.root.toFile() : this.root.resolve(prefix).toFile();
+ return new Iterable<LocalFSODescriptor>() {
+
+ @Override
+ public LocalFSOIterator iterator() {
+ return new LocalFSOIterator(
+ fileStorage, base, new org.caosdb.server.filesystem.Path(prefix == null ? "" : prefix));
+ }
+ };
+ }
}
diff --git a/src/main/java/org/caosdb/server/filesystem/RealFSODescriptorInterface.java b/src/main/java/org/caosdb/server/filesystem/RealFSODescriptorInterface.java
index 2d285365718634fe45764649a556b9d3b4f6ed07..aed0f89c052c0587c48826dadd1e5b9f6759844b 100644
--- a/src/main/java/org/caosdb/server/filesystem/RealFSODescriptorInterface.java
+++ b/src/main/java/org/caosdb/server/filesystem/RealFSODescriptorInterface.java
@@ -1,6 +1,7 @@
package org.caosdb.server.filesystem;
import java.io.File;
+import java.util.List;
/**
* Represents an object in a file storage (i.e. a POSIX file system, an S3 bucket server, or similar
@@ -40,4 +41,9 @@ public interface RealFSODescriptorInterface extends FSODescriptorInterface {
* null if the existence cannot be determined.
*/
public abstract Boolean exists();
+
+ public abstract RealFSODescriptorInterface resolveLinkTarget();
+
+ @Override
+ public abstract List<? extends RealFSODescriptorInterface> listChildren();
}
diff --git a/src/main/java/org/caosdb/server/filesystem/TemporaryFSODescriptor.java b/src/main/java/org/caosdb/server/filesystem/TemporaryFSODescriptor.java
index 7a7a398073c1762dc01f07ce6866a71f19237327..4c6d16963394beb97c3cba216229c4d544635ab8 100644
--- a/src/main/java/org/caosdb/server/filesystem/TemporaryFSODescriptor.java
+++ b/src/main/java/org/caosdb/server/filesystem/TemporaryFSODescriptor.java
@@ -64,7 +64,7 @@ public class TemporaryFSODescriptor implements RealFSODescriptorInterface, Undoa
}
@Override
- public List<? extends FSODescriptorInterface> listChildren() {
+ public List<? extends RealFSODescriptorInterface> listChildren() {
return null;
}
@@ -91,4 +91,10 @@ public class TemporaryFSODescriptor implements RealFSODescriptorInterface, Undoa
public String getTmpIdentifier() {
return fileId;
}
+
+ @Override
+ public RealFSODescriptorInterface resolveLinkTarget() {
+ // TODO Auto-generated method stub
+ return null;
+ }
}
diff --git a/src/main/java/org/caosdb/server/filesystem/consistency/AbstractConsistencyEvent.java b/src/main/java/org/caosdb/server/filesystem/consistency/AbstractConsistencyEvent.java
deleted file mode 100644
index fabedaf7ffdf216a301e6e751a829ea620e2b687..0000000000000000000000000000000000000000
--- a/src/main/java/org/caosdb/server/filesystem/consistency/AbstractConsistencyEvent.java
+++ /dev/null
@@ -1,115 +0,0 @@
-package org.caosdb.server.filesystem.consistency;
-
-import java.util.UUID;
-import org.caosdb.server.filesystem.FSODescriptorInterface;
-import org.caosdb.server.filesystem.FileStorageInterface;
-import org.caosdb.server.filesystem.FileSystem;
-
-public class AbstractConsistencyEvent implements ConsistencyEvent {
-
- private String fileStorageId;
- private String key;
- private String type;
- private String uuid = null;
- private Long timestamp;
-
- AbstractConsistencyEvent(String type, String fileStorageId, String key) {
- this.timestamp = System.currentTimeMillis();
- this.type = type;
- // this.parent = parent;
- this.fileStorageId = fileStorageId;
- this.key = key;
- }
-
- @Override
- public String getFileStorageId() {
- return fileStorageId;
- }
-
- @Override
- public String getKey() {
- return key;
- }
-
- @Override
- public String getEventType() {
- return type;
- }
- //
- // @Override
- // public ConsistencyEvent getParent() {
- // if (parent == null && type.contains(":")) {
- // String[] components = type.split(":");
- // components = Arrays.copyOfRange(components, 0, components.length - 2);
- // parent = new AbstractConsistencyEvent(String.join(":", components), fileStorageId, key);
- // }
- // return parent;
- // }
-
- @Override
- public boolean isSubtypeOf(String eventType) {
- if (!eventType.endsWith(":")) {
- return this.type.startsWith(eventType + ":");
- }
- return this.type.startsWith(eventType);
- }
-
- public static final ConsistencyEvent missingCapability(
- String capability, FileStorageInterface fileStorage) {
- return new AbstractConsistencyEvent("NEED_CAPABILITY:" + capability, fileStorage.getId(), null);
- }
-
- public static ConsistencyEvent unknownFile(FSODescriptorInterface next) {
- return new AbstractConsistencyEvent("UNKNOWN", next.getFileStorageId(), next.getKey());
- }
-
- public static ConsistencyEvent missingFile(FSODescriptorInterface next) {
- return new AbstractConsistencyEvent("MISSING", next.getFileStorageId(), next.getKey());
- }
-
- public static ConsistencyEvent wrongObjectType(
- FSODescriptorInterface fso, FileSystem.ObjectType type) {
- return new AbstractConsistencyEvent(
- "CHANGED:TYPE:TO_" + type.toString(), fso.getFileStorageId(), fso.getKey());
- }
-
- public static ConsistencyEvent brokenLink(FSODescriptorInterface fso) {
- return new AbstractConsistencyEvent("LINK:BROKEN", fso.getFileStorageId(), fso.getKey());
- }
-
- public static ConsistencyEvent linkToOtherFileStorage(FSODescriptorInterface fso) {
- return new AbstractConsistencyEvent(
- "LINK:TARGET:IN_OTHER_FILE_STORAGE", fso.getFileStorageId(), fso.getKey());
- }
-
- public static ConsistencyEvent linkToAlienTargetFromOuterSpace(FSODescriptorInterface fso) {
- return new AbstractConsistencyEvent(
- "LINK:TARGET:NOT_IN_FILE_STORAGE", fso.getFileStorageId(), fso.getKey());
- }
-
- public static ConsistencyEvent sizeMismatch(FSODescriptorInterface fso) {
- return new AbstractConsistencyEvent("CHANGED:SIZE", fso.getFileStorageId(), fso.getKey());
- }
-
- public static ConsistencyEvent hashMismatch(FSODescriptorInterface fso) {
- return new AbstractConsistencyEvent("CHANGED:HASH", fso.getFileStorageId(), fso.getKey());
- }
-
- public static ConsistencyEvent linkTargetChanged(FSODescriptorInterface fso) {
- return new AbstractConsistencyEvent(
- "CHANGED:LINK:TARGET", fso.getFileStorageId(), fso.getKey());
- }
-
- @Override
- public String getUUID() {
- if (uuid == null) {
- uuid = UUID.randomUUID().toString();
- }
- return uuid;
- }
-
- @Override
- public Long getTimestamp() {
- return timestamp;
- }
-}
diff --git a/src/main/java/org/caosdb/server/filesystem/consistency/ConsistencyCheck.java b/src/main/java/org/caosdb/server/filesystem/consistency/ConsistencyCheck.java
index 9eabd32b290afbf8ab17d33a5e2f51d8026c6379..8b52620551047a68d12b42763f189bd468a97112 100644
--- a/src/main/java/org/caosdb/server/filesystem/consistency/ConsistencyCheck.java
+++ b/src/main/java/org/caosdb/server/filesystem/consistency/ConsistencyCheck.java
@@ -1,30 +1,42 @@
package org.caosdb.server.filesystem.consistency;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
+import java.util.Objects;
import org.caosdb.datetime.UTCDateTime;
+import org.caosdb.server.database.DatabaseAccessManager;
import org.caosdb.server.database.access.Access;
-import org.caosdb.server.database.backend.transaction.ListFilesTransaction;
+import org.caosdb.server.database.backend.transaction.GetFileIterator;
+import org.caosdb.server.database.backend.transaction.GetVirtualFSO;
+import org.caosdb.server.database.backend.transaction.SetFileCheckedTimestamp;
+import org.caosdb.server.entity.EntityID;
import org.caosdb.server.filesystem.FSODescriptorInterface;
import org.caosdb.server.filesystem.FileStorageCapabilities;
import org.caosdb.server.filesystem.FileStorageInterface;
import org.caosdb.server.filesystem.FileSystem;
import org.caosdb.server.filesystem.FileSystem.ObjectType;
+import org.caosdb.server.filesystem.Hash;
import org.caosdb.server.filesystem.Path;
+import org.caosdb.server.filesystem.RealFSODescriptorInterface;
+import org.caosdb.server.filesystem.VirtualFSODescriptorInterface;
import org.caosdb.server.transaction.TransactionInterface;
-abstract class ConsistencyCheckStrategy implements TransactionInterface {
+abstract class ConsistencyCheckStrategy implements TransactionInterface, Runnable {
- private List<ConsistencyEvent> events = new LinkedList<>();
+ private List<ConsistencyEventInterface> events = new LinkedList<>();
private Access access;
+ protected Exception exception;
public abstract void execute();
- public void emit(ConsistencyEvent event) {
+ public void emit(ConsistencyEventInterface event) {
events.add(event);
}
- public List<ConsistencyEvent> getEvents() {
+ public List<ConsistencyEventInterface> getEvents() {
return events;
}
@@ -35,134 +47,210 @@ abstract class ConsistencyCheckStrategy implements TransactionInterface {
Access getAccess() {
return access;
}
+
+ @Override
+ public void run() {
+ execute();
+ }
+
+ public Exception getException() {
+ return exception;
+ }
}
class RunThroughInternalFileSystem extends ConsistencyCheckStrategy {
- private ListFilesTransaction listFilesTransaction;
+ private Long lastChecked;
+ private Path path;
+ private String fileStorageId;
public RunThroughInternalFileSystem(
- Long lastChecked, Path path, FileStorageInterface fileStorage, Integer limit) {
- this.listFilesTransaction = new ListFilesTransaction(fileStorage, path, lastChecked, limit);
+ Long lastChecked, Path path, FileStorageInterface fileStorage) {
+ this.lastChecked = lastChecked == null ? System.currentTimeMillis() : lastChecked;
+ this.path = path;
+ this.fileStorageId = fileStorage == null ? null : fileStorage.getId();
}
- public RunThroughInternalFileSystem(Long since, Integer limit) {
- this(since, null, null, limit);
+ @Override
+ public void execute() {
+ try {
+ this.setAccess(DatabaseAccessManager.getInstance().acquireReadAccess(this));
+
+ // test all files in file system.
+ final GetFileIterator it = new GetFileIterator(fileStorageId, path, lastChecked);
+ Iterator<VirtualFSODescriptorInterface> iterator =
+ execute(it, this.getAccess()).getIterator();
+
+ while (iterator != null && iterator.hasNext()) {
+ long timestamp = System.currentTimeMillis();
+ System.err.println("here");
+
+ if (DatabaseAccessManager.whoHasReservedWriteAccess() != null) {
+ // there is a thread waiting to write. pause this one and
+ // try to acquire a new read access which will be granted when
+ // the write thread is done.
+ this.getAccess().release();
+ this.setAccess(DatabaseAccessManager.getInstance().acquireReadAccess(this));
+ iterator = execute(it, this.getAccess()).getIterator();
+ System.err.println("here2");
+ }
+
+ final VirtualFSODescriptorInterface howItShouldBe = iterator.next();
+ System.err.println("check: " + howItShouldBe.toString());
+
+ check(howItShouldBe);
+
+ execute(
+ new SetFileCheckedTimestamp(howItShouldBe.getEntityId(), timestamp), this.getAccess());
+ getAccess().commit();
+ }
+ } catch (final Exception e) {
+ this.exception = e;
+ } finally {
+ this.getAccess().release();
+ }
}
- public RunThroughInternalFileSystem(Path path, Integer limit) {
- this(null, path, null, limit);
- }
+ private void check(VirtualFSODescriptorInterface howItShouldBe) {
+ RealFSODescriptorInterface howItActuallyIs = resolve(howItShouldBe);
+ Boolean exists = howItActuallyIs.exists();
+ if (exists != null && !exists) {
+ emit(ConsistencyEvent.missingFile(howItActuallyIs));
+ return;
+ }
+
+ if (howItShouldBe.getObjectType() != howItActuallyIs.getObjectType()) {
+ emit(ConsistencyEvent.wrongObjectType(howItShouldBe, howItActuallyIs.getObjectType()));
+ return;
+ }
- public RunThroughInternalFileSystem(FileStorageInterface fileStorage, Integer limit) {
- this(null, null, fileStorage, limit);
+ if (howItActuallyIs.getObjectType() == ObjectType.LINK) {
+ checkLinkIntact(howItShouldBe, howItActuallyIs);
+ } else if (howItActuallyIs.getObjectType() == ObjectType.FILE) {
+ checkSize(howItShouldBe, howItActuallyIs);
+ checkHash(howItShouldBe, howItActuallyIs);
+ } else if (howItActuallyIs.getObjectType() == ObjectType.DIRECTORY) {
+ checkDirectoryContent(howItShouldBe, howItActuallyIs);
+ }
}
- @Override
- public void execute() {
- for (FSODescriptorInterface howItShouldBe : listFiles()) {
- FSODescriptorInterface howItActuallyIs = resolve(howItShouldBe);
+ private void checkDirectoryContent(
+ VirtualFSODescriptorInterface howItShouldBe, RealFSODescriptorInterface howItActuallyIs) {
+ List<? extends VirtualFSODescriptorInterface> childrenHowItShouldBe =
+ howItShouldBe.listChildren();
+ Collections.sort(childrenHowItShouldBe);
+ List<? extends RealFSODescriptorInterface> childrenHowItActuallyIs =
+ howItActuallyIs.listChildren();
+ Collections.sort(childrenHowItActuallyIs);
+ boolean mismatch = false;
+ if (childrenHowItActuallyIs.size() == childrenHowItShouldBe.size()) {
+ Iterator<? extends RealFSODescriptorInterface> iterator1 = childrenHowItActuallyIs.iterator();
+ Iterator<? extends VirtualFSODescriptorInterface> iterator2 =
+ childrenHowItShouldBe.iterator();
+ while (iterator1.hasNext()) {
+ if (iterator1.next().compareTo(iterator2.next()) != 0) {
+ // damn, found a mismatching child
+ mismatch = true;
+ break;
+ }
+ }
+ if (!mismatch) {
+ return;
+ }
+ }
- if (howItActuallyIs == null) {
- continue;
+ // we already know, that there are problems. We just try and figure out the exact nature of the
+ // problems.
+ childrenHowItActuallyIs = new ArrayList<>(childrenHowItActuallyIs);
+ childrenHowItShouldBe = new ArrayList<>(childrenHowItShouldBe);
+ Iterator<? extends VirtualFSODescriptorInterface> iChildrenHowItShouldBe =
+ childrenHowItShouldBe.iterator();
+ while (iChildrenHowItShouldBe.hasNext()) {
+ Iterator<? extends RealFSODescriptorInterface> iChildrenHowItActuallyIs =
+ childrenHowItActuallyIs.iterator();
+ VirtualFSODescriptorInterface childHowItShouldBe = iChildrenHowItShouldBe.next();
+ while (iChildrenHowItActuallyIs.hasNext()) {
+ RealFSODescriptorInterface childHowItActuallyIs = iChildrenHowItActuallyIs.next();
+ if (childHowItActuallyIs.compareTo(childHowItShouldBe) == 0) {
+ // we have a match. remove from both lists
+ iChildrenHowItActuallyIs.remove();
+ iChildrenHowItShouldBe.remove();
+ }
}
+ }
- // TODO check permissions of the server:
- // read permissions if read capability
- // write permissions if any write capability
- // execute permissions for traversal (directory capability)
+ // everything that is left in the lists is either...
+ for (RealFSODescriptorInterface unknownObject : childrenHowItActuallyIs) {
+ // ... an unknown object
+ emit(ConsistencyEvent.unknownChild(unknownObject));
+ }
+ for (VirtualFSODescriptorInterface missingObject : childrenHowItShouldBe) {
+ // ... or a missing object
+ emit(ConsistencyEvent.missingChild(missingObject));
+ }
+ }
- if (howItShouldBe.getObjectType() != howItActuallyIs.getObjectType()) {
- emit(
- AbstractConsistencyEvent.wrongObjectType(
- howItShouldBe, howItActuallyIs.getObjectType()));
- }
+ private void checkHash(
+ FSODescriptorInterface howItShouldBe, FSODescriptorInterface howItActuallyIs) {
+ Hash hash1 = howItActuallyIs.getHash();
+ Hash hash2 = howItShouldBe.getHash();
+ if (hash1 != null && hash2 != null && !hash1.equals(hash2)) {
+ emit(ConsistencyEvent.hashMismatch(howItActuallyIs));
+ }
+ }
- if (howItActuallyIs.getObjectType() == ObjectType.LINK) {
- // checkLinkIntact(howItShouldBe, howItActuallyIs);
- continue;
- }
+ private void checkSize(
+ FSODescriptorInterface howItShouldBe, FSODescriptorInterface howItActuallyIs) {
+ Long size = howItActuallyIs.getSize();
+ if (size != null && !size.equals(howItShouldBe.getSize())) {
+ // size == null means we don't know.
+ emit(ConsistencyEvent.sizeMismatch(howItActuallyIs));
+ }
+ }
+
+ private void checkLinkIntact(
+ VirtualFSODescriptorInterface howItShouldBe, RealFSODescriptorInterface howItActuallyIs) {
+ EntityID linkTargetId = howItShouldBe.getLinkTarget();
- // TODO check owner and group
+ RealFSODescriptorInterface linkTargetActuallyIs = howItActuallyIs.resolveLinkTarget();
+ Boolean exists = linkTargetActuallyIs.exists();
+ if (exists != null && !exists) {
+ // if link is broken (target is not existing/reachable from the host file system of the server
+ // - in case of network file servers, it might be intact from a different host with different
+ // mounts)
+ emit(ConsistencyEvent.brokenLink(howItActuallyIs));
+ }
+
+ FSODescriptorInterface linkTargetShouldBe = retrieveFSODescriptor(linkTargetId);
+ if (!(Objects.equals(
+ linkTargetShouldBe.getFileStorageId(), linkTargetActuallyIs.getFileStorageId())
+ && Objects.equals(linkTargetShouldBe.getKey(), linkTargetActuallyIs.getKey()))) {
+ emit(ConsistencyEvent.linkTargetChanged(howItShouldBe));
+ }
- // checkSize(howItShouldBe, howItActuallyIs);
- //
- // checkHash(howItShouldBe, howItActuallyIs);
+ if (linkTargetActuallyIs.getFileStorageId() == null) {
+ // target does not belong to the CaosDB file system
+ emit(ConsistencyEvent.linkToAlienTargetFromOuterSpace(howItActuallyIs));
+ } else if (linkTargetActuallyIs
+ .getFileStorageId()
+ .equals(linkTargetShouldBe.getFileStorageId())) {
+ // link crosses file storage boundaries
+ emit(ConsistencyEvent.linkToOtherFileStorage(howItActuallyIs));
}
}
- private Iterable<FSODescriptorInterface> listFiles() {
- return execute(this.listFilesTransaction, getAccess()).getFiles();
+ private FSODescriptorInterface retrieveFSODescriptor(EntityID linkTargetId) {
+ // TODO Auto-generated method stub
+ return null;
}
- // private void checkHash(
- // FSODescriptorInterface howItShouldBe, FSODescriptorInterface howItActuallyIs) {
- // if (!howItShouldBe.hasHash()) {
- // return;
- // }
- //
- // String algorithm = howItShouldBe.getHash().getAlgorithm();
- // Hash hash = FileSystem.getHash(howItActuallyIs, algorithm);
- // if (hash != null && !hash.equals(howItShouldBe.getHash())) {
- // emit(AbstractConsistencyEvent.hashMismatch(howItActuallyIs));
- // }
- // }
- //
- // private void checkSize(
- // FSODescriptorInterface howItShouldBe, FSODescriptorInterface howItActuallyIs) {
- // if (!howItShouldBe.hasSize()) {
- // return;
- // }
- //
- // Long size = FileSystem.getSize(howItActuallyIs);
- // if (size != null && !size.equals(howItShouldBe.getSize())) {
- // emit(AbstractConsistencyEvent.sizeMismatch(howItActuallyIs));
- // }
- // }
- //
- // private void checkLinkIntact(
- // FSODescriptorInterface howItShouldBe, FSODescriptorInterface howItActuallyIs) {
- // EntityID linkTargetId = howItShouldBe.getLinkTarget();
- //
- // FSODescriptorInterface resolveLinkTarget = FileSystem.resolveLinkTarget(howItActuallyIs);
- // if (resolveLinkTarget.getFile() == null) {
- // // if link is broken (target is not existing/reachable from the host file system of the
- // server
- // // - in case of network file servers, it might be intact from a different host with
- // different
- // // mounts)
- // emit(AbstractConsistencyEvent.brokenLink(howItActuallyIs));
- // }
- //
- // if (resolveLinkTarget.getFileStorageId() == null) {
- // if (FileSystem.resolveKey(resolveLinkTarget)) {
- // // link crosses file storage boundaries
- // emit(AbstractConsistencyEvent.linkToOtherFileStorage(howItActuallyIs));
- // } else {
- // // target does not belong to the CaosDB file system
- // emit(AbstractConsistencyEvent.linkToAlienTargetFromOuterSpace(howItActuallyIs));
- // }
- // }
- //
- // FSODescriptorInterface linkTargetShouldBe = retrieveFSODescriptor(linkTargetId);
- // if (!(Objects.equal(linkTargetShouldBe.getFileStorageId(),
- // resolveLinkTarget.getFileStorageId())
- // && Objects.equal(linkTargetShouldBe.getKey(), resolveLinkTarget.getKey()))) {
- // emit(AbstractConsistencyEvent.linkTargetChanged(howItShouldBe));
- // }
- // }
- //
- // private FSODescriptorInterface retrieveFSODescriptor(EntityID linkTargetId) {
- // // TODO Auto-generated method stub
- // return null;
- // }
-
- private FSODescriptorInterface resolve(FSODescriptorInterface fso) {
- FSODescriptorInterface result = null;
+ private RealFSODescriptorInterface resolve(FSODescriptorInterface fso) {
+ RealFSODescriptorInterface result = null;
result = FileSystem.getInstance().resolve(fso.getFileStorageId(), fso.getKey());
Boolean exists = FileSystem.exists(fso);
if (exists != null && !exists) {
- emit(AbstractConsistencyEvent.missingFile(fso));
+ // exists == null means we cannot tell.
+ emit(ConsistencyEvent.missingFile(fso));
}
return result;
}
@@ -204,47 +292,59 @@ class FindUnknownInFileStorage extends ConsistencyCheckStrategy {
@Override
public void execute() {
- if (!hasNecessaryCapabilities()) {
+ try {
+ if (fileStorage != null) {
+ check(fileStorage);
+ return;
+ }
+
+ // else check all
+ for (String fileStorageId : FileSystem.getInstance().getFileStorages()) {
+ FileStorageInterface fileStorage = FileSystem.getInstance().getFileStorage(fileStorageId);
+ check(fileStorage);
+ }
+ } catch (Exception e) {
+ this.exception = e;
+ }
+ }
+
+ private void check(FileStorageInterface fileStorage) throws InterruptedException {
+ if (!hasNecessaryCapabilities(fileStorage)) {
return;
}
- FSODescriptorInterface next = getNextFSOToBeChecked();
- while (next != null) {
- // TODO check if next is known in the internal file system
+ Iterator<? extends RealFSODescriptorInterface> iterator =
+ fileStorage.list(directoryKey).iterator();
+ while (iterator.hasNext()) {
+ RealFSODescriptorInterface next = iterator.next();
if (!isKnown(next)) {
- emit(AbstractConsistencyEvent.unknownFile(next));
+ emit(ConsistencyEvent.unknownFile(next));
}
-
- next = getNextFSOToBeChecked();
}
}
- private boolean isKnown(FSODescriptorInterface next) {
- // TODO Auto-generated method stub
- return false;
+ private boolean isKnown(RealFSODescriptorInterface next) throws InterruptedException {
+ this.setAccess(DatabaseAccessManager.getInstance().acquireReadAccess(this));
+ try {
+ return execute(new GetVirtualFSO(next.getFileStorageId(), next.getKey()), getAccess())
+ .isKnown();
+ } finally {
+ this.getAccess().release();
+ }
}
- private boolean hasNecessaryCapabilities() {
+ private boolean hasNecessaryCapabilities(FileStorageInterface fileStorage) {
if (!fileStorage.getCapabilities().list) {
- emit(AbstractConsistencyEvent.missingCapability("LIST", fileStorage));
+ emit(ConsistencyEvent.missingCapability("LIST", fileStorage));
return false;
}
if (directoryKey != null && !fileStorage.getCapabilities().directory) {
- emit(AbstractConsistencyEvent.missingCapability("DIRECTORY", fileStorage));
+ emit(ConsistencyEvent.missingCapability("DIRECTORY", fileStorage));
return false;
}
return true;
}
- /*
- * Get next FSO which is to be checked (only files which haven't been checked since 'since' are being returned)
- *
- * Returns null if no next FSO is there which needs to be checked.
- */
- private FSODescriptorInterface getNextFSOToBeChecked() {
- return null;
- }
-
@Override
public UTCDateTime getTimestamp() {
// TODO Auto-generated method stub
@@ -252,22 +352,37 @@ class FindUnknownInFileStorage extends ConsistencyCheckStrategy {
}
}
-public class ConsistencyCheck implements Runnable {
+public class ConsistencyCheck extends Thread {
- private ConsistencyCheckStrategy strategy;
- private Access access = null;
+ private Path belowPath;
+ private Exception exception;
+ private List<ConsistencyEventInterface> events;
- public ConsistencyCheck(ConsistencyCheckStrategy strategy) {
- this.strategy = strategy;
+ public ConsistencyCheck(Path belowPath) {
+ this.belowPath = belowPath;
}
@Override
public void run() {
- // TODO handle exceptions
+ RunThroughInternalFileSystem check1 = new RunThroughInternalFileSystem(null, belowPath, null);
+ check1.run();
+ exception = check1.getException();
+ events = check1.getEvents();
+ if (exception != null) {
+ return;
+ }
- strategy.setAccess(access);
- strategy.execute();
+ FindUnknownInFileStorage check2 = new FindUnknownInFileStorage(null, null);
+ check2.run();
+ exception = check2.getException();
+ events.addAll(check2.getEvents());
+ }
- // TODO collect events
+ public Exception getException() {
+ return exception;
+ }
+
+ public List<ConsistencyEventInterface> getEvents() {
+ return events;
}
}
diff --git a/src/main/java/org/caosdb/server/filesystem/consistency/ConsistencyConfig.java b/src/main/java/org/caosdb/server/filesystem/consistency/ConsistencyConfig.java
index b176ddbbdf49a3162d744823107fb43c28d9203a..d4146fde4879edb111811410a4f2bc424b6affb9 100644
--- a/src/main/java/org/caosdb/server/filesystem/consistency/ConsistencyConfig.java
+++ b/src/main/java/org/caosdb/server/filesystem/consistency/ConsistencyConfig.java
@@ -15,6 +15,7 @@ public class ConsistencyConfig {
public static class EventsMapping<T> {
private Map<String, T> mapping = new HashMap<>();
+ protected T defaul;
@JsonAnySetter
public void set(String key, T val) {
@@ -22,22 +23,27 @@ public class ConsistencyConfig {
}
public T get(String event) {
- return mapping.get(event);
+ return mapping.getOrDefault(event, defaul);
}
}
/**
- * The EventsLogConfig maps {@link ConsistencyEvent}s to {@link LogLevel}s.
+ * The EventsLogConfig maps {@link ConsistencyEventInterface}s to {@link LogLevel}s.
*
* <p>When the consistency check of a file storage fires an event this mapping determines the
* severity of it.
*
* @author tf
*/
- public static class EventsLogConfig extends EventsMapping<LogLevel> {}
+ public static class EventsLogConfig extends EventsMapping<LogLevel> {
+ public EventsLogConfig() {
+ this.defaul = LogLevel.ERROR;
+ this.set(ConsistencyEvent.UNKNOWN, LogLevel.WARN);
+ }
+ }
/**
- * The AutoUpdateConfig maps {@link ConsistencyEvent}s to {@link AutoUpdateAction}s.
+ * The AutoUpdateConfig maps {@link ConsistencyEventInterface}s to {@link AutoUpdateAction}s.
*
* <p>When the consistency check of a file storage fires an event this mapping determines whether
* and how the server attempts to react to the event automatically. E.g. by inserting a newly
@@ -47,6 +53,11 @@ public class ConsistencyConfig {
*/
public static class AutoUpdateConfig extends EventsMapping<AutoUpdateAction> {}
- public EventsLogConfig eventsLogConfig = null;
- public AutoUpdateConfig autoUpdateConfig = null;
+ public EventsLogConfig eventsLogConfig = new EventsLogConfig();
+ public AutoUpdateConfig autoUpdateConfig = new AutoUpdateConfig();
+ private static ConsistencyConfig instance = new ConsistencyConfig();
+
+ public static ConsistencyConfig getInstance() {
+ return instance;
+ }
}
diff --git a/src/main/java/org/caosdb/server/filesystem/consistency/ConsistencyEvent.java b/src/main/java/org/caosdb/server/filesystem/consistency/ConsistencyEvent.java
index f0c6b0b931285d3488a4c95a4ca93d97f8a5bbd4..373808396bcae936d4ea688f0b91c7dce974d7df 100644
--- a/src/main/java/org/caosdb/server/filesystem/consistency/ConsistencyEvent.java
+++ b/src/main/java/org/caosdb/server/filesystem/consistency/ConsistencyEvent.java
@@ -1,39 +1,139 @@
package org.caosdb.server.filesystem.consistency;
-public interface ConsistencyEvent {
-
- /**
- * Return a universal unique identifier for this event. This can be used by other events or by
- * triggered actions for referencing this event.
- */
- public String getUUID();
-
- /**
- * If applicable, return the id of the file storage which this event applies to.
- *
- * <p>Otherwise, return null.
- */
- public String getFileStorageId();
-
- /**
- * If applicable, return the key of the file which this event applies to.
- *
- * <p>Otherwise, return null.
- *
- * <p>If this is not null, {@link #getFileStorageId()} must not be null.
- */
- public String getKey();
-
- /**
- * Return the name of this event. The name of events of t
- *
- * @return
- */
- public String getEventType();
-
- /** Return true iff this event is a subtype of the given other event type. */
- public boolean isSubtypeOf(String eventType);
-
- /** Return the unix time stamp when this event was initialized. */
- Long getTimestamp();
+import java.util.UUID;
+import org.caosdb.server.filesystem.FSODescriptorInterface;
+import org.caosdb.server.filesystem.FileStorageInterface;
+import org.caosdb.server.filesystem.FileSystem;
+
+public class ConsistencyEvent implements ConsistencyEventInterface {
+
+ public static final String UNKNOWN = "UNKNOWN";
+ private String fileStorageId;
+ private String key;
+ private String type;
+ private String uuid = null;
+ private Long timestamp;
+
+ ConsistencyEvent(String type, String fileStorageId, String key) {
+ this.timestamp = System.currentTimeMillis();
+ this.type = type;
+ // this.parent = parent;
+ this.fileStorageId = fileStorageId;
+ this.key = key;
+ }
+
+ @Override
+ public String getFileStorageId() {
+ return fileStorageId;
+ }
+
+ @Override
+ public String getKey() {
+ return key;
+ }
+
+ @Override
+ public String getEventType() {
+ return type;
+ }
+ //
+ // @Override
+ // public ConsistencyEvent getParent() {
+ // if (parent == null && type.contains(":")) {
+ // String[] components = type.split(":");
+ // components = Arrays.copyOfRange(components, 0, components.length - 2);
+ // parent = new AbstractConsistencyEvent(String.join(":", components), fileStorageId, key);
+ // }
+ // return parent;
+ // }
+
+ @Override
+ public boolean isSubtypeOf(String eventType) {
+ if (!eventType.endsWith(":")) {
+ return this.type.startsWith(eventType + ":");
+ }
+ return this.type.startsWith(eventType);
+ }
+
+ public static final ConsistencyEventInterface missingCapability(
+ String capability, FileStorageInterface fileStorage) {
+ return new ConsistencyEvent("NEED_CAPABILITY:" + capability, fileStorage.getId(), null);
+ }
+
+ public static ConsistencyEventInterface unknownFile(FSODescriptorInterface next) {
+ return new ConsistencyEvent(UNKNOWN, next.getFileStorageId(), next.getKey());
+ }
+
+ public static ConsistencyEventInterface missingFile(FSODescriptorInterface next) {
+ return new ConsistencyEvent("MISSING", next.getFileStorageId(), next.getKey());
+ }
+
+ public static ConsistencyEventInterface wrongObjectType(
+ FSODescriptorInterface fso, FileSystem.ObjectType type) {
+ return new ConsistencyEvent(
+ "CHANGED:TYPE:TO_" + type.toString(), fso.getFileStorageId(), fso.getKey());
+ }
+
+ public static ConsistencyEventInterface brokenLink(FSODescriptorInterface fso) {
+ return new ConsistencyEvent("LINK:BROKEN", fso.getFileStorageId(), fso.getKey());
+ }
+
+ public static ConsistencyEventInterface linkToOtherFileStorage(FSODescriptorInterface fso) {
+ return new ConsistencyEvent(
+ "LINK:TARGET:IN_OTHER_FILE_STORAGE", fso.getFileStorageId(), fso.getKey());
+ }
+
+ public static ConsistencyEventInterface linkToAlienTargetFromOuterSpace(
+ FSODescriptorInterface fso) {
+ return new ConsistencyEvent(
+ "LINK:TARGET:NOT_IN_FILE_STORAGE", fso.getFileStorageId(), fso.getKey());
+ }
+
+ public static ConsistencyEventInterface sizeMismatch(FSODescriptorInterface fso) {
+ return new ConsistencyEvent("CHANGED:SIZE", fso.getFileStorageId(), fso.getKey());
+ }
+
+ public static ConsistencyEventInterface hashMismatch(FSODescriptorInterface fso) {
+ return new ConsistencyEvent("CHANGED:HASH", fso.getFileStorageId(), fso.getKey());
+ }
+
+ public static ConsistencyEventInterface linkTargetChanged(FSODescriptorInterface fso) {
+ return new ConsistencyEvent("CHANGED:LINK:TARGET", fso.getFileStorageId(), fso.getKey());
+ }
+
+ @Override
+ public String getUUID() {
+ if (uuid == null) {
+ uuid = UUID.randomUUID().toString();
+ }
+ return uuid;
+ }
+
+ @Override
+ public Long getTimestamp() {
+ return timestamp;
+ }
+
+ public static ConsistencyEventInterface missingChild(FSODescriptorInterface missingObject) {
+ return missingFile(missingObject);
+ }
+
+ public static ConsistencyEventInterface unknownChild(FSODescriptorInterface unknownObject) {
+ return unknownFile(unknownObject);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder result = new StringBuilder();
+ result.append(type);
+ if (fileStorageId != null) {
+ result.append(" - ");
+ result.append(fileStorageId);
+ if (key != null) {
+ result.append(":");
+ result.append(key);
+ }
+ }
+ return result.toString();
+ }
}
diff --git a/src/main/java/org/caosdb/server/filesystem/consistency/ConsistencyEventInterface.java b/src/main/java/org/caosdb/server/filesystem/consistency/ConsistencyEventInterface.java
new file mode 100644
index 0000000000000000000000000000000000000000..1c100cc7111bcf49401d9478b3bf91e77d1145ed
--- /dev/null
+++ b/src/main/java/org/caosdb/server/filesystem/consistency/ConsistencyEventInterface.java
@@ -0,0 +1,35 @@
+package org.caosdb.server.filesystem.consistency;
+
+public interface ConsistencyEventInterface {
+
+ /**
+ * Return a universal unique identifier for this event. This can be used by other events or by
+ * triggered actions for referencing this event.
+ */
+ public String getUUID();
+
+ /**
+ * If applicable, return the id of the file storage which this event applies to.
+ *
+ * <p>Otherwise, return null.
+ */
+ public String getFileStorageId();
+
+ /**
+ * If applicable, return the key of the file which this event applies to.
+ *
+ * <p>Otherwise, return null.
+ *
+ * <p>If this is not null, {@link #getFileStorageId()} must not be null.
+ */
+ public String getKey();
+
+ /** Return the type of this event. */
+ public String getEventType();
+
+ /** Return true iff this event is a subtype of the given other event type. */
+ public boolean isSubtypeOf(String eventType);
+
+ /** Return the unix time stamp when this event was initialized. */
+ Long getTimestamp();
+}
diff --git a/src/main/java/org/caosdb/server/jobs/core/CheckFileStorageConsistency.java b/src/main/java/org/caosdb/server/jobs/core/CheckFileStorageConsistency.java
index ff8cbc50578a79bec53869fe456e484b93b8a244..bd6b42d202f7b6acbc7bf16aba37d753fb219007 100644
--- a/src/main/java/org/caosdb/server/jobs/core/CheckFileStorageConsistency.java
+++ b/src/main/java/org/caosdb/server/jobs/core/CheckFileStorageConsistency.java
@@ -29,24 +29,31 @@ import java.io.FileReader;
import java.io.IOException;
import java.io.PrintStream;
import java.io.Reader;
+import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import org.caosdb.api.entity.v1.MessageCode;
import org.caosdb.server.CaosDBServer;
import org.caosdb.server.database.exceptions.TransactionException;
import org.caosdb.server.entity.Message;
+import org.caosdb.server.entity.Message.MessageType;
+import org.caosdb.server.entity.xml.ToElementable;
import org.caosdb.server.filesystem.FileSystem;
+import org.caosdb.server.filesystem.Path;
import org.caosdb.server.filesystem.RealFSODescriptorInterface;
+import org.caosdb.server.filesystem.consistency.ConsistencyCheck;
+import org.caosdb.server.filesystem.consistency.ConsistencyConfig;
+import org.caosdb.server.filesystem.consistency.ConsistencyConfig.EventsLogConfig;
+import org.caosdb.server.filesystem.consistency.ConsistencyConfig.LogLevel;
+import org.caosdb.server.filesystem.consistency.ConsistencyEventInterface;
import org.caosdb.server.jobs.FlagJob;
import org.caosdb.server.jobs.JobAnnotation;
-import org.caosdb.server.transaction.FileStorageConsistencyCheck;
import org.caosdb.server.transaction.Retrieve;
import org.caosdb.server.transaction.Transaction;
import org.caosdb.server.utils.Observable;
import org.caosdb.server.utils.Observer;
import org.caosdb.server.utils.Undoable;
-import org.jdom2.Document;
-import org.jdom2.output.Format;
-import org.jdom2.output.XMLOutputter;
+import org.jdom2.Element;
@JobAnnotation(
flag = "fileStorageConsistency",
@@ -63,7 +70,7 @@ public class CheckFileStorageConsistency extends FlagJob {
public static final Pattern parseArgs =
Pattern.compile("\\s*-t\\s*([0-9]+)\\s*|\\s*-c\\s*([^\\s]+)\\s*|\\s*([^\\s]+\\s*)");
private int timeout = 1000 * 30; // milliseconds
- private String location = "";
+ private Path location = null;
public static enum TestCase {
FILE_DOES_NOT_EXIST,
@@ -96,28 +103,12 @@ public class CheckFileStorageConsistency extends FlagJob {
}
if (matcher.group(3) != null) {
// group three is file system location
- this.location = matcher.group(3);
+ this.location = new Path(matcher.group(3));
}
}
}
- final FileStorageConsistencyCheck test = new FileStorageConsistencyCheck(this.location);
- test.setOnFinish(
- new Runnable() {
- @Override
- public void run() {
-
- final Document doc = new Document(test.toElement());
- final XMLOutputter out = new XMLOutputter(Format.getPrettyFormat());
- try {
- out.output(doc, new PrintStream(new File("./ConsistencyTest.xml")));
- } catch (final FileNotFoundException e) {
- throw new TransactionException(e);
- } catch (final IOException e) {
- throw new TransactionException(e);
- }
- }
- });
+ ConsistencyCheck test = new ConsistencyCheck(this.location);
test.start();
try {
test.join(this.timeout);
@@ -138,10 +129,56 @@ public class CheckFileStorageConsistency extends FlagJob {
throw new TransactionException(e);
}
- getContainer().addMessage(test);
+ getContainer().addMessage(createMessages(test));
}
}
+ private ToElementable createMessages(ConsistencyCheck test) {
+ final List<ConsistencyEventInterface> events = test.getEvents();
+ return new ToElementable() {
+
+ @Override
+ public void addToElement(Element ret) {
+ EventsLogConfig c = ConsistencyConfig.getInstance().eventsLogConfig;
+ if (events.size() == 0) {
+ if (location != null) {
+ ret.addContent(
+ new Message("File system below " + location + "/ is consistent.").toElement());
+ } else {
+ ret.addContent(new Message("File system is consistent.").toElement());
+ }
+ } else {
+ for (ConsistencyEventInterface event : events) {
+ LogLevel level = c.get(event.getEventType());
+ switch (level) {
+ case ERROR:
+ ret.addContent(
+ new Message(
+ MessageType.Error, MessageCode.MESSAGE_CODE_UNKNOWN, event.toString())
+ .toElement());
+ break;
+ case WARN:
+ ret.addContent(
+ new Message(
+ MessageType.Warning, MessageCode.MESSAGE_CODE_UNKNOWN, event.toString())
+ .toElement());
+ break;
+ case INFO:
+ ret.addContent(
+ new Message(
+ MessageType.Info, MessageCode.MESSAGE_CODE_UNKNOWN, event.toString())
+ .toElement());
+ break;
+ default:
+ // ignore
+ break;
+ }
+ }
+ }
+ }
+ };
+ }
+
private void test(final TestCase testCase) throws IOException, Message {
switch (testCase) {
case FILE_DOES_NOT_EXIST:
@@ -168,7 +205,7 @@ public class CheckFileStorageConsistency extends FlagJob {
case UNKNOWN_FILE:
final File dir =
- FileSystem.getInstance().getFileStorage("default").resolve("debug/").getFile();
+ FileSystem.getInstance().getFileStorage("default").resolve("test_unknown").getFile();
dir.mkdirs();
@@ -179,7 +216,7 @@ public class CheckFileStorageConsistency extends FlagJob {
public void undo() {
try {
Undoable delete =
- FileSystem.getInstance().getFileStorage("default").delete("debug/");
+ FileSystem.getInstance().getFileStorage("default").delete("test_unknown");
delete.cleanUp();
} catch (Message e) {
e.printStackTrace();
diff --git a/src/main/java/org/caosdb/server/transaction/ChecksumUpdater.java b/src/main/java/org/caosdb/server/transaction/ChecksumUpdater.java
index 791ce3a6b30ae2e8f5e44e3842ba541de901ca2a..04d4bfc6f5302908e8bc4e18701f6877765bbbc8 100644
--- a/src/main/java/org/caosdb/server/transaction/ChecksumUpdater.java
+++ b/src/main/java/org/caosdb/server/transaction/ChecksumUpdater.java
@@ -72,7 +72,12 @@ public class ChecksumUpdater extends WriteTransaction
}
// we calculate the hash, even though we don't have reserved write access...
- final Hash checksum = calcChecksum(fileEntity);
+ final Hash checksum =
+ FileSystem.getInstance()
+ .resolve(
+ fileEntity.getFSODescriptor().getFileStorageId(),
+ fileEntity.getFSODescriptor().getKey())
+ .getHash();
if (checksum == null) {
// TODO file system does not support hashing? Mark this file appropriately
continue;
@@ -120,10 +125,6 @@ public class ChecksumUpdater extends WriteTransaction
}
}
- private Hash calcChecksum(final EntityInterface fileEntity) {
- return FileSystem.getHash(fileEntity.getFSODescriptor(), "SHA-512");
- }
-
private EntityInterface getNextUpdateableFileEntity() throws InterruptedException {
final Access weakAccess = DatabaseAccessManager.getInstance().acquireReadAccess(this);
try {
diff --git a/src/main/java/org/caosdb/server/transaction/FileStorageConsistencyCheck.java b/src/main/java/org/caosdb/server/transaction/FileStorageConsistencyCheck.java
index df876fdc418bbf1985a6e78b0de68b98888060ec..a9400eb937077aabd100a58982be82066a25d402 100644
--- a/src/main/java/org/caosdb/server/transaction/FileStorageConsistencyCheck.java
+++ b/src/main/java/org/caosdb/server/transaction/FileStorageConsistencyCheck.java
@@ -1,244 +1,245 @@
-/*
- * ** header v3.0
- * This file is a part of the CaosDB Project.
- *
- * Copyright (C) 2018 Research Group Biomedical Physics,
- * Max-Planck-Institute for Dynamics and Self-Organization Göttingen
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
- * ** end header
- */
-package org.caosdb.server.transaction;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map.Entry;
-import java.util.TimeZone;
-import org.caosdb.api.entity.v1.MessageCode;
-import org.caosdb.datetime.UTCDateTime;
-import org.caosdb.server.database.DatabaseAccessManager;
-import org.caosdb.server.database.access.Access;
-import org.caosdb.server.database.backend.transaction.FileConsistencyCheck;
-import org.caosdb.server.database.backend.transaction.GetFileEntityByPath;
-import org.caosdb.server.database.backend.transaction.GetFileIterator;
-import org.caosdb.server.database.backend.transaction.RetrieveAllUncheckedFiles;
-import org.caosdb.server.database.backend.transaction.SetFileCheckedTimestamp;
-import org.caosdb.server.database.exceptions.EntityDoesNotExistException;
-import org.caosdb.server.database.proto.SparseEntity;
-import org.caosdb.server.entity.EntityID;
-import org.caosdb.server.entity.Message;
-import org.caosdb.server.entity.xml.ToElementable;
-import org.caosdb.server.filesystem.FileSystem;
-import org.caosdb.server.filesystem.Path;
-import org.jdom2.Element;
-
-public class FileStorageConsistencyCheck extends Thread
- implements ToElementable, TransactionInterface {
-
- private Access access = null;
- private final HashMap<String, Integer> results = new HashMap<String, Integer>();
- private Exception exception = null;
- private Runnable finishRunnable = null;
- private final String location;
- private Long ts = null;
- private UTCDateTime timestamp = null;
-
- public Exception getException() {
- return this.exception;
- }
-
- public FileStorageConsistencyCheck(final String location) {
- setDaemon(true);
- this.timestamp = UTCDateTime.SystemMillisToUTCDateTime(System.currentTimeMillis());
- this.location = location.startsWith("/") ? location.replaceFirst("^/", "") : location;
- }
-
- @Override
- public void run() {
- for (final String storage : FileSystem.getInstance().getFileStorages()) {
- try {
- this.access = DatabaseAccessManager.getInstance().acquireReadAccess(this);
-
- // test all files in file system.
- final GetFileIterator it = new GetFileIterator(storage, this.location);
- final Iterator<String> iterator = execute(it, this.access).getIterator();
-
- this.ts = System.currentTimeMillis();
- while (iterator != null && iterator.hasNext()) {
- final String path = iterator.next();
-
- if (DatabaseAccessManager.whoHasReservedWriteAccess() != null) {
- // there is a thread waiting to write. pause this one and
- // try to acquire a new read access which will be granted when
- // the write thread is done.
- this.access.release();
- this.access = DatabaseAccessManager.getInstance().acquireReadAccess(this);
- }
-
- try {
- final GetFileEntityByPath t =
- execute(new GetFileEntityByPath(new Path(path)), this.access);
- final int result =
- execute(new FileConsistencyCheck(path, t.getSize(), t.getHash()), this.access)
- .getResult();
-
- if (result != FileConsistencyCheck.OK) {
- this.results.put(path, result);
- }
-
- execute(new SetFileCheckedTimestamp(t.getId(), this.ts), this.access);
- } catch (final EntityDoesNotExistException e) {
- this.results.put(path, FileConsistencyCheck.UNKNOWN_FILE);
- continue;
- }
- }
-
- // test all remaining file records
- final Iterator<SparseEntity> iterator2 =
- execute(new RetrieveAllUncheckedFiles(this.ts, this.location), this.access)
- .getIterator();
- while (iterator2 != null && iterator2.hasNext()) {
-
- final SparseEntity entity = iterator2.next();
- final int result =
- execute(
- new FileConsistencyCheck(entity.fileKey, entity.fileSize, entity.fileHash),
- this.access)
- .getResult();
-
- if (result != FileConsistencyCheck.OK) {
- this.results.put(entity.filePath.toString(), result);
- }
-
- execute(new SetFileCheckedTimestamp(new EntityID(entity.id), this.ts), this.access);
- }
-
- } catch (final Exception e) {
- this.exception = e;
- } finally {
- this.access.release();
- }
- }
- synchronized (this.results) {
- if (this.finishRunnable != null) {
- this.finishRunnable.run();
- }
- }
- }
-
- public HashMap<String, Integer> getResults() {
- return this.results;
- }
-
- public void setOnFinish(final Runnable r) {
- synchronized (this.results) {
- this.finishRunnable = r;
- }
- }
-
- @Override
- public void addToElement(final Element e) {
- if (this.ts != null) {
- e.setAttribute(
- "timestamp",
- UTCDateTime.SystemMillisToUTCDateTime(this.ts).toDateTimeString(TimeZone.getDefault()));
- }
- if (this.location != null) {
- e.setAttribute("location", this.location);
- }
-
- if (getException() != null) {
- final StringBuilder sb = new StringBuilder();
- sb.append(getException().toString());
- for (final StackTraceElement t : getException().getStackTrace()) {
- sb.append('\n').append(t.toString());
- }
-
- e.addContent(
- new Message(
- "Error",
- MessageCode.MESSAGE_CODE_UNKNOWN,
- "An exception was thrown.",
- sb.toString())
- .toElement());
- }
-
- final List<Message> results2Messages = results2Messages(getResults(), this.location);
- for (final Message m : results2Messages) {
- e.addContent(m.toElement());
- }
- }
-
- public Element toElement() {
- final Element results = new Element("Results");
- addToElement(results);
- return results;
- }
-
- private static List<Message> results2Messages(
- final HashMap<String, Integer> results, final String location) {
- final ArrayList<Message> ret = new ArrayList<Message>();
- if (results.isEmpty()) {
- if (location.length() > 0) {
- ret.add(new Message("File system below " + location + " is consistent."));
- } else {
- ret.add(new Message("File system is consistent."));
- }
- }
- for (final Entry<String, Integer> r : results.entrySet()) {
- switch (r.getValue()) {
- case FileConsistencyCheck.FILE_DOES_NOT_EXIST:
- ret.add(
- new Message(
- "Error",
- MessageCode.MESSAGE_CODE_UNKNOWN,
- r.getKey() + ": File does not exist."));
- break;
- case FileConsistencyCheck.FILE_MODIFIED:
- ret.add(
- new Message(
- "Error", MessageCode.MESSAGE_CODE_UNKNOWN, r.getKey() + ": File was modified."));
- break;
- case FileConsistencyCheck.UNKNOWN_FILE:
- ret.add(
- new Message(
- "Warning", MessageCode.MESSAGE_CODE_UNKNOWN, r.getKey() + ": Unknown file."));
- break;
- case FileConsistencyCheck.NONE:
- ret.add(
- new Message(
- "Warning",
- MessageCode.MESSAGE_CODE_UNKNOWN,
- r.getKey() + ": Test result not available."));
- break;
- default:
- break;
- }
- }
- return ret;
- }
-
- @Override
- public void execute() throws Exception {
- run();
- }
-
- @Override
- public UTCDateTime getTimestamp() {
- return timestamp;
- }
-}
+/// *
+// * ** header v3.0
+// * This file is a part of the CaosDB Project.
+// *
+// * Copyright (C) 2018 Research Group Biomedical Physics,
+// * Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+// *
+// * This program is free software: you can redistribute it and/or modify
+// * it under the terms of the GNU Affero General Public License as
+// * published by the Free Software Foundation, either version 3 of the
+// * License, or (at your option) any later version.
+// *
+// * This program is distributed in the hope that it will be useful,
+// * but WITHOUT ANY WARRANTY; without even the implied warranty of
+// * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// * GNU Affero General Public License for more details.
+// *
+// * You should have received a copy of the GNU Affero General Public License
+// * along with this program. If not, see <https://www.gnu.org/licenses/>.
+// *
+// * ** end header
+// */
+// package org.caosdb.server.transaction;
+//
+// import java.util.ArrayList;
+// import java.util.HashMap;
+// import java.util.Iterator;
+// import java.util.List;
+// import java.util.Map.Entry;
+// import java.util.TimeZone;
+// import org.caosdb.api.entity.v1.MessageCode;
+// import org.caosdb.datetime.UTCDateTime;
+// import org.caosdb.server.database.DatabaseAccessManager;
+// import org.caosdb.server.database.access.Access;
+// import org.caosdb.server.database.backend.transaction.FileConsistencyCheck;
+// import org.caosdb.server.database.backend.transaction.GetFileEntityByPath;
+// import org.caosdb.server.database.backend.transaction.GetFileIterator;
+// import org.caosdb.server.database.backend.transaction.RetrieveAllUncheckedFiles;
+// import org.caosdb.server.database.backend.transaction.SetFileCheckedTimestamp;
+// import org.caosdb.server.database.exceptions.EntityDoesNotExistException;
+// import org.caosdb.server.database.proto.SparseEntity;
+// import org.caosdb.server.entity.EntityID;
+// import org.caosdb.server.entity.Message;
+// import org.caosdb.server.entity.xml.ToElementable;
+// import org.caosdb.server.filesystem.FileSystem;
+// import org.caosdb.server.filesystem.Path;
+// import org.jdom2.Element;
+//
+// public class FileStorageConsistencyCheck extends Thread
+// implements ToElementable, TransactionInterface {
+//
+// private Access access = null;
+// private final HashMap<String, Integer> results = new HashMap<String, Integer>();
+// private Exception exception = null;
+// private Runnable finishRunnable = null;
+// private final String location;
+// private Long ts = null;
+// private UTCDateTime timestamp = null;
+//
+// public Exception getException() {
+// return this.exception;
+// }
+//
+// public FileStorageConsistencyCheck(final String location) {
+// setDaemon(true);
+// this.timestamp = UTCDateTime.SystemMillisToUTCDateTime(System.currentTimeMillis());
+// this.location = location.startsWith("/") ? location.replaceFirst("^/", "") : location;
+// }
+//
+// @Override
+// public void run() {
+// for (final String storage : FileSystem.getInstance().getFileStorages()) {
+// try {
+// this.access = DatabaseAccessManager.getInstance().acquireReadAccess(this);
+//
+// // test all files in file system.
+// final GetFileIterator it = new GetFileIterator(storage, this.location);
+// final Iterator<String> iterator = execute(it, this.access).getIterator();
+//
+// this.ts = System.currentTimeMillis();
+// while (iterator != null && iterator.hasNext()) {
+// final String path = iterator.next();
+//
+// if (DatabaseAccessManager.whoHasReservedWriteAccess() != null) {
+// // there is a thread waiting to write. pause this one and
+// // try to acquire a new read access which will be granted when
+// // the write thread is done.
+// this.access.release();
+// this.access = DatabaseAccessManager.getInstance().acquireReadAccess(this);
+// }
+//
+// try {
+// final GetFileEntityByPath t =
+// execute(new GetFileEntityByPath(new Path(path)), this.access);
+// final int result =
+// execute(new FileConsistencyCheck(path, t.getSize(), t.getHash()), this.access)
+// .getResult();
+//
+// if (result != FileConsistencyCheck.OK) {
+// this.results.put(path, result);
+// }
+//
+// execute(new SetFileCheckedTimestamp(t.getId(), this.ts), this.access);
+// } catch (final EntityDoesNotExistException e) {
+// this.results.put(path, FileConsistencyCheck.UNKNOWN_FILE);
+// continue;
+// }
+// }
+//
+// // test all remaining file records
+// final Iterator<SparseEntity> iterator2 =
+// execute(new RetrieveAllUncheckedFiles(this.ts, this.location), this.access)
+// .getIterator();
+// while (iterator2 != null && iterator2.hasNext()) {
+//
+// final SparseEntity entity = iterator2.next();
+// final int result =
+// execute(
+// new FileConsistencyCheck(entity.fileKey, entity.fileSize, entity.fileHash),
+// this.access)
+// .getResult();
+//
+// if (result != FileConsistencyCheck.OK) {
+// this.results.put(entity.filePath.toString(), result);
+// }
+//
+// execute(new SetFileCheckedTimestamp(new EntityID(entity.id), this.ts), this.access);
+// }
+//
+// } catch (final Exception e) {
+// this.exception = e;
+// } finally {
+// this.access.release();
+// }
+// }
+// synchronized (this.results) {
+// if (this.finishRunnable != null) {
+// this.finishRunnable.run();
+// }
+// }
+// }
+//
+// public HashMap<String, Integer> getResults() {
+// return this.results;
+// }
+//
+// public void setOnFinish(final Runnable r) {
+// synchronized (this.results) {
+// this.finishRunnable = r;
+// }
+// }
+//
+// @Override
+// public void addToElement(final Element e) {
+// if (this.ts != null) {
+// e.setAttribute(
+// "timestamp",
+// UTCDateTime.SystemMillisToUTCDateTime(this.ts).toDateTimeString(TimeZone.getDefault()));
+// }
+// if (this.location != null) {
+// e.setAttribute("location", this.location);
+// }
+//
+// if (getException() != null) {
+// final StringBuilder sb = new StringBuilder();
+// sb.append(getException().toString());
+// for (final StackTraceElement t : getException().getStackTrace()) {
+// sb.append('\n').append(t.toString());
+// }
+//
+// e.addContent(
+// new Message(
+// "Error",
+// MessageCode.MESSAGE_CODE_UNKNOWN,
+// "An exception was thrown.",
+// sb.toString())
+// .toElement());
+// }
+//
+// final List<Message> results2Messages = results2Messages(getResults(), this.location);
+// for (final Message m : results2Messages) {
+// e.addContent(m.toElement());
+// }
+// }
+//
+// public Element toElement() {
+// final Element results = new Element("Results");
+// addToElement(results);
+// return results;
+// }
+//
+// private static List<Message> results2Messages(
+// final HashMap<String, Integer> results, final String location) {
+// final ArrayList<Message> ret = new ArrayList<Message>();
+// if (results.isEmpty()) {
+// if (location.length() > 0) {
+// ret.add(new Message("File system below " + location + " is consistent."));
+// } else {
+// ret.add(new Message("File system is consistent."));
+// }
+// }
+// for (final Entry<String, Integer> r : results.entrySet()) {
+// switch (r.getValue()) {
+// case FileConsistencyCheck.FILE_DOES_NOT_EXIST:
+// ret.add(
+// new Message(
+// "Error",
+// MessageCode.MESSAGE_CODE_UNKNOWN,
+// r.getKey() + ": File does not exist."));
+// break;
+// case FileConsistencyCheck.FILE_MODIFIED:
+// ret.add(
+// new Message(
+// "Error", MessageCode.MESSAGE_CODE_UNKNOWN, r.getKey() + ": File was
+// modified."));
+// break;
+// case FileConsistencyCheck.UNKNOWN_FILE:
+// ret.add(
+// new Message(
+// "Warning", MessageCode.MESSAGE_CODE_UNKNOWN, r.getKey() + ": Unknown file."));
+// break;
+// case FileConsistencyCheck.NONE:
+// ret.add(
+// new Message(
+// "Warning",
+// MessageCode.MESSAGE_CODE_UNKNOWN,
+// r.getKey() + ": Test result not available."));
+// break;
+// default:
+// break;
+// }
+// }
+// return ret;
+// }
+//
+// @Override
+// public void execute() throws Exception {
+// run();
+// }
+//
+// @Override
+// public UTCDateTime getTimestamp() {
+// return timestamp;
+// }
+// }