Skip to content
Snippets Groups Projects
Commit 25a4673c authored by Daniel's avatar Daniel
Browse files

Merge branch 'dev' into f-more-sql-benchmarking

parents 60e596c2 1b938aea
Branches
Tags
No related merge requests found
Show whitespace changes
Inline Side-by-side
Showing
with 114 additions and 69 deletions
src/main/java/caosdb/server/FileSystem.java
+ 20
− 12
View file @ 25a4673c
......@@ -24,16 +24,6 @@
package caosdb.server;
import caosdb.server.database.access.Access;
import caosdb.server.database.backend.transaction.GetFileRecordByPath;
import caosdb.server.database.misc.TransactionBenchmark;
import caosdb.server.entity.EntityInterface;
import caosdb.server.entity.FileProperties;
import caosdb.server.entity.Message;
import caosdb.server.utils.FileUtils;
import caosdb.server.utils.ServerMessages;
import caosdb.server.utils.Utils;
import com.google.common.io.Files;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
......@@ -44,6 +34,17 @@ import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.regex.Pattern;
import org.apache.commons.fileupload.FileItemStream;
import com.google.common.io.Files;
import caosdb.server.database.access.Access;
import caosdb.server.database.backend.transaction.GetFileRecordByPath;
import caosdb.server.database.exceptions.EntityDoesNotExistException;
import caosdb.server.database.misc.TransactionBenchmark;
import caosdb.server.entity.EntityInterface;
import caosdb.server.entity.FileProperties;
import caosdb.server.entity.Message;
import caosdb.server.utils.FileUtils;
import caosdb.server.utils.ServerMessages;
import caosdb.server.utils.Utils;
public class FileSystem {
private static String filesystem = null;
......@@ -324,15 +325,22 @@ public class FileSystem {
final GetFileRecordByPath t = new GetFileRecordByPath(file.getPath());
t.setAccess(access);
t.setTransactionBenchmark(b);
try {
t.executeTransaction();
} catch (EntityDoesNotExistException e) {
// could not determine which entity owns this path
// this is usually the case when target is a directory
}
if (t.getEntity() != null) {
final Integer foreign = t.getId();
if (foreign != null && foreign.equals(entity.getId())) {
// entity already owns this path
return true;
}
throw ServerMessages.TARGET_PATH_EXISTS;
}
}
// another entity owns this path
throw ServerMessages.TARGET_PATH_EXISTS;
}
return true;
......
src/main/java/caosdb/server/database/backend/implementation/UnixFileSystem/UnixFileSystemGetFileIterator.java
+ 3
− 3
View file @ 25a4673c
......@@ -22,14 +22,14 @@
*/
package caosdb.server.database.backend.implementation.UnixFileSystem;
import java.io.File;
import java.util.Arrays;
import java.util.Iterator;
import caosdb.server.FileSystem;
import caosdb.server.database.access.Access;
import caosdb.server.database.backend.interfaces.GetFileIteratorImpl;
import caosdb.server.database.exceptions.TransactionException;
import caosdb.server.entity.Message;
import java.io.File;
import java.util.Arrays;
import java.util.Iterator;
public class UnixFileSystemGetFileIterator extends UnixFileSystemTransaction
implements GetFileIteratorImpl {
......
src/main/java/caosdb/server/jobs/core/InsertFilesInDir.java
+ 47
− 31
View file @ 25a4673c
......@@ -22,6 +22,12 @@
*/
package caosdb.server.jobs.core;
import java.io.File;
import java.io.IOException;
import java.util.LinkedList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import caosdb.server.CaosDBException;
import caosdb.server.CaosDBServer;
import caosdb.server.FileSystem;
......@@ -42,12 +48,6 @@ import caosdb.server.utils.EntityStatus;
import caosdb.server.utils.FileUtils;
import caosdb.server.utils.Undoable;
import caosdb.server.utils.Utils;
import java.io.File;
import java.io.IOException;
import java.util.LinkedList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@JobAnnotation(
flag = "InsertFilesInDir",
......@@ -62,6 +62,9 @@ public class InsertFilesInDir extends FlagJob {
private Pattern include = null;
private Pattern exclude = null;
private boolean forceSymLinks = false;
private Pattern valueParser =
Pattern.compile(
"(?:(?:-p\\s*([^\\s]*?)\\s+)|(?:-i\\s*([^\\s]*?)\\s+)|(?:-e\\s*([^\\s]*?)\\s+)|(--force-allow-symlinks\\s+))|([^-].*)");
/**
* @return a List of directories which subdirs are allowed to be batch-added. Needs to be
......@@ -86,14 +89,10 @@ public class InsertFilesInDir extends FlagJob {
return ret;
}
@Override
protected void job(final String value) {
public String parseValue(String value) {
String dirStr = value;
final Pattern pattern =
Pattern.compile(
"(?:(?:-p\\s*([^\\s]*?)\\s+)|(?:-i\\s*([^\\s]*?)\\s+)|(?:-e\\s*([^\\s]*?)\\s+)|(--force-allow-symlinks\\s+))|([^-].*)");
final Matcher matcher = pattern.matcher(value);
String ret = value;
final Matcher matcher = valueParser.matcher(value);
while (matcher.find()) {
if (matcher.group(1) != null) {
this.prefix = matcher.group(1).replaceFirst("/$", "") + "/";
......@@ -108,9 +107,16 @@ public class InsertFilesInDir extends FlagJob {
this.forceSymLinks = true;
}
if (matcher.group(5) != null) {
dirStr = matcher.group(5);
ret = matcher.group(5);
}
}
return ret;
}
@Override
protected void job(final String value) {
String dirStr = parseValue(value);
final File dir = new File(dirStr);
......@@ -247,25 +253,35 @@ public class InsertFilesInDir extends FlagJob {
return i;
}
boolean isExcluded(File f) throws IOException {
return this.exclude != null && this.exclude.matcher(f.getCanonicalPath()).find();
}
boolean isNotIncluded(File f) throws IOException {
return this.include != null && !this.include.matcher(f.getCanonicalPath()).find();
}
private boolean shouldBeProcessed(final File sub) throws IOException {
if (this.include != null && !this.include.matcher(sub.getCanonicalPath()).matches()) {
if (sub.isFile()) {
if (this.isNotIncluded(sub)) {
getContainer()
.addMessage(
new Message(
MessageType.Warning,
1,
"Not explicitly included directory or file: " + sub.getCanonicalPath()));
"Not explicitly included file: " + sub.getCanonicalPath()));
return false;
}
if (this.exclude != null && this.exclude.matcher(sub.getCanonicalPath()).matches()) {
if (this.isExcluded(sub)) {
getContainer()
.addMessage(
new Message(
MessageType.Warning,
2,
"Explicitly excluded directory or file: " + sub.getCanonicalPath()));
"Explicitly excluded file: " + sub.getCanonicalPath()));
return false;
}
}
if (sub.isHidden()) {
getContainer()
.addMessage(
......
src/main/java/caosdb/server/transaction/FileStorageConsistencyCheck.java
+ 25
− 23
View file @ 25a4673c
......@@ -22,6 +22,13 @@
*/
package caosdb.server.transaction;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map.Entry;
import java.util.TimeZone;
import org.jdom2.Element;
import caosdb.datetime.UTCDateTime;
import caosdb.server.database.DatabaseMonitor;
import caosdb.server.database.access.Access;
......@@ -30,17 +37,11 @@ import caosdb.server.database.backend.transaction.GetFileIterator;
import caosdb.server.database.backend.transaction.GetFileRecordByPath;
import caosdb.server.database.backend.transaction.RetrieveAllUncheckedFiles;
import caosdb.server.database.backend.transaction.SetFileCheckedTimestamp;
import caosdb.server.database.exceptions.EntityDoesNotExistException;
import caosdb.server.database.proto.SparseEntity;
import caosdb.server.entity.Message;
import caosdb.server.entity.xml.ToElementable;
import caosdb.server.utils.SHA512;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map.Entry;
import java.util.TimeZone;
import org.jdom2.Element;
public class FileStorageConsistencyCheck extends Thread
implements ToElementable, TransactionInterface {
......@@ -81,17 +82,13 @@ public class FileStorageConsistencyCheck extends Thread
}
final String path = iterator.next();
// FIXME this prevents all files with ".thumbnail" from being checked.
if (path.contains(".thumbnail")) {
// this prevents all thumbnails from being checked.
if (path.contains(".thumbnails/")) {
continue;
}
try {
final GetFileRecordByPath t = execute(new GetFileRecordByPath(path), this.access);
if (t.getEntity() == null) {
this.results.put(path, FileConsistencyCheck.UNKNOWN_FILE);
continue;
}
final int result =
execute(
new FileConsistencyCheck(
......@@ -104,6 +101,11 @@ public class FileStorageConsistencyCheck extends Thread
}
execute(new SetFileCheckedTimestamp(t.getId(), this.ts), this.access);
} catch (EntityDoesNotExistException e) {
this.results.put(path, FileConsistencyCheck.UNKNOWN_FILE);
continue;
}
}
// test all remaining file records
......
src/test/java/caosdb/server/jobs/core/TestInsertFilesInDir.java 0 → 100644
+ 19
− 0
View file @ 25a4673c
package caosdb.server.jobs.core;
import static org.junit.Assert.assertTrue;
import java.io.File;
import java.io.IOException;
import org.junit.Test;
public class TestInsertFilesInDir {
@Test
public void testExclude() throws IOException {
InsertFilesInDir job = new InsertFilesInDir();
job.init(null, null, null);
job.parseValue("-e ^.*test.*$ test");
File testFile = new File("test.dat");
assertTrue(job.isExcluded(testFile));
}
}
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment