diff --git a/CHANGELOG.md b/CHANGELOG.md index b91d1dcac52b8b1d4596570507fce17d75d77dce..c8b974ae5ae05b07da2e56de557e55f9b78c3645 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed ### -* The default behavior of the query FIND SomeName [...] (as well as COUNT and SELECT) is being made configurable and changes +* The default behavior of the query `FIND SomeName [...]` (as well as COUNT and SELECT) is being + made configurable and changes: * `FIND SomeName` will be interpreted as `FIND <FIND_QUERY_DEFAULT_ROLE> SomeName` from now on where `FIND_QUERY_DEFAULT_ROLE` is a newly introduced server property. @@ -21,7 +22,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `FIND <FIND_QUERY_DEFAULT_ROLE>`. * Of course, administrators can choose to retain the old behavior by setting `FIND_QUERY_DEFAULT_ROLE=ENTITY`. -* CQL now treats `WITH` and `WITH A` equivalently. This is for [#192](https://gitlab.com/caosdb/caosdb-server/-/issues/192). +* CQL now treats `WITH` and `WITH A` equivalently. Issue: [#192](https://gitlab.com/caosdb/caosdb-server/-/issues/192) +* The InsertFilesInDir FlagJob now creates File entities without a name. The previous behavior + caused severe performance problems for very large numbers of files. Issue: [#197](https://gitlab.com/caosdb/caosdb-server/-/issues/197) ### Deprecated ### diff --git a/CITATION.cff b/CITATION.cff index ff126ff3b04d1baec72a6c6d5cb7c4f8aff77e3b..d6e80aaa203bf09b085514ff25aa5927b7965f32 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -13,6 +13,9 @@ authors: - family-names: tom Wörden given-names: Henrik orcid: https://orcid.org/0000-0002-5549-578X + - family-names: Spreckelsen + given-names: Florian + orcid: https://orcid.org/0000-0002-6856-2910 - family-names: Parlitz given-names: Ulrich orcid: https://orcid.org/0000-0003-3058-1435 @@ -22,4 +25,4 @@ authors: title: "CaosDB - Server" version: 0.8.1 doi: 10.3390/data4020083 -date-released: 2022-11-07 \ No newline at end of file +date-released: 2022-11-07 diff --git a/src/main/java/org/caosdb/server/jobs/core/InsertFilesInDir.java b/src/main/java/org/caosdb/server/jobs/core/InsertFilesInDir.java index 5b5418286a81d442822aed0e7252fa4557fb21ff..6390ad7cf6da2483a1589432b2c82ba7dfd1dfae 100644 --- a/src/main/java/org/caosdb/server/jobs/core/InsertFilesInDir.java +++ b/src/main/java/org/caosdb/server/jobs/core/InsertFilesInDir.java @@ -60,7 +60,7 @@ import org.caosdb.server.utils.Utils; loadOnDefault = false, stage = TransactionStage.INIT, description = - "For expert users only! Risk of creating spam records!\nValue of this flag might be any directory on the servers local file system which is part of the server's back-end file storage. This job will insert every readable, nonhidden file in said directory into the database and link the file with a symlink. This is useful to add a huge amount of files without actully copying them to the back-end file storage. If you call this job on a directory more than once every file that was recently added to the source directory is inserted. Every yet known file is left untouched. \nOptional parameter -e EXCLUDE: A regular expression of files which are to be ignored. \n Optional parameter -i INCLUDE: a regular expression of files which are to be included. By default, all files are included. The -e takes precedence. \nOptional parameter -p PREFIX: Stores all new files into the directory PREFIX in the server's file system.\nOptional parameter --force-allow-symlinks: Simlinks in your data are a source of problems for the database. Therefore, simlinks are ignored by default. This option allows symlinks (but still generates simlink warnings). \nPrepend/Dry run: Call this flag with a retrieve transaction (HTTP GET) and it will only count all files and list them without actually inserting them.") + "For expert users only! Risk of creating spam records!\nValue of this flag may be any directory on the servers local file system which is part of the server's back-end file storage. This job will insert every readable, nonhidden file in said directory into the database and link the file with a symlink. This is useful to add a huge amount of files without actually copying them to the back-end file storage. If you call this job on a directory more than once every file that was recently added to the source directory is inserted. Every already known file is left untouched. \nOptional parameter -e EXCLUDE: A regular expression of files which are to be ignored. \n Optional parameter -i INCLUDE: a regular expression of files which are to be included. By default, all files are included. The -e takes precedence. \nOptional parameter -p PREFIX: Stores all new files into the directory PREFIX in the server's file system.\nOptional parameter --force-allow-symlinks: Symlinks in your data are a source of problems for the database. Therefore, simlinks are ignored by default. This option allows symlinks (but still generates simlink warnings). \nPrepend/Dry run: Call this flag with a retrieve transaction (HTTP GET) and it will only count all files and list them without actually inserting them.") public class InsertFilesInDir extends FlagJob { private File tmp = null; @@ -95,6 +95,11 @@ public class InsertFilesInDir extends FlagJob { return ret; } + /** + * Parse the value string and store the content in this Job. + * + * @return The path to the source directory as given by the flag parameter. + */ public String parseValue(String value) { String ret = value; @@ -217,7 +222,7 @@ public class InsertFilesInDir extends FlagJob { } else { i++; final String targetPath = root + sub.getName(); - final EntityInterface newFileEntity = createInsertFileEntity(sub.getName()); + final EntityInterface newFileEntity = createInsertFileEntity(); final long size = sub.length(); final FileProperties fp = new FileProperties(null, targetPath, size); newFileEntity.setFileProperties(fp); @@ -266,14 +271,13 @@ public class InsertFilesInDir extends FlagJob { * Create a new InsertEntity (if this is an actual run) or a new RetrieveEntity (in dry-run mode) * with {@link Role.File}. * - * @param name the file name * @return new File entity */ - private EntityInterface createInsertFileEntity(String name) { + private EntityInterface createInsertFileEntity() { if (getTransaction() instanceof WriteTransactionInterface) { - return new InsertEntity(name, Role.File); + return new InsertEntity((String) null, Role.File); } - EntityInterface result = new RetrieveEntity(name); + EntityInterface result = new RetrieveEntity((String) null); result.setRole(Role.File); return result; } diff --git a/src/test/docker/Dockerfile b/src/test/docker/Dockerfile index ec670b7d4b43709054e9794f501aea111a059a0a..1752f88c159a1d6d950b30c7bf6dac32c8823b06 100644 --- a/src/test/docker/Dockerfile +++ b/src/test/docker/Dockerfile @@ -4,7 +4,7 @@ RUN apt-get update && \ git make mariadb-server maven openjdk-11-jdk-headless \ plantuml \ python3-pip screen libpam0g-dev unzip curl shunit2 \ - python3-sphinx \ + python3-sphinx RUN pip3 install javasphinx recommonmark sphinx-rtd-theme sphinxcontrib-plantuml sphinx-a4doc