Merge branch 'f-no-automatic-file-naming' into 'dev'

ENH: No automatic name for mass-imported File entities

See merge request !85

CHANGELOG.md

@@ -11,7 +11,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed ###
 
-* The default behavior of the query FIND SomeName [...] (as well as COUNT and SELECT) is being made configurable and changes
+* The default behavior of the query `FIND SomeName [...]` (as well as COUNT and SELECT) is being
+  made configurable and changes:
   * `FIND SomeName` will be interpreted as `FIND <FIND_QUERY_DEFAULT_ROLE>
     SomeName` from now on where `FIND_QUERY_DEFAULT_ROLE` is a newly introduced
     server property.
@@ -21,7 +22,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
     `FIND <FIND_QUERY_DEFAULT_ROLE>`.
   * Of course, administrators can choose to retain the old behavior by setting
     `FIND_QUERY_DEFAULT_ROLE=ENTITY`.
-* CQL now treats `WITH` and `WITH A` equivalently. This is for [#192](https://gitlab.com/caosdb/caosdb-server/-/issues/192).
+* CQL now treats `WITH` and `WITH A` equivalently. Issue: [#192](https://gitlab.com/caosdb/caosdb-server/-/issues/192)
+* The InsertFilesInDir FlagJob now creates File entities without a name.  The previous behavior
+  caused severe performance problems for very large numbers of files. Issue: [#197](https://gitlab.com/caosdb/caosdb-server/-/issues/197)
 
 ### Deprecated ###
 

CITATION.cff

@@ -13,6 +13,9 @@ authors:
   - family-names: tom Wörden
     given-names: Henrik
     orcid: https://orcid.org/0000-0002-5549-578X
+  - family-names: Spreckelsen
+    given-names: Florian
+    orcid: https://orcid.org/0000-0002-6856-2910
   - family-names: Parlitz
     given-names: Ulrich
     orcid: https://orcid.org/0000-0003-3058-1435
@@ -22,4 +25,4 @@ authors:
 title: "CaosDB - Server"
 version: 0.8.1
 doi: 10.3390/data4020083
-date-released: 2022-11-07
\ No newline at end of file
+date-released: 2022-11-07

src/main/java/org/caosdb/server/jobs/core/InsertFilesInDir.java

@@ -60,7 +60,7 @@ import org.caosdb.server.utils.Utils;
     loadOnDefault = false,
     stage = TransactionStage.INIT,
     description =
-        "For expert users only! Risk of creating spam records!\nValue of this flag might be any directory on the servers local file system which is part of the server's back-end file storage. This job will insert every readable, nonhidden file in said directory into the database and link the file with a symlink. This is useful to add a huge amount of files without actully copying them to the back-end file storage. If you call this job on a directory more than once every file that was recently added to the source directory is inserted. Every yet known file is left untouched. \nOptional parameter -e EXCLUDE: A regular expression of files which are to be ignored. \n Optional parameter -i INCLUDE: a regular expression of files which are to be included. By default, all files are included. The -e takes precedence. \nOptional parameter -p PREFIX: Stores all new files into the directory PREFIX in the server's file system.\nOptional parameter --force-allow-symlinks: Simlinks in your data are a source of problems for the database. Therefore, simlinks are ignored by default. This option allows symlinks (but still generates simlink warnings). \nPrepend/Dry run: Call this flag with a retrieve transaction (HTTP GET) and it will only count all files and list them without actually inserting them.")
+        "For expert users only! Risk of creating spam records!\nValue of this flag may be any directory on the servers local file system which is part of the server's back-end file storage. This job will insert every readable, nonhidden file in said directory into the database and link the file with a symlink. This is useful to add a huge amount of files without actually copying them to the back-end file storage. If you call this job on a directory more than once every file that was recently added to the source directory is inserted. Every already known file is left untouched. \nOptional parameter -e EXCLUDE: A regular expression of files which are to be ignored. \n Optional parameter -i INCLUDE: a regular expression of files which are to be included. By default, all files are included. The -e takes precedence. \nOptional parameter -p PREFIX: Stores all new files into the directory PREFIX in the server's file system.\nOptional parameter --force-allow-symlinks: Symlinks in your data are a source of problems for the database. Therefore, simlinks are ignored by default. This option allows symlinks (but still generates simlink warnings). \nPrepend/Dry run: Call this flag with a retrieve transaction (HTTP GET) and it will only count all files and list them without actually inserting them.")
 public class InsertFilesInDir extends FlagJob {
 
   private File tmp = null;
@@ -95,6 +95,11 @@ public class InsertFilesInDir extends FlagJob {
     return ret;
   }
 
+  /**
+   * Parse the value string and store the content in this Job.
+   *
+   * @return The path to the source directory as given by the flag parameter.
+   */
   public String parseValue(String value) {
 
     String ret = value;
@@ -217,7 +222,7 @@ public class InsertFilesInDir extends FlagJob {
         } else {
           i++;
           final String targetPath = root + sub.getName();
-          final EntityInterface newFileEntity = createInsertFileEntity(sub.getName());
+          final EntityInterface newFileEntity = createInsertFileEntity();
           final long size = sub.length();
           final FileProperties fp = new FileProperties(null, targetPath, size);
           newFileEntity.setFileProperties(fp);
@@ -266,14 +271,13 @@ public class InsertFilesInDir extends FlagJob {
    * Create a new InsertEntity (if this is an actual run) or a new RetrieveEntity (in dry-run mode)
    * with {@link Role.File}.
    *
-   * @param name the file name
    * @return new File entity
    */
-  private EntityInterface createInsertFileEntity(String name) {
+  private EntityInterface createInsertFileEntity() {
     if (getTransaction() instanceof WriteTransactionInterface) {
-      return new InsertEntity(name, Role.File);
+      return new InsertEntity((String) null, Role.File);
     }
-    EntityInterface result = new RetrieveEntity(name);
+    EntityInterface result = new RetrieveEntity((String) null);
     result.setRole(Role.File);
     return result;
   }

src/test/docker/Dockerfile

@@ -4,7 +4,7 @@ RUN apt-get update && \
     git make mariadb-server maven openjdk-11-jdk-headless \
     plantuml \
     python3-pip screen libpam0g-dev unzip curl shunit2 \
-    python3-sphinx \
+    python3-sphinx
 
 RUN pip3 install javasphinx recommonmark sphinx-rtd-theme sphinxcontrib-plantuml sphinx-a4doc