ADD script that imports files

src/caosadvancedtools/loadFiles.py

+#!/usr/bin/env python
+# encoding: utf-8
+#
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2018 Research Group Biomedical Physics,
+# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+
+import math
+import sys
+from argparse import ArgumentParser
+
+import caosdb as db
+
+
+def convert_size(size):
+    if (size == 0):
+        return '0B'
+    size_name = ("B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB")
+    i = int(math.floor(math.log(size, 1024)))
+    p = math.pow(1024, i)
+    s = round(size / p, 2)
+
+    return '%s %s' % (s, size_name[i])
+
+
+def loadpath(path, include, exclude, prefix, dryrun, forceAllowSymlinks):
+
+    if dryrun is True:
+        print("DRYRUN")
+        files = db.Container().retrieve(
+            unique=False,
+            raise_exception_on_error=True,
+            flags={"InsertFilesInDir": ("-p " + prefix + " " if prefix else "")
+                   + ("-e " + exclude + " " if exclude else "")
+                   + ("-i " + include + " " if include else "")
+                   + ("--force-allow-symlinks " if forceAllowSymlinks else "")
+                   + path})
+    else:
+        # new files (inserting them using the insertFilesInDir feature of
+        # the server, which inserts files via symlinks)
+        files = db.Container().insert(
+            unique=False,
+            raise_exception_on_error=True,
+            flags={"InsertFilesInDir": ("-p " + prefix + " " if prefix else "")
+                   + ("-e " + exclude + " " if exclude else "")
+                   + ("-i " + include + " " if include else "")
+                   + ("--force-allow-symlinks " if forceAllowSymlinks else "")
+                   + path})
+
+    totalsize = 0  # collecting total size of all new files
+
+    for f in files:
+        totalsize += f.size
+
+    print("\n\nTOTAL " + str(len(files)) +
+          " NEW files (" + convert_size(totalsize) + ")")
+
+    return
+
+
+def main(argv=None):
+    '''Command line options.'''
+
+    if argv is None:
+        argv = sys.argv
+    else:
+        sys.argv.extend(argv)
+
+    # Setup argument parser
+    parser = ArgumentParser()
+    parser.add_argument("-i", "--include", dest="include",
+                        help="only include paths matching this regex pattern. "
+                        "Note: exclude is given preference over include.",
+                        metavar="RE")
+    parser.add_argument("-e", "--exclude", dest="exclude",
+                        help="exclude paths matching this regex pattern.",
+                        metavar="RE")
+    parser.add_argument("-p", "--prefix", dest="prefix",
+                        help="store files with this prefix into the server's"
+                        " file system.")
+    parser.add_argument("-d", "--dry-run", dest="dryrun", action="store_true",
+                        help="Just simulate the insertion of the files.")
+    parser.add_argument('-t', '--timeout', dest="timeout",
+                        help="timeout in seconds for the database requests. "
+                        "0 means no timeout. [defaults to the global "
+                        "setting: %(default)s]",
+                        metavar="TIMEOUT",
+                        default=db.get_config().get("Connection", "timeout"))
+    parser.add_argument(dest="path",
+                        help="path to folder with source file(s) "
+                        "[default: %(default)s]", metavar="path")
+    parser.add_argument("--force-allow-symlinks", dest="forceAllowSymlinks",
+                        help="Force the processing of symlinks. By default, "
+                        "the server will ignore symlinks in the inserted "
+                        "directory tree.", action="store_true")
+    args = parser.parse_args()
+
+    db.get_connection().timeout = float(args.timeout)
+
+    loadpath(
+        path=args.path,
+        include=args.include,
+        exclude=args.exclude,
+        prefix=args.prefix,
+        dryrun=args.dryrun,
+
+        forceAllowSymlinks=args.forceAllowSymlinks,
+    )
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())