diff --git a/src/caosadvancedtools/loadFiles.py b/src/caosadvancedtools/loadFiles.py new file mode 100755 index 0000000000000000000000000000000000000000..953cfc458eba42eeabd1df1dfeaa6c828afc0d0b --- /dev/null +++ b/src/caosadvancedtools/loadFiles.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python +# encoding: utf-8 +# +# ** header v3.0 +# This file is a part of the CaosDB Project. +# +# Copyright (C) 2018 Research Group Biomedical Physics, +# Max-Planck-Institute for Dynamics and Self-Organization Göttingen +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. +# +# ** end header +# + +import math +import sys +from argparse import ArgumentParser + +import caosdb as db + + +def convert_size(size): + if (size == 0): + return '0B' + size_name = ("B", "KB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB") + i = int(math.floor(math.log(size, 1024))) + p = math.pow(1024, i) + s = round(size / p, 2) + + return '%s %s' % (s, size_name[i]) + + +def loadpath(path, include, exclude, prefix, dryrun, forceAllowSymlinks): + + if dryrun is True: + print("DRYRUN") + files = db.Container().retrieve( + unique=False, + raise_exception_on_error=True, + flags={"InsertFilesInDir": ("-p " + prefix + " " if prefix else "") + + ("-e " + exclude + " " if exclude else "") + + ("-i " + include + " " if include else "") + + ("--force-allow-symlinks " if forceAllowSymlinks else "") + + path}) + else: + # new files (inserting them using the insertFilesInDir feature of + # the server, which inserts files via symlinks) + files = db.Container().insert( + unique=False, + raise_exception_on_error=True, + flags={"InsertFilesInDir": ("-p " + prefix + " " if prefix else "") + + ("-e " + exclude + " " if exclude else "") + + ("-i " + include + " " if include else "") + + ("--force-allow-symlinks " if forceAllowSymlinks else "") + + path}) + + totalsize = 0 # collecting total size of all new files + + for f in files: + totalsize += f.size + + print("\n\nTOTAL " + str(len(files)) + + " NEW files (" + convert_size(totalsize) + ")") + + return + + +def main(argv=None): + '''Command line options.''' + + if argv is None: + argv = sys.argv + else: + sys.argv.extend(argv) + + # Setup argument parser + parser = ArgumentParser() + parser.add_argument("-i", "--include", dest="include", + help="only include paths matching this regex pattern. " + "Note: exclude is given preference over include.", + metavar="RE") + parser.add_argument("-e", "--exclude", dest="exclude", + help="exclude paths matching this regex pattern.", + metavar="RE") + parser.add_argument("-p", "--prefix", dest="prefix", + help="store files with this prefix into the server's" + " file system.") + parser.add_argument("-d", "--dry-run", dest="dryrun", action="store_true", + help="Just simulate the insertion of the files.") + parser.add_argument('-t', '--timeout', dest="timeout", + help="timeout in seconds for the database requests. " + "0 means no timeout. [defaults to the global " + "setting: %(default)s]", + metavar="TIMEOUT", + default=db.get_config().get("Connection", "timeout")) + parser.add_argument(dest="path", + help="path to folder with source file(s) " + "[default: %(default)s]", metavar="path") + parser.add_argument("--force-allow-symlinks", dest="forceAllowSymlinks", + help="Force the processing of symlinks. By default, " + "the server will ignore symlinks in the inserted " + "directory tree.", action="store_true") + args = parser.parse_args() + + db.get_connection().timeout = float(args.timeout) + + loadpath( + path=args.path, + include=args.include, + exclude=args.exclude, + prefix=args.prefix, + dryrun=args.dryrun, + + forceAllowSymlinks=args.forceAllowSymlinks, + ) + + return 0 + + +if __name__ == "__main__": + sys.exit(main())