diff --git a/src/caosadvancedtools/loadFiles.py b/src/caosadvancedtools/loadFiles.py index ffeda159c3de97e70098582284254093492325c7..982cb9f3de193a2bc31096a6832122b390c98832 100755 --- a/src/caosadvancedtools/loadFiles.py +++ b/src/caosadvancedtools/loadFiles.py @@ -51,6 +51,9 @@ def convert_size(size): def combine_ignore_files(caosdbignore, localignore, dirname=None): + """appends the contents of localignore to caosdbignore + and saves the result and returns the name + """ tmp = NamedTemporaryFile(delete=False, mode="w", dir=dirname, prefix=".caosdbignore") with open(caosdbignore, "r") as base: tmp.write(base.read()) @@ -61,21 +64,29 @@ def combine_ignore_files(caosdbignore, localignore, dirname=None): def compile_file_list(caosdbignore, localpath): + """creates a list of files + that contain all files under localpath except those excluded by caosdbignore + """ from gitignore_parser import parse_gitignore matches = parse_gitignore(caosdbignore) current_ignore = caosdbignore non_ignored_files = [] ignore_files = [] for root, dirs, files in os.walk(localpath): + # remove local ignore files that do no longer apply to the current subtree (branch switch) while len(ignore_files) > 0 and not root.startswith(ignore_files[-1][0]): shutil.os.remove(ignore_files[-1][1]) ignore_files.pop() + + # use the global one if there are no more local ones if len(ignore_files) > 0: current_ignore = ignore_files[-1][1] matches = parse_gitignore(current_ignore) else: current_ignore = caosdbignore matches = parse_gitignore(current_ignore) + + # create a new local ignore file if ".caosdbignore" in files: current_ignore = combine_ignore_files(current_ignore, os.path.join(root, ".caosdbignore"), @@ -84,6 +95,8 @@ def compile_file_list(caosdbignore, localpath): dirname=root) ignore_files.append((root, current_ignore)) matches = parse_gitignore(current_ignore) + + # actually append files that are not ignored for fi in files: fullpath = os.path.join(root, fi) if not matches(fullpath): @@ -92,6 +105,10 @@ def compile_file_list(caosdbignore, localpath): def create_re_for_file_list(files, localroot, remoteroot): + """creates a regular expression + that matches file paths contained in the files argument and all parent directories. + The prefix localroot is replaced by the prefix remoteroot. + """ regexp = "" for fi in files: path = fi @@ -108,10 +125,8 @@ def loadpath(path, include, exclude, prefix, dryrun, forceAllowSymlinks, caosdbi localpath=None): if caosdbignore: + # create list of files and create regular expression for small chunks filelist = compile_file_list(caosdbignore, localpath) - for fi in filelist: - print(fi) - fulllist = filelist index = 0 @@ -124,8 +139,8 @@ def loadpath(path, include, exclude, prefix, dryrun, forceAllowSymlinks, caosdbi else: includes = [include] + # if no caosdbignore file is used, this iterates over a single include for include in includes: - print(include) if dryrun: logger.info("Performin a dryrun!") files = db.Container().retrieve(