diff --git a/.docker/Dockerfile b/.docker/Dockerfile
index 876f252299991f2fa4410994b73259c3593c2198..5b26c03c927780e804f190ee6c8dee1e6813a18e 100644
--- a/.docker/Dockerfile
+++ b/.docker/Dockerfile
@@ -25,7 +25,7 @@ ADD https://gitlab.com/api/v4/projects/13656973/repository/branches/dev \
RUN git clone https://gitlab.com/caosdb/caosdb-pylib.git && \
cd caosdb-pylib && git checkout dev && pip3 install .
# At least recommonmark 0.6 required.
-RUN pip3 install -U html2text pycodestyle pylint recommonmark sphinx-rtd-theme
+RUN pip3 install -U html2text pycodestyle pylint recommonmark sphinx-rtd-theme gitignore-parser
COPY . /git
RUN rm -r /git/.git \
&& mv /git/.docker/pycaosdb.ini /git/integrationtests
diff --git a/CHANGELOG.md b/CHANGELOG.md
index abca90fe2f0dc1ebc5a22b7aa83f182b6ff7d280..f88538c8b1d4bff0109f68847430aeb53e7a01e9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,6 +10,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Unauthorized inserts can now be cached. Note that the Crawler cannot postpone
inserts but the Cache has the functionality now.
+- caosdbignore; You can add one or more `.caosdbignore` files to the directory
+ structure that you want to make available in CaosDB and the run loadFiles.
+ The syntax is that of `.gitignore` files. For more information see `loadFiles`
+ section of the Crawler in the documentation.
### Changed ###
diff --git a/integrationtests/caosdbignore b/integrationtests/caosdbignore
new file mode 100644
index 0000000000000000000000000000000000000000..84b3636238e655e9ea7569ca3989f0ca73abb27c
--- /dev/null
+++ b/integrationtests/caosdbignore
@@ -0,0 +1,2 @@
+lol
+~README.md
diff --git a/integrationtests/extroot/Software/2010_TestSoftware/2019-02-03_v0.1/~README.md b/integrationtests/extroot/Software/2010_TestSoftware/2019-02-03_v0.1/~README.md
new file mode 100644
index 0000000000000000000000000000000000000000..f2e41136eac73c39554dede1fd7e67b12502d577
--- /dev/null
+++ b/integrationtests/extroot/Software/2010_TestSoftware/2019-02-03_v0.1/~README.md
@@ -0,0 +1 @@
+stuff
diff --git a/integrationtests/extroot/Software/2010_TestSoftware/lol b/integrationtests/extroot/Software/2010_TestSoftware/lol
new file mode 100644
index 0000000000000000000000000000000000000000..afd4a162a20a53aaeebf053a257f147759a52d98
--- /dev/null
+++ b/integrationtests/extroot/Software/2010_TestSoftware/lol
@@ -0,0 +1,2 @@
+lol
+
diff --git a/integrationtests/filldb.sh b/integrationtests/filldb.sh
index 9f55365eb595537b43caa9b197c8bc31ea1e69cb..9ee9060e5078b5d48ea5a9972b8ade119378dbc6 100755
--- a/integrationtests/filldb.sh
+++ b/integrationtests/filldb.sh
@@ -5,7 +5,7 @@ python3 -m caosadvancedtools.loadFiles /opt/caosdb/mnt/extroot/ExperimentalData
python3 -m caosadvancedtools.loadFiles /opt/caosdb/mnt/extroot/DataAnalysis
python3 -m caosadvancedtools.loadFiles /opt/caosdb/mnt/extroot/SimulationData
python3 -m caosadvancedtools.loadFiles /opt/caosdb/mnt/extroot/Publications
-python3 -m caosadvancedtools.loadFiles /opt/caosdb/mnt/extroot/Software
+python3 -m caosadvancedtools.loadFiles -c caosdbignore -l $PWD/extroot/Software /opt/caosdb/mnt/extroot/Software
python3 insert_model.py
python3 insert_some.py
python3 crawl.py /
diff --git a/integrationtests/test_crawler_with_cfoods.py b/integrationtests/test_crawler_with_cfoods.py
index 4efef87cef52e4a2a20a615afe210c32f52a276a..19b1f8ff10e365031d6940c7b904e3656eda2861 100755
--- a/integrationtests/test_crawler_with_cfoods.py
+++ b/integrationtests/test_crawler_with_cfoods.py
@@ -33,6 +33,15 @@ def get_entity_with_id(eid):
return db.execute_query("FIND "+str(eid), unique=True)
+class LoadFilesTest(unittest.TestCase):
+ def test_lol(self):
+ # check whether ignored files were insered
+ res = db.execute_query("FIND FILE WHICH IS STORED AT '**/lol'")
+ assert len(res) == 0
+ res = db.execute_query("FIND FILE WHICH IS STORED AT '**/~README.md'")
+ assert len(res) == 0
+
+
class CrawlerTest(unittest.TestCase):
def test_experiment(self):
diff --git a/setup.py b/setup.py
index e5803191c2311e9018b164bc2632854106dcd4ae..39d806c1dcd4abca6f6f2d756d7b9f1e41c4ba95 100755
--- a/setup.py
+++ b/setup.py
@@ -162,6 +162,7 @@ def setup_package():
"xlrd>=2.0",
],
extras_require={"h5-crawler": ["h5py>=3.3.0", ],
+ "gitignore-parser ": ["gitignore-parser >=0.1.0", ],
},
packages=find_packages('src'),
package_dir={'': 'src'},
diff --git a/src/caosadvancedtools/loadFiles.py b/src/caosadvancedtools/loadFiles.py
index 445374b3b3bddf94eefd7952e190bf52155765a8..210732aa2bd31b7f493af263df24cc3e7a56611b 100755
--- a/src/caosadvancedtools/loadFiles.py
+++ b/src/caosadvancedtools/loadFiles.py
@@ -25,10 +25,14 @@
import argparse
import logging
+import os
import math
import sys
+import re
from argparse import ArgumentParser
+from tempfile import NamedTemporaryFile
+import shutil
import caosdb as db
logger = logging.getLogger(__name__)
@@ -46,37 +50,135 @@ def convert_size(size):
return '%s %s' % (s, size_name[i])
-def loadpath(path, include, exclude, prefix, dryrun, forceAllowSymlinks):
+def combine_ignore_files(caosdbignore, localignore, dirname=None):
+ """appends the contents of localignore to caosdbignore and saves the result
+ and returns the name
- if dryrun:
- logger.info("Performin a dryrun!")
- files = db.Container().retrieve(
- unique=False,
- raise_exception_on_error=True,
- flags={"InsertFilesInDir": ("-p " + prefix + " " if prefix else "")
- + ("-e " + exclude + " " if exclude else "")
- + ("-i " + include + " " if include else "")
- + ("--force-allow-symlinks " if forceAllowSymlinks else "")
- + path})
+ """
+
+ tmp = NamedTemporaryFile(delete=False, mode="w",
+ dir=dirname, prefix=".caosdbignore")
+ with open(caosdbignore, "r") as base:
+ tmp.write(base.read())
+ with open(localignore, "r") as local:
+ tmp.write(local.read())
+ tmp.close()
+ return tmp.name
+
+
+def compile_file_list(caosdbignore, localpath):
+ """creates a list of files that contain all files under localpath except
+ those excluded by caosdbignore
+
+ """
+
+ from gitignore_parser import parse_gitignore
+ matches = parse_gitignore(caosdbignore)
+ current_ignore = caosdbignore
+ non_ignored_files = []
+ ignore_files = []
+ for root, dirs, files in os.walk(localpath):
+ # remove local ignore files that do no longer apply to the current subtree (branch switch)
+ while len(ignore_files) > 0 and not root.startswith(ignore_files[-1][0]):
+ shutil.os.remove(ignore_files[-1][1])
+ ignore_files.pop()
+
+ # use the global one if there are no more local ones
+ if len(ignore_files) > 0:
+ current_ignore = ignore_files[-1][1]
+ matches = parse_gitignore(current_ignore)
+ else:
+ current_ignore = caosdbignore
+ matches = parse_gitignore(current_ignore)
+
+ # create a new local ignore file
+ if ".caosdbignore" in files:
+ current_ignore = combine_ignore_files(current_ignore,
+ os.path.join(
+ root, ".caosdbignore"),
+ # due to the logic of gitignore_parser the file
+ # has to be written to this folder
+ dirname=root)
+ ignore_files.append((root, current_ignore))
+ matches = parse_gitignore(current_ignore)
+
+ # actually append files that are not ignored
+ for fi in files:
+ fullpath = os.path.join(root, fi)
+ if not matches(fullpath):
+ non_ignored_files.append(fullpath)
+ return non_ignored_files
+
+
+def create_re_for_file_list(files, localroot, remoteroot):
+ """creates a regular expression that matches file paths contained in the
+ files argument and all parent directories. The prefix localroot is replaced
+ by the prefix remoteroot.
+
+ """
+ regexp = ""
+ for fi in files:
+ path = fi
+ reg = ""
+ while path != localroot and path != "/" and path != "":
+ print(path, localroot)
+ reg = "(/"+re.escape(os.path.basename(path)) + reg + ")?"
+ path = os.path.dirname(path)
+ regexp += "|" + re.escape(remoteroot) + reg
+ return "^("+regexp[1:]+")$"
+
+
+def loadpath(path, include, exclude, prefix, dryrun, forceAllowSymlinks, caosdbignore=None,
+ localpath=None):
+
+ if caosdbignore:
+ # create list of files and create regular expression for small chunks
+ filelist = compile_file_list(caosdbignore, localpath)
+ fulllist = filelist
+
+ index = 0
+ step_size = 3
+ includes = []
+ while index < len(fulllist):
+ subset = fulllist[index:min(index+step_size, len(fulllist))]
+ includes.append(create_re_for_file_list(subset, localpath, path))
+ index += step_size
else:
- # new files (inserting them using the insertFilesInDir feature of
- # the server, which inserts files via symlinks)
- files = db.Container().insert(
- unique=False,
- raise_exception_on_error=True,
- flags={"InsertFilesInDir": ("-p " + prefix + " " if prefix else "")
- + ("-e " + exclude + " " if exclude else "")
- + ("-i " + include + " " if include else "")
- + ("--force-allow-symlinks " if forceAllowSymlinks else "")
- + path})
+ includes = [include]
- totalsize = 0 # collecting total size of all new files
+ # if no caosdbignore file is used, this iterates over a single include
+ for include in includes:
+ if dryrun:
+ logger.info("Performin a dryrun!")
+ files = db.Container().retrieve(
+ unique=False,
+ raise_exception_on_error=True,
+ flags={"InsertFilesInDir": ("-p " + prefix + " " if prefix else "")
+ + ("-e " + exclude + " " if exclude else "")
+ + ("-i " + include + " " if include else "")
+ + ("--force-allow-symlinks " if forceAllowSymlinks else "")
+ + path})
+ else:
+ # new files (inserting them using the insertFilesInDir feature of
+ # the server, which inserts files via symlinks)
+ files = db.Container().insert(
+ unique=False,
+ raise_exception_on_error=True,
+ flags={"InsertFilesInDir": ("-p " + prefix + " " if prefix else "")
+ + ("-e " + exclude + " " if exclude else "")
+ + ("-i " + include + " " if include else "")
+ + ("--force-allow-symlinks " if forceAllowSymlinks else "")
+ + path})
- for f in files:
- totalsize += f.size
+ totalsize = 0 # collecting total size of all new files
- logger.info("Made in total {} new files with a combined size of {} "
- "accessible.".format(len(files), convert_size(totalsize)))
+ for f in files:
+ totalsize += f.size
+
+ print("Made in total {} new files with a combined size of {} "
+ "accessible.".format(len(files), convert_size(totalsize)))
+ logger.info("Made in total {} new files with a combined size of {} "
+ "accessible.".format(len(files), convert_size(totalsize)))
return
@@ -91,6 +193,18 @@ def main(argv=None):
# Setup argument parser
parser = ArgumentParser()
+ parser.add_argument("-p", "--prefix", dest="prefix",
+ help="store files with this prefix into the server's"
+ " file system.")
+ parser.add_argument("-c", "--caosdbignore", help="""
+Path to a caosdbignore file that defines which files shall be included and which do not.
+The syntax is the same as in a gitignore file. You must also provide the localpath option
+since the check is done locally.
+"""
+ )
+ parser.add_argument("-l", "--localpath", help="Path to the root directory on this machine. "
+ "This is needed if a caosdbignore file is used since the check is done "
+ "locally")
parser.add_argument("-i", "--include", dest="include",
help="""
only include paths matching this regex pattern.
@@ -104,9 +218,6 @@ exclude is given preference over include.
parser.add_argument("-e", "--exclude", dest="exclude",
help="exclude paths matching this regex pattern.",
metavar="RE")
- parser.add_argument("-p", "--prefix", dest="prefix",
- help="store files with this prefix into the server's"
- " file system.")
parser.add_argument("-d", "--dry-run", dest="dryrun", action="store_true",
help="Just simulate the insertion of the files.")
parser.add_argument('-t', '--timeout', dest="timeout",
@@ -127,6 +238,17 @@ exclude is given preference over include.
"directory tree.", action="store_true")
args = parser.parse_args()
+ if args.caosdbignore and (args.exclude or args.include):
+ raise ValueError(
+ "Do not use a caosdbignore file and in- or exclude simultaneously!")
+
+ if args.caosdbignore and not args.localpath:
+ raise ValueError("To use caosdbignore you must supply a local path!")
+
+ if args.localpath and (args.exclude or args.include):
+ raise ValueError(
+ "Do not use a localpath and in- or exclude simultaneously!")
+
con = db.get_connection()
con.timeout = float(args.timeout)
con._login()
@@ -137,8 +259,9 @@ exclude is given preference over include.
exclude=args.exclude,
prefix=args.prefix,
dryrun=args.dryrun,
-
forceAllowSymlinks=args.forceAllowSymlinks,
+ caosdbignore=args.caosdbignore,
+ localpath=args.localpath,
)
return 0
diff --git a/src/doc/crawler.rst b/src/doc/crawler.rst
index 4b99c97e6db16e5691f373fa5fb4903e4d078155..aea023192c0b95c784d5ac91ade12d0c30591d42 100644
--- a/src/doc/crawler.rst
+++ b/src/doc/crawler.rst
@@ -121,6 +121,14 @@ as seen by the CaosDB server (The actual path may vary. This is the used
in the LinkAhead distribution of CaosDB). In this case the root file
system as seen from within the CaosDB docker process is used.
+
+You can provide a ``.caosdbignore`` file as a commandline option to the above
+loadFiles command. The syntax of that file is the same as for `gitignore
+<https://git-scm.com/docs/gitignore>`_ files. Note, that you can have additional
+``.caosdbignore`` files at lower levels which are appended to the current ignore
+file and have an effect of the respective subtree.
+
+
Extending the Crawlers
======================
diff --git a/tox.ini b/tox.ini
index dde34b987b9b08bfdfc51a06dd46a9a0e0494f28..ddfb4714f6b86d93e9567fe3c1410f2a1ce965cf 100644
--- a/tox.ini
+++ b/tox.ini
@@ -7,6 +7,7 @@ deps=nose
git+https://gitlab.indiscale.com/caosdb/src/caosdb-pylib.git@dev
pytest
pytest-cov
+ gitignore-parser
openpyxl
xlrd == 1.2
h5py
diff --git a/unittests/caosdbignore-example b/unittests/caosdbignore-example
new file mode 100644
index 0000000000000000000000000000000000000000..04500046def6f8d1a7efb1cdceeff7e85f8e1f91
--- /dev/null
+++ b/unittests/caosdbignore-example
@@ -0,0 +1,2 @@
+README.md
+.caosdbignore*
diff --git a/unittests/data/Publications/Posters/2019-02-03_something/.caosdbignore b/unittests/data/Publications/Posters/2019-02-03_something/.caosdbignore
new file mode 100644
index 0000000000000000000000000000000000000000..43b9cb201ed2ae7e193a1755d4e935a8d7b17a23
--- /dev/null
+++ b/unittests/data/Publications/Posters/2019-02-03_something/.caosdbignore
@@ -0,0 +1 @@
+!README.md
diff --git a/unittests/test_caosdbignore.py b/unittests/test_caosdbignore.py
new file mode 100644
index 0000000000000000000000000000000000000000..9394bf0c8b177bc6df132ee0b43dcb2753d70f28
--- /dev/null
+++ b/unittests/test_caosdbignore.py
@@ -0,0 +1,68 @@
+#!/usr/bin/env python3
+# encoding: utf-8
+#
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2022 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2022 Henrik tom Wörden <h.tomwoerden@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+
+
+import os
+import re
+from tempfile import NamedTemporaryFile
+import unittest
+
+from caosadvancedtools.loadFiles import compile_file_list, create_re_for_file_list, combine_ignore_files
+
+BASEDIR = os.path.dirname(os.path.realpath(__file__))
+
+
+class Caosdbignore(unittest.TestCase):
+ def setUp(self):
+ pass
+
+ def test_compile(self):
+ files = compile_file_list(os.path.join(BASEDIR, "caosdbignore-example"),
+ os.path.join(BASEDIR, "data"))
+ assert len(files) == 3
+ assert os.path.join(BASEDIR, "data", "datatypes.xlsx") in files
+ assert os.path.join(BASEDIR, "data", "README.xlsx") in files
+ assert os.path.join(BASEDIR, "data", "Publications/Posters/2019-02-03_something/README.md") in files
+
+ def test_regex(self):
+ files = [r"/dies/ist/simple", r"/dies/eh(er)/nich?t"]
+ regex = create_re_for_file_list(files, "/dies", "/dies")
+ assert re.match(regex, files[0]) is not None
+ assert re.match(regex, files[1]) is not None
+ assert re.match(regex, "/dies/ist") is not None
+ assert re.match(regex, "/die") is None
+ assert re.match(regex, files[0]+files[1]) is None
+ assert re.match(regex, "d") is None
+
+ def test_combine(self):
+ fi1 = NamedTemporaryFile(delete=False, mode="w")
+ fi1.write("ha")
+ fi1.close()
+ fi2 = NamedTemporaryFile(delete=False, mode="w")
+ fi2.write("ha")
+ fi2.close()
+ fi_new = combine_ignore_files(fi1.name, fi2.name)
+ with open(fi_new, "r") as fi:
+ assert "haha" == fi.read()