From 5956b2cb8a6d98e1272f000d9828f97dbf184220 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Henrik=20tom=20W=C3=B6rden?= <h.tomwoerden@indiscale.com>
Date: Sat, 17 Aug 2019 13:23:42 +0000
Subject: [PATCH] Reversed the order of iteration when crawling
---
.docker/Dockerfile | 4 ++--
src/caosadvancedtools/crawler.py | 30 +++++++++++++++++-------------
2 files changed, 19 insertions(+), 15 deletions(-)
diff --git a/.docker/Dockerfile b/.docker/Dockerfile
index 65b05d3e..c6867939 100644
--- a/.docker/Dockerfile
+++ b/.docker/Dockerfile
@@ -7,9 +7,9 @@ RUN git clone https://gitlab.com/caosdb/caosdb-pylib.git && \
cd caosdb-pylib && pip3 install .
RUN git clone https://gitlab.com/caosdb/caosdb-models.git && \
cd caosdb-models && pip3 install .
-ADD https://gitlab.com/api/v4/projects/13601752/repository/branches/project_cfood \
+ADD https://gitlab.com/api/v4/projects/13601752/repository/branches/master \
scifolder_version.json
-RUN git clone -b project_cfood \
+RUN git clone \
https://gitlab.com/henrik_indiscale/scifolder.git && \
cd scifolder && pip3 install .
COPY . /git
diff --git a/src/caosadvancedtools/crawler.py b/src/caosadvancedtools/crawler.py
index 04b941d9..0a580592 100644
--- a/src/caosadvancedtools/crawler.py
+++ b/src/caosadvancedtools/crawler.py
@@ -79,22 +79,21 @@ class Crawler(object):
print("-"*60)
print("Matching files against CFoods")
- for crawled_file in files:
+ for Cfood in self.food:
if self.verbosity >= VERBOSE:
- print("Matching {}...".format(crawled_file.path))
- # if crawled_file.size == 0:
- # crawled_file.add_message(
- # type="Warning", description="This file is empty. Shouldn't we delete it?")
- # self.report.append(crawled_file)
+ print("Matching against {}...".format(Cfood.__name__))
- # continue
+ for crawled_file in files:
+ if self.verbosity >= DEBUG:
+ print("Matching {}...".format(crawled_file.path))
- for Cfood in self.food:
if Cfood.match(crawled_file.path) is not None:
matches[crawled_file.path] += 1
if self.verbosity >= VERBOSE:
- print("{} matched.".format(Cfood.__name__))
+ print("{} matched\n{}.".format(
+ Cfood.__class__.__name__,
+ crawled_file.path))
try:
cfoods.append(Cfood(crawled_file, access=self.access,
verbosity=self.verbosity))
@@ -106,14 +105,19 @@ class Crawler(object):
print("-"*60)
print("Trying to attach files to created CFoods")
- for crawled_file in files:
+ for cfood in cfoods:
if self.verbosity >= VERBOSE:
- print("Matching {}...".format(crawled_file.path))
+ print("Matching against {}...".format(Cfood.__name__))
+
+ for crawled_file in files:
+ if self.verbosity >= DEBUG:
+ print("Matching {}...".format(crawled_file.path))
- for cfood in cfoods:
if cfood.looking_for(crawled_file):
if self.verbosity >= VERBOSE:
- print("{} matched.".format(cfood.__class__.__name__))
+ print("{} matched\n{}.".format(
+ Cfood.__class__.__name__,
+ crawled_file.path))
cfood.attach(crawled_file)
matches[crawled_file.path] += 1
--
GitLab