From ecabc99f56c0b9cf67aaf1c7b66f1d9ca0b4581a Mon Sep 17 00:00:00 2001
From: Alexander Schlemmer <alexander@mail-schlemmer.de>
Date: Thu, 22 Sep 2022 20:10:28 +0200
Subject: [PATCH] ENH: crawler definition may now contain up to two documents
---
src/caoscrawler/crawl.py | 11 ++++++++++-
1 file changed, 10 insertions(+), 1 deletion(-)
diff --git a/src/caoscrawler/crawl.py b/src/caoscrawler/crawl.py
index d9e23a69..be3305e8 100644
--- a/src/caoscrawler/crawl.py
+++ b/src/caoscrawler/crawl.py
@@ -230,7 +230,16 @@ class Crawler(object):
# Load the cfood from a yaml file:
with open(crawler_definition_path, "r") as f:
- crawler_definition = yaml.safe_load(f)
+ crawler_definitions = list(yaml.safe_load_all(f))
+
+ if len(crawler_definitions) == 1:
+ # Simple case, just one document:
+ crawler_definition = crawler_definitions[0]
+ elif len(crawler_definition) == 2:
+ crawler_definition = crawler_definitions[1]
+ else:
+ raise RuntimeError("Crawler definition must not contain more than two documents.")
+
# TODO: at this point this function can already load the cfood schema extensions
# from the crawler definition and add them to the yaml schema that will be
--
GitLab