Merge branch 'dev' into f-fix-crawler-overwrite

CHANGELOG.md

@@ -11,6 +11,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed ###
 
+- `_AbstractDictElementConverter` uses `re.DOTALL` for `match_value`
+
 ### Deprecated ###
 
 ### Removed ###

src/caoscrawler/converters.py

@@ -638,7 +638,7 @@ class _AbstractDictElementConverter(Converter):
         m1 = re.match(self.definition["match_name"], element.name)
         if m1 is None:
             return None
-        m2 = re.match(self.definition["match_value"], str(element.value))
+        m2 = re.match(self.definition["match_value"], str(element.value), re.DOTALL)
         if m2 is None:
             return None
         values = dict()

src/doc/macros.rst

@@ -56,15 +56,43 @@ The same version using cfood macros could be defined as follows:
       - name: README
         filename: ^README.md$
 
-
 The "MarkdownFile" key and its value will be replaced by everything that is 
 given below "definition" in the Macro.
+        
+The expanded version of `ExperimentalData` will look like:
+
+.. _example_files_2_expanded:
+.. code-block:: yaml
 
+  ExperimentalData:
+    match: ExperimentalData
+    subtree:
+      README_filename:
+        match: ^README.md$
+        records:
+          README:
+            file: README_filename
+            parents:
+            - MarkdownFile
+            path: README_filename
+            role: File
+        type: SimpleFile
+    type: Directory
+
+This :ref:`example<_example_files_2>` can also be found in the macro unit tests (see :func:`unittests.test_macros.test_documentation_example_2`).
 
 
 Complex Example
 ===============
 
+The following, more complex example, demonstrates the use
+of macro variable substitutions that generate crawler variable substitutions:
+
+- `$$$nodename` will lead to a macro variable substitution of variable `$nodename` during macro expansion.
+- `$$` will be turned into `$`
+- So in the crawler cfood, the string will appear as `$value` if variable `nodename` would be set to `value` when using the macro.
+
+
 .. _example_1:
 .. code-block:: yaml
    
@@ -88,3 +116,118 @@ Complex Example
                file: $$$nodename
              Simulation:
                $recordtype: +$File
+
+The expanded version of :ref:`example<_example_1>` can be seen in :ref:`example<_example_1_expanded>`.
+
+
+.. _example_1_expanded:
+.. code-block:: yaml
+                
+  SimulationData:
+    match: SimulationData
+    subtree:
+      Dataset:
+        match: .*
+        records:
+          File:
+            file: $Dataset
+            parents:
+            - DatasetFile
+            path: $Dataset
+            role: File
+          Simulation:
+            DatasetFile: +$File
+        type: SimpleFile
+    type: Directory
+
+This :ref:`example<_example_1>` can also be found in the macro unit tests (see :func:`unittests.test_macros.test_documentation_example_1`).
+
+
+
+Using Macros Multiple Times
+===========================
+
+To use the same macro multiple times in the same yaml node, lists can be used:
+
+.. _example_multiple:
+.. code-block:: yaml
+
+    ---
+    metadata:
+      macros:
+        - !defmacro
+          name: test_twice
+          params:
+            macro_name: default_name
+            a: 4
+          definition:
+            $macro_name:
+              something:
+                a: $a
+    ---
+    extroot: !macro
+      test_twice:
+      - macro_name: once  # <- This is the first replacement of the macro
+      - macro_name: twice # <- This is the second one, with different arguments
+        a: 5
+      - {}                # <- This is the third one, just using default arguments
+
+        
+This :ref:`example<_example_multiple>` is taken from the macro unit tests (see :func:`unittests.test_macros.test_use_macro_twice`).
+
+The example will be expanded to:
+
+.. _example_multiple_expanded:
+.. code-block:: yaml
+
+    extroot:
+      default_name:
+        something:
+          a: '4'
+      once:
+        something:
+          a: '4'
+      twice:
+        something:
+          a: '5'
+
+
+
+
+Limitation
+----------
+
+Currently it is not possible to use the same macro twice in the same yaml node, but in different
+positions. Consider:
+
+.. _example_multiple_limitation:
+.. code-block:: yaml
+
+    ---
+    metadata:
+      macros:
+        - !defmacro
+          name: test_twice
+          params:
+            macro_name: default_name
+            a: 4
+          definition:
+            $macro_name:
+              something:
+                a: $a
+    ---
+    extroot: !macro
+      test_twice:
+      - macro_name: once  # <- This is the first replacement of the macro
+        
+      Other_node:
+        type: test
+        
+      test_twice:  # This is NOT possible as each
+                   #  dictionary element can only appear once in a yaml node.
+      - macro_name: twice # <- This is the second one, with different arguments
+        a: 5
+      - {}                # <- This is the third one, just using default arguments
+
+However, this should not be a real limitation, as the crawler is designed in a way,
+that the order of the nodes in the same level should not matter.

unittests/test_converters.py

@@ -32,6 +32,7 @@ from caoscrawler.converters import (Converter, ConverterValidationError,
                                     DictConverter, DirectoryConverter,
                                     handle_value, MarkdownFileConverter,
                                     JSONFileConverter)
+from caoscrawler.converters import _AbstractDictElementConverter
 from caoscrawler.crawl import Crawler
 from caoscrawler.stores import GeneralStore
 from caoscrawler.structure_elements import (File, DictTextElement,
@@ -385,3 +386,21 @@ MyElement:
         list(yaml.safe_load_all(two_doc_yaml)))
     assert "MyElement" in two_doc_definitions
     assert two_doc_definitions["MyElement"]["type"] == one_doc_definitions["MyElement"]["type"]
+
+
+def test_abstract_dict_element_converter():
+    definition = yaml.safe_load("""
+match_name: text
+match_value: .*begin(?P<text>.*)end
+    """)
+    converter = _AbstractDictElementConverter(
+        definition, "test_converter",
+        None  # This is possible when "subtree" is not used
+    )
+    element = DictTextElement("text", """
+begin
+bla
+end""")
+    val = converter.match(element)
+    assert val is not None
+    assert val["text"] == "\nbla\n"

unittests/test_directories/examples_article/SimulationData/2020_climate-model-predict/2020-02-01/README.md

 ---
 responsible: AuthorE
-description: >
+description: >-
   Code for fitting the predictive model to the
   training data and for predicting the average
   annual temperature for all measurement stations

unittests/test_macros.py

@@ -295,7 +295,6 @@ extroot3: !macro
     assert cfood["extroot3"]["test_four"] is None
 
 
-# @pytest.mark.xfail(reason="Fix multiple usage of the same macro.")
 def test_use_macro_twice():
     """Test that the same macro can be used twice with different parameters in
     the same CFood element if the name depends on the parameters.
@@ -328,3 +327,85 @@ extroot: !macro
     assert cfood["extroot"]["once"]["something"]["a"] == "4"
     assert cfood["extroot"]["twice"]["something"]["a"] == "5"
     assert cfood["extroot"]["default_name"]["something"]["a"] == "4"
+
+    # Code sample to generate the expanded macro:
+    # with open("expanded_test_macro.yaml", "w") as f:
+    #     f.write(yaml.dump(cfood))
+
+
+def test_documentation_example_2():
+
+    cfood = _temp_file_load("""
+---
+metadata:
+  macros:
+  - !defmacro
+    name: MarkdownFile
+    params:
+      name: null
+      filename: null
+    definition:
+      ${name}_filename:
+        type: SimpleFile
+        match: $filename
+        records:
+          $name:
+            parents:
+            - MarkdownFile
+            role: File
+            path: ${name}_filename
+            file: ${name}_filename
+---
+ExperimentalData:
+  type: Directory
+  match: ExperimentalData
+  subtree: !macro
+    MarkdownFile:
+    - name: README
+      filename: ^README.md$
+    """)
+
+    # Code sample to generate the expanded macro:
+    # with open("expanded_test_macro.yaml", "w") as f:
+    #     f.write(yaml.dump(cfood))
+
+
+def test_documentation_example_1():
+
+    cfood = _temp_file_load("""
+---
+metadata:
+  macros:
+  - !defmacro
+    name: SimulationDatasetFile
+    params:
+      match: null
+      recordtype: null
+      nodename: null
+    definition:
+      $nodename:
+        match: $match
+        type: SimpleFile
+        records:
+          File:
+            parents:
+            - $recordtype
+            role: File
+            path: $$$nodename
+            file: $$$nodename
+          Simulation:
+            $recordtype: +$File
+---
+SimulationData:
+  type: Directory
+  match: SimulationData
+  subtree: !macro
+    SimulationDatasetFile:
+    - match: .*
+      recordtype: DatasetFile
+      nodename: Dataset
+    """)
+
+    # Code sample to generate the expanded macro:
+    # with open("expanded_test_macro.yaml", "w") as f:
+    #     f.write(yaml.dump(cfood))