diff --git a/src/caoscrawler/converters.py b/src/caoscrawler/converters.py index e3f72b10ce2694853d6bc0644c736f0d621ed881..697bd5e4d66b0babb699b8a5ca0606bb94bd3fd3 100644 --- a/src/caoscrawler/converters.py +++ b/src/caoscrawler/converters.py @@ -35,7 +35,7 @@ from .structure_elements import (StructureElement, Directory, File, Dict, JSONFi DictIntegerElement, DictBooleanElement, DictFloatElement, DictDictElement, TextElement, DictTextElement, DictElement, DictListElement) -from typing import Optional, Union +from typing import List, Optional, Union from abc import abstractmethod from string import Template import yaml_header_tools @@ -301,6 +301,11 @@ class Converter(object): records, self.definition["records"]) + def filter_children(self, children_with_strings: List[], expr, groupname, rule): + """Filter children according to regexp `expr` and `rule`.""" + + pass + @abstractmethod def typecheck(self, element: StructureElement): pass @@ -675,7 +680,8 @@ class TableConverter(Converter): # The option can often either be a single value or a list of values. # In the latter case each element of the list will be converted to the defined type. if isinstance(el, list): - option_dict[opt_name] = [opt_conversion(el_el) for el_el in el] + option_dict[opt_name] = [ + opt_conversion(el_el) for el_el in el] else: option_dict[opt_name] = opt_conversion(el) return option_dict diff --git a/unittests/test_converters.py b/unittests/test_converters.py index 7a6987b8b3fae9d747f2440de202df5d10a34cc0..0be8dbe7e5782c9f3889095d085df59246e27c4e 100644 --- a/unittests/test_converters.py +++ b/unittests/test_converters.py @@ -272,3 +272,24 @@ def test_variable_replacement(): assert handle_value(["a", "b"], values) == (["a", "b"], "single") assert handle_value(["$a", "$b"], values) == (["4", "68"], "single") + + +def test_filter_children_of_directory(converter_registry): + """Verify that children (i.e., files) in a directory are filtered or sorted + correctly. + + """ + test_dir = Directory("examples_filter_children", rfp("test_directories", "examples_filter_children")) + + dc = DirectoryConverter( + defnition={ + "match": "(.*)", + "filter": { + "expr": "", + "group": "date", + "filter": "only_max" + }, + name="TestOnlyMaxDirectoryConverter", + converter_registry=converter_registry + } + )