Skip to content
Snippets Groups Projects
Commit d74340b7 authored by Henrik tom Wörden's avatar Henrik tom Wörden
Browse files

MAINT: remove staticmethod definitions, move test file

parent 02a694e8
No related branches found
No related tags found
2 merge requests!53Release 0.1,!9ENH: introduce JSON converter
...@@ -365,14 +365,12 @@ class MarkdownFileConverter(Converter): ...@@ -365,14 +365,12 @@ class MarkdownFileConverter(Converter):
class JSONFileConverter(Converter): class JSONFileConverter(Converter):
@staticmethod def typecheck(self, element: StructureElement):
def typecheck(element: StructureElement): print(type(element))
return isinstance(element, JSONFile) return isinstance(element, File)
def match(self, element: StructureElement): def match(self, element: StructureElement):
if not JSONFileConverter.typecheck(element): if not self.typecheck(element):
# TODO(salexan) Should we be more precise than just raising runtime
# errors here?
raise RuntimeError("Element must be a file") raise RuntimeError("Element must be a file")
m = re.match(self.definition["match"], element.name) m = re.match(self.definition["match"], element.name)
if m is None: if m is None:
...@@ -383,9 +381,8 @@ class JSONFileConverter(Converter): ...@@ -383,9 +381,8 @@ class JSONFileConverter(Converter):
pass pass
return m.groupdict() return m.groupdict()
@staticmethod def create_children(self, generalStore: GeneralStore, element: StructureElement):
def create_children(generalStore: GeneralStore, element: StructureElement): if not self.typecheck(element):
if not JSONFileConverter.typecheck(element):
raise RuntimeError("A JSON file is needed to create children") raise RuntimeError("A JSON file is needed to create children")
with open(element.path, 'r') as json_file: with open(element.path, 'r') as json_file:
json_data = json.load(json_file) json_data = json.load(json_file)
...@@ -398,9 +395,8 @@ class JSONFileConverter(Converter): ...@@ -398,9 +395,8 @@ class JSONFileConverter(Converter):
class DictConverter(Converter): class DictConverter(Converter):
# TODO use Dict as typecheck? # TODO use Dict as typecheck?
@staticmethod def create_children(self, generalStore: GeneralStore, element: StructureElement):
def create_children(generalStore: GeneralStore, element: StructureElement): if not self.typecheck(element):
if not DictConverter.typecheck(element):
raise RuntimeError("A dict is needed to create children") raise RuntimeError("A dict is needed to create children")
children = [] children = []
...@@ -419,8 +415,7 @@ class DictConverter(Converter): ...@@ -419,8 +415,7 @@ class DictConverter(Converter):
return children return children
@staticmethod def typecheck(self, element: StructureElement):
def typecheck(element: StructureElement):
return isinstance(element, Dict) return isinstance(element, Dict)
# TODO use Dict as typecheck? # TODO use Dict as typecheck?
......
...@@ -38,7 +38,8 @@ from test_tool import rfp, dircheckstr ...@@ -38,7 +38,8 @@ from test_tool import rfp, dircheckstr
def test_json(): def test_json():
crawler_definition_path = rfp("jsontest_cfood.yml") crawler_definition_path = rfp("test_directories", "single_file_test_data",
"jsontest_cfood.yml")
json_file_path = rfp("test_directories", "single_file_test_data", "testjson.json") json_file_path = rfp("test_directories", "single_file_test_data", "testjson.json")
crawler = Crawler(debug=True) crawler = Crawler(debug=True)
...@@ -49,12 +50,10 @@ def test_json(): ...@@ -49,12 +50,10 @@ def test_json():
crawler.start_crawling( crawler.start_crawling(
JSONFile(os.path.basename(json_file_path), json_file_path), JSONFile(os.path.basename(json_file_path), json_file_path),
crawler_definition, crawler_definition,
converter_registry, converter_registry
JSONFileConverter
) )
subd = crawler.debug_tree subd = crawler.debug_tree
subc = crawler.debug_metadata subc = crawler.debug_metadata
#print(json.dumps(subd, indent=3)) #print(json.dumps(subd, indent=3))
print(subd) print(subd)
print(subc) print(subc)
lskdjf
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment