From 017e244687902dec50ac622082cd7edcb4bc1a58 Mon Sep 17 00:00:00 2001 From: florian <f.spreckelsen@inidscale.com> Date: Tue, 17 May 2022 13:41:33 +0200 Subject: [PATCH] MAINT: Rename module in source files --- integrationtests/basic_example/test.py | 19 +++++++---- .../realworld_example/test_dataset_crawler.py | 11 +++--- setup.cfg | 4 +-- src/caoscrawler/crawl.py | 28 +++++++-------- src/doc/conf.py | 12 +++---- unittests/test_cache.py | 2 +- unittests/test_converters.py | 34 +++++++++---------- unittests/test_entity_comparison.py | 11 +++--- unittests/test_file_identifiables.py | 5 +-- unittests/test_identifiable_adapters.py | 2 +- unittests/test_identified_cache.py | 5 +-- unittests/test_json.py | 6 ++-- unittests/test_schema.py | 2 +- unittests/test_tool.py | 8 ++--- unittests/test_tool_extended.py | 8 ++--- unittests/test_variable_substitutions.py | 8 ++--- 16 files changed, 88 insertions(+), 77 deletions(-) diff --git a/integrationtests/basic_example/test.py b/integrationtests/basic_example/test.py index 35ee9918..6e35f7f2 100755 --- a/integrationtests/basic_example/test.py +++ b/integrationtests/basic_example/test.py @@ -33,9 +33,9 @@ from caosdb import EmptyUniqueQueryError import argparse import sys from argparse import RawTextHelpFormatter -from newcrawler import Crawler +from caoscrawler import Crawler import caosdb as db -from newcrawler.identifiable_adapters import CaosDBIdentifiableAdapter +from caoscrawler.identifiable_adapters import CaosDBIdentifiableAdapter import pytest from caosadvancedtools.models.parser import parse_model_from_yaml import yaml @@ -105,7 +105,8 @@ def crawler_extended(ident): updateList = cr.updateList fileList = [r for r in updateList if r.role == "File"] for f in fileList: - f.file = rfp("..", "unittests", "test_directories", "examples_article", f.file) + f.file = rfp("..", "unittests", "test_directories", + "examples_article", f.file) return cr @@ -201,7 +202,8 @@ def test_identifiable_update(clear_database, usemodel, ident, crawler): if (record.parents[0].name == "Measurement" and record.get_property("date").value == "2020-01-03"): # maybe a bit weird, but add an email address to a measurement - record.add_property(name="email", value="testperson@testaccount.test") + record.add_property( + name="email", value="testperson@testaccount.test") print("one change") break ins, ups = cr.synchronize() @@ -240,7 +242,8 @@ def test_identifiable_update(clear_database, usemodel, ident, crawler): def test_file_insertion_dry(clear_database, usemodel, ident): crawler_extended = Crawler(debug=True, identifiableAdapter=ident) - crawl_standard_test_directory(crawler_extended, cfood="scifolder_extended.yml") + crawl_standard_test_directory( + crawler_extended, cfood="scifolder_extended.yml") updateList = crawler_extended.updateList fileList = [r for r in updateList if r.role == "File"] assert len(fileList) == 11 @@ -281,7 +284,8 @@ def test_file_update(clear_database, usemodel, ident, crawler_extended): updateList = cr.updateList fileList = [r for r in updateList if r.role == "File"] for f in fileList: - f.file = rfp("..", "unittests", "test_directories", "examples_article", f.file) + f.file = rfp("..", "unittests", "test_directories", + "examples_article", f.file) ins2, ups2 = cr.synchronize(commit_changes=True) assert len(ups1) == 0 assert len(ups2) == 0 @@ -297,7 +301,8 @@ def test_file_update(clear_database, usemodel, ident, crawler_extended): updateList = cr2.updateList fileList = [r for r in updateList if r.role == "File"] for f in fileList: - f.file = rfp("..", "unittests", "test_directories", "examples_article", f.file) + f.file = rfp("..", "unittests", "test_directories", + "examples_article", f.file) ins3, ups3 = cr2.synchronize(commit_changes=True) assert len(ups3) == 11 diff --git a/integrationtests/realworld_example/test_dataset_crawler.py b/integrationtests/realworld_example/test_dataset_crawler.py index 206020ce..8713f490 100644 --- a/integrationtests/realworld_example/test_dataset_crawler.py +++ b/integrationtests/realworld_example/test_dataset_crawler.py @@ -29,10 +29,10 @@ import os import caosdb as db -from newcrawler.crawl import Crawler -from newcrawler.converters import JSONFileConverter, DictConverter -from newcrawler.identifiable_adapters import CaosDBIdentifiableAdapter -from newcrawler.structure_elements import File, JSONFile, Directory +from caoscrawler.crawl import Crawler +from caoscrawler.converters import JSONFileConverter, DictConverter +from caoscrawler.identifiable_adapters import CaosDBIdentifiableAdapter +from caoscrawler.structure_elements import File, JSONFile, Directory import pytest from caosadvancedtools.models.parser import parse_model_from_json_schema, parse_model_from_yaml @@ -86,7 +86,8 @@ def test_dataset( ).add_parent("Person").add_property("full_name")) crawler = Crawler(debug=True, identifiableAdapter=ident) - crawler_definition = crawler.load_definition(os.path.join(DATADIR, "dataset_cfoods.yml")) + crawler_definition = crawler.load_definition( + os.path.join(DATADIR, "dataset_cfoods.yml")) # print(json.dumps(crawler_definition, indent=3)) # Load and register converter packages: converter_registry = crawler.load_converters(crawler_definition) diff --git a/setup.cfg b/setup.cfg index 0d5048c2..909eb3f6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [metadata] -name = newcrawler +name = caoscrawler version = 0.1 author = Alexander Schlemmer author_email = alexander.schlemmer@ds.mpg.de @@ -35,4 +35,4 @@ per-file-ignores = __init__.py:F401 [options.entry_points] console_scripts = - crawler2.0 = newcrawler.crawl:main + crawler2.0 = caoscrawler.crawl:main diff --git a/src/caoscrawler/crawl.py b/src/caoscrawler/crawl.py index de350a2a..b0f576a2 100644 --- a/src/caoscrawler/crawl.py +++ b/src/caoscrawler/crawl.py @@ -205,7 +205,7 @@ class Crawler(object): # tested in the next lines of code: # Load the cfood schema: - with open(files('newcrawler').joinpath('cfood-schema.yml'), "r") as f: + with open(files('caoscrawler').joinpath('cfood-schema.yml'), "r") as f: schema = yaml.safe_load(f) # Add custom converters to converter enum in schema: @@ -260,43 +260,43 @@ class Crawler(object): converter_registry: dict[str, dict[str, str]] = { "Directory": { "converter": "DirectoryConverter", - "package": "newcrawler.converters"}, + "package": "caoscrawler.converters"}, "SimpleFile": { "converter": "SimpleFileConverter", - "package": "newcrawler.converters"}, + "package": "caoscrawler.converters"}, "MarkdownFile": { "converter": "MarkdownFileConverter", - "package": "newcrawler.converters"}, + "package": "caoscrawler.converters"}, "File": { "converter": "FileConverter", - "package": "newcrawler.converters"}, + "package": "caoscrawler.converters"}, "JSONFile": { "converter": "JSONFileConverter", - "package": "newcrawler.converters"}, + "package": "caoscrawler.converters"}, "Dict": { "converter": "DictConverter", - "package": "newcrawler.converters"}, + "package": "caoscrawler.converters"}, "DictBooleanElement": { "converter": "DictBooleanElementConverter", - "package": "newcrawler.converters"}, + "package": "caoscrawler.converters"}, "DictFloatElement": { "converter": "DictFloatElementConverter", - "package": "newcrawler.converters"}, + "package": "caoscrawler.converters"}, "DictTextElement": { "converter": "DictTextElementConverter", - "package": "newcrawler.converters"}, + "package": "caoscrawler.converters"}, "DictIntegerElement": { "converter": "DictIntegerElementConverter", - "package": "newcrawler.converters"}, + "package": "caoscrawler.converters"}, "DictListElement": { "converter": "DictListElementConverter", - "package": "newcrawler.converters"}, + "package": "caoscrawler.converters"}, "DictDictElement": { "converter": "DictDictElementConverter", - "package": "newcrawler.converters"}, + "package": "caoscrawler.converters"}, "TextElement": { "converter": "TextElementConverter", - "package": "newcrawler.converters"} + "package": "caoscrawler.converters"} } # More converters from definition file: diff --git a/src/doc/conf.py b/src/doc/conf.py index 825f878b..fb37cdd9 100644 --- a/src/doc/conf.py +++ b/src/doc/conf.py @@ -28,7 +28,7 @@ import sphinx_rtd_theme # noqa: E402 # -- Project information ----------------------------------------------------- -project = 'caosdb-newcrawler' +project = 'caosdb-caoscrawler' copyright = '2021, MPIDS' author = 'Alexander Schlemmer' @@ -115,7 +115,7 @@ html_static_path = ['_static'] # -- Options for HTMLHelp output --------------------------------------------- # Output file base name for HTML help builder. -htmlhelp_basename = 'caosdb-newcrawlerdoc' +htmlhelp_basename = 'caosdb-caoscrawlerdoc' # -- Options for LaTeX output ------------------------------------------------ @@ -142,7 +142,7 @@ latex_elements = { # (source start file, target name, title, # author, documentclass [howto, manual, or own class]). latex_documents = [ - (master_doc, 'caosdb-newcrawler.tex', 'caosdb-newcrawler Documentation', + (master_doc, 'caosdb-caoscrawler.tex', 'caosdb-caoscrawler Documentation', 'MPIDS', 'manual'), ] @@ -152,7 +152,7 @@ latex_documents = [ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - (master_doc, 'caosdb-newcrawler', 'caosdb-newcrawler documentation', + (master_doc, 'caosdb-caoscrawler', 'caosdb-caoscrawler documentation', [author], 1) ] @@ -163,8 +163,8 @@ man_pages = [ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - (master_doc, 'caosdb-newcrawler', 'caosdb-newcrawler documentation', - author, 'caosdb-newcrawler', 'One line description of project.', + (master_doc, 'caosdb-caoscrawler', 'caosdb-caoscrawler documentation', + author, 'caosdb-caoscrawler', 'One line description of project.', 'Miscellaneous'), ] diff --git a/unittests/test_cache.py b/unittests/test_cache.py index 7061b63c..135316b9 100644 --- a/unittests/test_cache.py +++ b/unittests/test_cache.py @@ -5,7 +5,7 @@ import caosdb as db from pytest import raises -from newcrawler.identified_cache import _create_hashable_string as create_hash_string +from caoscrawler.identified_cache import _create_hashable_string as create_hash_string def test_normal_hash_creation(): diff --git a/unittests/test_converters.py b/unittests/test_converters.py index ab67a5c9..5f56486b 100644 --- a/unittests/test_converters.py +++ b/unittests/test_converters.py @@ -27,16 +27,16 @@ test the converters module """ -from newcrawler.converters import Converter -from newcrawler.stores import GeneralStore -from newcrawler.converters import (ConverterValidationError, - MarkdownFileConverter, JSONFileConverter, - DictConverter) -from newcrawler.structure_elements import Directory -from newcrawler.structure_elements import (File, DictTextElement, - DictListElement, DictElement, - DictBooleanElement, DictDictElement, - DictIntegerElement, DictFloatElement) +from caoscrawler.converters import Converter +from caoscrawler.stores import GeneralStore +from caoscrawler.converters import (ConverterValidationError, + MarkdownFileConverter, JSONFileConverter, + DictConverter) +from caoscrawler.structure_elements import Directory +from caoscrawler.structure_elements import (File, DictTextElement, + DictListElement, DictElement, + DictBooleanElement, DictDictElement, + DictIntegerElement, DictFloatElement) from test_tool import rfp @@ -50,25 +50,25 @@ def converter_registry(): converter_registry: dict[str, dict[str, str]] = { "Directory": { "converter": "DirectoryConverter", - "package": "newcrawler.converters"}, + "package": "caoscrawler.converters"}, "MarkdownFile": { "converter": "MarkdownFileConverter", - "package": "newcrawler.converters"}, + "package": "caoscrawler.converters"}, "Dict": { "converter": "DictConverter", - "package": "newcrawler.converters"}, + "package": "caoscrawler.converters"}, "DictTextElement": { "converter": "DictTextElementConverter", - "package": "newcrawler.converters"}, + "package": "caoscrawler.converters"}, "DictListElement": { "converter": "DictListElementConverter", - "package": "newcrawler.converters"}, + "package": "caoscrawler.converters"}, "TextElement": { "converter": "TextElementConverter", - "package": "newcrawler.converters"}, + "package": "caoscrawler.converters"}, "JSONFile": { "converter": "JSONFileConverter", - "package": "newcrawler.converters"}, + "package": "caoscrawler.converters"}, } for key, value in converter_registry.items(): diff --git a/unittests/test_entity_comparison.py b/unittests/test_entity_comparison.py index 750828bb..549bc4f4 100644 --- a/unittests/test_entity_comparison.py +++ b/unittests/test_entity_comparison.py @@ -7,7 +7,7 @@ import caosdb as db import pytest from pytest import raises -from newcrawler.crawl import check_identical +from caoscrawler.crawl import check_identical def test_compare_entities(): @@ -36,8 +36,10 @@ def test_compare_entities(): # This is confusing, but needed: record1.add_property(name="field_with_type", value=42, datatype=db.INTEGER) record2.add_property(name="field_with_type", value=42) - assert not check_identical(record1, record2) # not identical, because record1 sets the datatype - assert check_identical(record2, record1) # identical, because record2 sets the datatype + # not identical, because record1 sets the datatype + assert not check_identical(record1, record2) + # identical, because record2 sets the datatype + assert check_identical(record2, record1) record2.get_property("field_with_type").datatype = db.INTEGER assert check_identical(record1, record2) assert check_identical(record2, record1) @@ -81,7 +83,8 @@ def test_compare_entities(): setattr(record1, attribute, values[0]) # not identical, because record1 sets the datatype assert not check_identical(record1, record2) - assert check_identical(record2, record1) # identical, because record2 sets the datatype + # identical, because record2 sets the datatype + assert check_identical(record2, record1) setattr(record2, attribute, values[1]) assert not check_identical(record1, record2) diff --git a/unittests/test_file_identifiables.py b/unittests/test_file_identifiables.py index ae5c536a..b0b98019 100644 --- a/unittests/test_file_identifiables.py +++ b/unittests/test_file_identifiables.py @@ -7,7 +7,7 @@ import caosdb as db import pytest from pytest import raises -from newcrawler.identifiable_adapters import LocalStorageIdentifiableAdapter +from caoscrawler.identifiable_adapters import LocalStorageIdentifiableAdapter def test_file_identifiable(): @@ -41,7 +41,8 @@ def test_file_identifiable(): assert file_obj.checksum != identifiable.checksum # This is the wrong method, so it should definitely return None: - identified_file = ident.retrieve_identified_record_for_identifiable(identifiable) + identified_file = ident.retrieve_identified_record_for_identifiable( + identifiable) assert identified_file is None # This is the correct method to use: identified_file = ident.get_file(identifiable) diff --git a/unittests/test_identifiable_adapters.py b/unittests/test_identifiable_adapters.py index df311f1b..ef7998a4 100644 --- a/unittests/test_identifiable_adapters.py +++ b/unittests/test_identifiable_adapters.py @@ -29,7 +29,7 @@ test identifiable_adapters module import os from datetime import datetime -from newcrawler.identifiable_adapters import ( +from caoscrawler.identifiable_adapters import ( CaosDBIdentifiableAdapter, IdentifiableAdapter) import caosdb as db diff --git a/unittests/test_identified_cache.py b/unittests/test_identified_cache.py index 9a103463..33add97d 100644 --- a/unittests/test_identified_cache.py +++ b/unittests/test_identified_cache.py @@ -27,12 +27,13 @@ test identified_cache module """ -from newcrawler.identified_cache import _create_hashable_string, IdentifiedCache +from caoscrawler.identified_cache import _create_hashable_string, IdentifiedCache import caosdb as db def test_create_hash(): - assert _create_hashable_string(db.Record("A").add_parent("B")) == "P<B>N<A>" + assert _create_hashable_string( + db.Record("A").add_parent("B")) == "P<B>N<A>" assert _create_hashable_string(db.Record("A") .add_parent("B").add_property('a', 5)) == "P<B>N<A>a:5" assert (_create_hashable_string( diff --git a/unittests/test_json.py b/unittests/test_json.py index 237eca74..97d9831d 100644 --- a/unittests/test_json.py +++ b/unittests/test_json.py @@ -33,9 +33,9 @@ from pytest import raises import caosdb as db -from newcrawler.converters import JSONFileConverter, DictConverter -from newcrawler.crawl import Crawler -from newcrawler.structure_elements import File, JSONFile +from caoscrawler.converters import JSONFileConverter, DictConverter +from caoscrawler.crawl import Crawler +from caoscrawler.structure_elements import File, JSONFile from test_tool import rfp, dircheckstr diff --git a/unittests/test_schema.py b/unittests/test_schema.py index 4af29e28..0736698e 100644 --- a/unittests/test_schema.py +++ b/unittests/test_schema.py @@ -6,7 +6,7 @@ from importlib_resources import files import caosdb as db from os.path import join, dirname -from newcrawler import Crawler +from caoscrawler import Crawler import pytest from pytest import raises diff --git a/unittests/test_tool.py b/unittests/test_tool.py index 9fe127e1..1e7f1006 100755 --- a/unittests/test_tool.py +++ b/unittests/test_tool.py @@ -3,9 +3,9 @@ # Adapted from check-sfs # A. Schlemmer, 06/2021 -from newcrawler import Crawler -from newcrawler.structure_elements import File, DictTextElement, DictListElement -from newcrawler.identifiable_adapters import IdentifiableAdapter, LocalStorageIdentifiableAdapter +from caoscrawler import Crawler +from caoscrawler.structure_elements import File, DictTextElement, DictListElement +from caoscrawler.identifiable_adapters import IdentifiableAdapter, LocalStorageIdentifiableAdapter from functools import partial from copy import deepcopy from unittest.mock import MagicMock, Mock @@ -30,7 +30,7 @@ def dircheckstr(*pathcomponents): """ Return the debug tree identifier for a given path. """ - return "newcrawler.structure_elements.Directory: " + basename(join(*pathcomponents)) + ", " + rfp("test_directories", "examples_article", *pathcomponents) + return "caoscrawler.structure_elements.Directory: " + basename(join(*pathcomponents)) + ", " + rfp("test_directories", "examples_article", *pathcomponents) @pytest.fixture diff --git a/unittests/test_tool_extended.py b/unittests/test_tool_extended.py index d9f37064..d0b431a5 100644 --- a/unittests/test_tool_extended.py +++ b/unittests/test_tool_extended.py @@ -3,9 +3,9 @@ # Adapted from check-sfs # A. Schlemmer, 06/2021 -from newcrawler import Crawler -from newcrawler.structure_elements import File, DictTextElement, DictListElement -from newcrawler.identifiable_adapters import IdentifiableAdapter, LocalStorageIdentifiableAdapter +from caoscrawler import Crawler +from caoscrawler.structure_elements import File, DictTextElement, DictListElement +from caoscrawler.identifiable_adapters import IdentifiableAdapter, LocalStorageIdentifiableAdapter from functools import partial from copy import deepcopy from unittest.mock import MagicMock, Mock @@ -30,7 +30,7 @@ def dircheckstr(*pathcomponents, structure_element_type="Directory"): """ Return the debug tree identifier for a given path. """ - return ("newcrawler.structure_elements." + structure_element_type + ": " + + return ("caoscrawler.structure_elements." + structure_element_type + ": " + basename(join(*pathcomponents)) + ", " + rfp("test_directories", "examples_article", *pathcomponents)) diff --git a/unittests/test_variable_substitutions.py b/unittests/test_variable_substitutions.py index b8c10d85..071bf464 100644 --- a/unittests/test_variable_substitutions.py +++ b/unittests/test_variable_substitutions.py @@ -2,9 +2,9 @@ # Tests for variable substitutions # A. Schlemmer, 05/2022 -from newcrawler import Crawler -from newcrawler.structure_elements import File, DictTextElement, DictListElement -from newcrawler.identifiable_adapters import IdentifiableAdapter, LocalStorageIdentifiableAdapter +from caoscrawler import Crawler +from caoscrawler.structure_elements import File, DictTextElement, DictListElement +from caoscrawler.identifiable_adapters import IdentifiableAdapter, LocalStorageIdentifiableAdapter from functools import partial from copy import deepcopy from unittest.mock import MagicMock, Mock @@ -29,7 +29,7 @@ def dircheckstr(element_type, *pathcomponents): """ Return the debug tree identifier for a given path. """ - return "newcrawler.structure_elements." + element_type + ": " + basename(join(*pathcomponents)) + ", " + rfp("test_directories", "example_substitutions", *pathcomponents) + return "caoscrawler.structure_elements." + element_type + ": " + basename(join(*pathcomponents)) + ", " + rfp("test_directories", "example_substitutions", *pathcomponents) @pytest.fixture -- GitLab