From 017e244687902dec50ac622082cd7edcb4bc1a58 Mon Sep 17 00:00:00 2001
From: florian <f.spreckelsen@inidscale.com>
Date: Tue, 17 May 2022 13:41:33 +0200
Subject: [PATCH] MAINT: Rename module in source files
---
integrationtests/basic_example/test.py | 19 +++++++----
.../realworld_example/test_dataset_crawler.py | 11 +++---
setup.cfg | 4 +--
src/caoscrawler/crawl.py | 28 +++++++--------
src/doc/conf.py | 12 +++----
unittests/test_cache.py | 2 +-
unittests/test_converters.py | 34 +++++++++----------
unittests/test_entity_comparison.py | 11 +++---
unittests/test_file_identifiables.py | 5 +--
unittests/test_identifiable_adapters.py | 2 +-
unittests/test_identified_cache.py | 5 +--
unittests/test_json.py | 6 ++--
unittests/test_schema.py | 2 +-
unittests/test_tool.py | 8 ++---
unittests/test_tool_extended.py | 8 ++---
unittests/test_variable_substitutions.py | 8 ++---
16 files changed, 88 insertions(+), 77 deletions(-)
diff --git a/integrationtests/basic_example/test.py b/integrationtests/basic_example/test.py
index 35ee9918..6e35f7f2 100755
--- a/integrationtests/basic_example/test.py
+++ b/integrationtests/basic_example/test.py
@@ -33,9 +33,9 @@ from caosdb import EmptyUniqueQueryError
import argparse
import sys
from argparse import RawTextHelpFormatter
-from newcrawler import Crawler
+from caoscrawler import Crawler
import caosdb as db
-from newcrawler.identifiable_adapters import CaosDBIdentifiableAdapter
+from caoscrawler.identifiable_adapters import CaosDBIdentifiableAdapter
import pytest
from caosadvancedtools.models.parser import parse_model_from_yaml
import yaml
@@ -105,7 +105,8 @@ def crawler_extended(ident):
updateList = cr.updateList
fileList = [r for r in updateList if r.role == "File"]
for f in fileList:
- f.file = rfp("..", "unittests", "test_directories", "examples_article", f.file)
+ f.file = rfp("..", "unittests", "test_directories",
+ "examples_article", f.file)
return cr
@@ -201,7 +202,8 @@ def test_identifiable_update(clear_database, usemodel, ident, crawler):
if (record.parents[0].name == "Measurement" and
record.get_property("date").value == "2020-01-03"):
# maybe a bit weird, but add an email address to a measurement
- record.add_property(name="email", value="testperson@testaccount.test")
+ record.add_property(
+ name="email", value="testperson@testaccount.test")
print("one change")
break
ins, ups = cr.synchronize()
@@ -240,7 +242,8 @@ def test_identifiable_update(clear_database, usemodel, ident, crawler):
def test_file_insertion_dry(clear_database, usemodel, ident):
crawler_extended = Crawler(debug=True, identifiableAdapter=ident)
- crawl_standard_test_directory(crawler_extended, cfood="scifolder_extended.yml")
+ crawl_standard_test_directory(
+ crawler_extended, cfood="scifolder_extended.yml")
updateList = crawler_extended.updateList
fileList = [r for r in updateList if r.role == "File"]
assert len(fileList) == 11
@@ -281,7 +284,8 @@ def test_file_update(clear_database, usemodel, ident, crawler_extended):
updateList = cr.updateList
fileList = [r for r in updateList if r.role == "File"]
for f in fileList:
- f.file = rfp("..", "unittests", "test_directories", "examples_article", f.file)
+ f.file = rfp("..", "unittests", "test_directories",
+ "examples_article", f.file)
ins2, ups2 = cr.synchronize(commit_changes=True)
assert len(ups1) == 0
assert len(ups2) == 0
@@ -297,7 +301,8 @@ def test_file_update(clear_database, usemodel, ident, crawler_extended):
updateList = cr2.updateList
fileList = [r for r in updateList if r.role == "File"]
for f in fileList:
- f.file = rfp("..", "unittests", "test_directories", "examples_article", f.file)
+ f.file = rfp("..", "unittests", "test_directories",
+ "examples_article", f.file)
ins3, ups3 = cr2.synchronize(commit_changes=True)
assert len(ups3) == 11
diff --git a/integrationtests/realworld_example/test_dataset_crawler.py b/integrationtests/realworld_example/test_dataset_crawler.py
index 206020ce..8713f490 100644
--- a/integrationtests/realworld_example/test_dataset_crawler.py
+++ b/integrationtests/realworld_example/test_dataset_crawler.py
@@ -29,10 +29,10 @@ import os
import caosdb as db
-from newcrawler.crawl import Crawler
-from newcrawler.converters import JSONFileConverter, DictConverter
-from newcrawler.identifiable_adapters import CaosDBIdentifiableAdapter
-from newcrawler.structure_elements import File, JSONFile, Directory
+from caoscrawler.crawl import Crawler
+from caoscrawler.converters import JSONFileConverter, DictConverter
+from caoscrawler.identifiable_adapters import CaosDBIdentifiableAdapter
+from caoscrawler.structure_elements import File, JSONFile, Directory
import pytest
from caosadvancedtools.models.parser import parse_model_from_json_schema, parse_model_from_yaml
@@ -86,7 +86,8 @@ def test_dataset(
).add_parent("Person").add_property("full_name"))
crawler = Crawler(debug=True, identifiableAdapter=ident)
- crawler_definition = crawler.load_definition(os.path.join(DATADIR, "dataset_cfoods.yml"))
+ crawler_definition = crawler.load_definition(
+ os.path.join(DATADIR, "dataset_cfoods.yml"))
# print(json.dumps(crawler_definition, indent=3))
# Load and register converter packages:
converter_registry = crawler.load_converters(crawler_definition)
diff --git a/setup.cfg b/setup.cfg
index 0d5048c2..909eb3f6 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -1,5 +1,5 @@
[metadata]
-name = newcrawler
+name = caoscrawler
version = 0.1
author = Alexander Schlemmer
author_email = alexander.schlemmer@ds.mpg.de
@@ -35,4 +35,4 @@ per-file-ignores = __init__.py:F401
[options.entry_points]
console_scripts =
- crawler2.0 = newcrawler.crawl:main
+ crawler2.0 = caoscrawler.crawl:main
diff --git a/src/caoscrawler/crawl.py b/src/caoscrawler/crawl.py
index de350a2a..b0f576a2 100644
--- a/src/caoscrawler/crawl.py
+++ b/src/caoscrawler/crawl.py
@@ -205,7 +205,7 @@ class Crawler(object):
# tested in the next lines of code:
# Load the cfood schema:
- with open(files('newcrawler').joinpath('cfood-schema.yml'), "r") as f:
+ with open(files('caoscrawler').joinpath('cfood-schema.yml'), "r") as f:
schema = yaml.safe_load(f)
# Add custom converters to converter enum in schema:
@@ -260,43 +260,43 @@ class Crawler(object):
converter_registry: dict[str, dict[str, str]] = {
"Directory": {
"converter": "DirectoryConverter",
- "package": "newcrawler.converters"},
+ "package": "caoscrawler.converters"},
"SimpleFile": {
"converter": "SimpleFileConverter",
- "package": "newcrawler.converters"},
+ "package": "caoscrawler.converters"},
"MarkdownFile": {
"converter": "MarkdownFileConverter",
- "package": "newcrawler.converters"},
+ "package": "caoscrawler.converters"},
"File": {
"converter": "FileConverter",
- "package": "newcrawler.converters"},
+ "package": "caoscrawler.converters"},
"JSONFile": {
"converter": "JSONFileConverter",
- "package": "newcrawler.converters"},
+ "package": "caoscrawler.converters"},
"Dict": {
"converter": "DictConverter",
- "package": "newcrawler.converters"},
+ "package": "caoscrawler.converters"},
"DictBooleanElement": {
"converter": "DictBooleanElementConverter",
- "package": "newcrawler.converters"},
+ "package": "caoscrawler.converters"},
"DictFloatElement": {
"converter": "DictFloatElementConverter",
- "package": "newcrawler.converters"},
+ "package": "caoscrawler.converters"},
"DictTextElement": {
"converter": "DictTextElementConverter",
- "package": "newcrawler.converters"},
+ "package": "caoscrawler.converters"},
"DictIntegerElement": {
"converter": "DictIntegerElementConverter",
- "package": "newcrawler.converters"},
+ "package": "caoscrawler.converters"},
"DictListElement": {
"converter": "DictListElementConverter",
- "package": "newcrawler.converters"},
+ "package": "caoscrawler.converters"},
"DictDictElement": {
"converter": "DictDictElementConverter",
- "package": "newcrawler.converters"},
+ "package": "caoscrawler.converters"},
"TextElement": {
"converter": "TextElementConverter",
- "package": "newcrawler.converters"}
+ "package": "caoscrawler.converters"}
}
# More converters from definition file:
diff --git a/src/doc/conf.py b/src/doc/conf.py
index 825f878b..fb37cdd9 100644
--- a/src/doc/conf.py
+++ b/src/doc/conf.py
@@ -28,7 +28,7 @@ import sphinx_rtd_theme # noqa: E402
# -- Project information -----------------------------------------------------
-project = 'caosdb-newcrawler'
+project = 'caosdb-caoscrawler'
copyright = '2021, MPIDS'
author = 'Alexander Schlemmer'
@@ -115,7 +115,7 @@ html_static_path = ['_static']
# -- Options for HTMLHelp output ---------------------------------------------
# Output file base name for HTML help builder.
-htmlhelp_basename = 'caosdb-newcrawlerdoc'
+htmlhelp_basename = 'caosdb-caoscrawlerdoc'
# -- Options for LaTeX output ------------------------------------------------
@@ -142,7 +142,7 @@ latex_elements = {
# (source start file, target name, title,
# author, documentclass [howto, manual, or own class]).
latex_documents = [
- (master_doc, 'caosdb-newcrawler.tex', 'caosdb-newcrawler Documentation',
+ (master_doc, 'caosdb-caoscrawler.tex', 'caosdb-caoscrawler Documentation',
'MPIDS', 'manual'),
]
@@ -152,7 +152,7 @@ latex_documents = [
# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
- (master_doc, 'caosdb-newcrawler', 'caosdb-newcrawler documentation',
+ (master_doc, 'caosdb-caoscrawler', 'caosdb-caoscrawler documentation',
[author], 1)
]
@@ -163,8 +163,8 @@ man_pages = [
# (source start file, target name, title, author,
# dir menu entry, description, category)
texinfo_documents = [
- (master_doc, 'caosdb-newcrawler', 'caosdb-newcrawler documentation',
- author, 'caosdb-newcrawler', 'One line description of project.',
+ (master_doc, 'caosdb-caoscrawler', 'caosdb-caoscrawler documentation',
+ author, 'caosdb-caoscrawler', 'One line description of project.',
'Miscellaneous'),
]
diff --git a/unittests/test_cache.py b/unittests/test_cache.py
index 7061b63c..135316b9 100644
--- a/unittests/test_cache.py
+++ b/unittests/test_cache.py
@@ -5,7 +5,7 @@
import caosdb as db
from pytest import raises
-from newcrawler.identified_cache import _create_hashable_string as create_hash_string
+from caoscrawler.identified_cache import _create_hashable_string as create_hash_string
def test_normal_hash_creation():
diff --git a/unittests/test_converters.py b/unittests/test_converters.py
index ab67a5c9..5f56486b 100644
--- a/unittests/test_converters.py
+++ b/unittests/test_converters.py
@@ -27,16 +27,16 @@
test the converters module
"""
-from newcrawler.converters import Converter
-from newcrawler.stores import GeneralStore
-from newcrawler.converters import (ConverterValidationError,
- MarkdownFileConverter, JSONFileConverter,
- DictConverter)
-from newcrawler.structure_elements import Directory
-from newcrawler.structure_elements import (File, DictTextElement,
- DictListElement, DictElement,
- DictBooleanElement, DictDictElement,
- DictIntegerElement, DictFloatElement)
+from caoscrawler.converters import Converter
+from caoscrawler.stores import GeneralStore
+from caoscrawler.converters import (ConverterValidationError,
+ MarkdownFileConverter, JSONFileConverter,
+ DictConverter)
+from caoscrawler.structure_elements import Directory
+from caoscrawler.structure_elements import (File, DictTextElement,
+ DictListElement, DictElement,
+ DictBooleanElement, DictDictElement,
+ DictIntegerElement, DictFloatElement)
from test_tool import rfp
@@ -50,25 +50,25 @@ def converter_registry():
converter_registry: dict[str, dict[str, str]] = {
"Directory": {
"converter": "DirectoryConverter",
- "package": "newcrawler.converters"},
+ "package": "caoscrawler.converters"},
"MarkdownFile": {
"converter": "MarkdownFileConverter",
- "package": "newcrawler.converters"},
+ "package": "caoscrawler.converters"},
"Dict": {
"converter": "DictConverter",
- "package": "newcrawler.converters"},
+ "package": "caoscrawler.converters"},
"DictTextElement": {
"converter": "DictTextElementConverter",
- "package": "newcrawler.converters"},
+ "package": "caoscrawler.converters"},
"DictListElement": {
"converter": "DictListElementConverter",
- "package": "newcrawler.converters"},
+ "package": "caoscrawler.converters"},
"TextElement": {
"converter": "TextElementConverter",
- "package": "newcrawler.converters"},
+ "package": "caoscrawler.converters"},
"JSONFile": {
"converter": "JSONFileConverter",
- "package": "newcrawler.converters"},
+ "package": "caoscrawler.converters"},
}
for key, value in converter_registry.items():
diff --git a/unittests/test_entity_comparison.py b/unittests/test_entity_comparison.py
index 750828bb..549bc4f4 100644
--- a/unittests/test_entity_comparison.py
+++ b/unittests/test_entity_comparison.py
@@ -7,7 +7,7 @@ import caosdb as db
import pytest
from pytest import raises
-from newcrawler.crawl import check_identical
+from caoscrawler.crawl import check_identical
def test_compare_entities():
@@ -36,8 +36,10 @@ def test_compare_entities():
# This is confusing, but needed:
record1.add_property(name="field_with_type", value=42, datatype=db.INTEGER)
record2.add_property(name="field_with_type", value=42)
- assert not check_identical(record1, record2) # not identical, because record1 sets the datatype
- assert check_identical(record2, record1) # identical, because record2 sets the datatype
+ # not identical, because record1 sets the datatype
+ assert not check_identical(record1, record2)
+ # identical, because record2 sets the datatype
+ assert check_identical(record2, record1)
record2.get_property("field_with_type").datatype = db.INTEGER
assert check_identical(record1, record2)
assert check_identical(record2, record1)
@@ -81,7 +83,8 @@ def test_compare_entities():
setattr(record1, attribute, values[0])
# not identical, because record1 sets the datatype
assert not check_identical(record1, record2)
- assert check_identical(record2, record1) # identical, because record2 sets the datatype
+ # identical, because record2 sets the datatype
+ assert check_identical(record2, record1)
setattr(record2, attribute, values[1])
assert not check_identical(record1, record2)
diff --git a/unittests/test_file_identifiables.py b/unittests/test_file_identifiables.py
index ae5c536a..b0b98019 100644
--- a/unittests/test_file_identifiables.py
+++ b/unittests/test_file_identifiables.py
@@ -7,7 +7,7 @@ import caosdb as db
import pytest
from pytest import raises
-from newcrawler.identifiable_adapters import LocalStorageIdentifiableAdapter
+from caoscrawler.identifiable_adapters import LocalStorageIdentifiableAdapter
def test_file_identifiable():
@@ -41,7 +41,8 @@ def test_file_identifiable():
assert file_obj.checksum != identifiable.checksum
# This is the wrong method, so it should definitely return None:
- identified_file = ident.retrieve_identified_record_for_identifiable(identifiable)
+ identified_file = ident.retrieve_identified_record_for_identifiable(
+ identifiable)
assert identified_file is None
# This is the correct method to use:
identified_file = ident.get_file(identifiable)
diff --git a/unittests/test_identifiable_adapters.py b/unittests/test_identifiable_adapters.py
index df311f1b..ef7998a4 100644
--- a/unittests/test_identifiable_adapters.py
+++ b/unittests/test_identifiable_adapters.py
@@ -29,7 +29,7 @@ test identifiable_adapters module
import os
from datetime import datetime
-from newcrawler.identifiable_adapters import (
+from caoscrawler.identifiable_adapters import (
CaosDBIdentifiableAdapter, IdentifiableAdapter)
import caosdb as db
diff --git a/unittests/test_identified_cache.py b/unittests/test_identified_cache.py
index 9a103463..33add97d 100644
--- a/unittests/test_identified_cache.py
+++ b/unittests/test_identified_cache.py
@@ -27,12 +27,13 @@
test identified_cache module
"""
-from newcrawler.identified_cache import _create_hashable_string, IdentifiedCache
+from caoscrawler.identified_cache import _create_hashable_string, IdentifiedCache
import caosdb as db
def test_create_hash():
- assert _create_hashable_string(db.Record("A").add_parent("B")) == "P<B>N<A>"
+ assert _create_hashable_string(
+ db.Record("A").add_parent("B")) == "P<B>N<A>"
assert _create_hashable_string(db.Record("A")
.add_parent("B").add_property('a', 5)) == "P<B>N<A>a:5"
assert (_create_hashable_string(
diff --git a/unittests/test_json.py b/unittests/test_json.py
index 237eca74..97d9831d 100644
--- a/unittests/test_json.py
+++ b/unittests/test_json.py
@@ -33,9 +33,9 @@ from pytest import raises
import caosdb as db
-from newcrawler.converters import JSONFileConverter, DictConverter
-from newcrawler.crawl import Crawler
-from newcrawler.structure_elements import File, JSONFile
+from caoscrawler.converters import JSONFileConverter, DictConverter
+from caoscrawler.crawl import Crawler
+from caoscrawler.structure_elements import File, JSONFile
from test_tool import rfp, dircheckstr
diff --git a/unittests/test_schema.py b/unittests/test_schema.py
index 4af29e28..0736698e 100644
--- a/unittests/test_schema.py
+++ b/unittests/test_schema.py
@@ -6,7 +6,7 @@ from importlib_resources import files
import caosdb as db
from os.path import join, dirname
-from newcrawler import Crawler
+from caoscrawler import Crawler
import pytest
from pytest import raises
diff --git a/unittests/test_tool.py b/unittests/test_tool.py
index 9fe127e1..1e7f1006 100755
--- a/unittests/test_tool.py
+++ b/unittests/test_tool.py
@@ -3,9 +3,9 @@
# Adapted from check-sfs
# A. Schlemmer, 06/2021
-from newcrawler import Crawler
-from newcrawler.structure_elements import File, DictTextElement, DictListElement
-from newcrawler.identifiable_adapters import IdentifiableAdapter, LocalStorageIdentifiableAdapter
+from caoscrawler import Crawler
+from caoscrawler.structure_elements import File, DictTextElement, DictListElement
+from caoscrawler.identifiable_adapters import IdentifiableAdapter, LocalStorageIdentifiableAdapter
from functools import partial
from copy import deepcopy
from unittest.mock import MagicMock, Mock
@@ -30,7 +30,7 @@ def dircheckstr(*pathcomponents):
"""
Return the debug tree identifier for a given path.
"""
- return "newcrawler.structure_elements.Directory: " + basename(join(*pathcomponents)) + ", " + rfp("test_directories", "examples_article", *pathcomponents)
+ return "caoscrawler.structure_elements.Directory: " + basename(join(*pathcomponents)) + ", " + rfp("test_directories", "examples_article", *pathcomponents)
@pytest.fixture
diff --git a/unittests/test_tool_extended.py b/unittests/test_tool_extended.py
index d9f37064..d0b431a5 100644
--- a/unittests/test_tool_extended.py
+++ b/unittests/test_tool_extended.py
@@ -3,9 +3,9 @@
# Adapted from check-sfs
# A. Schlemmer, 06/2021
-from newcrawler import Crawler
-from newcrawler.structure_elements import File, DictTextElement, DictListElement
-from newcrawler.identifiable_adapters import IdentifiableAdapter, LocalStorageIdentifiableAdapter
+from caoscrawler import Crawler
+from caoscrawler.structure_elements import File, DictTextElement, DictListElement
+from caoscrawler.identifiable_adapters import IdentifiableAdapter, LocalStorageIdentifiableAdapter
from functools import partial
from copy import deepcopy
from unittest.mock import MagicMock, Mock
@@ -30,7 +30,7 @@ def dircheckstr(*pathcomponents, structure_element_type="Directory"):
"""
Return the debug tree identifier for a given path.
"""
- return ("newcrawler.structure_elements." + structure_element_type + ": " +
+ return ("caoscrawler.structure_elements." + structure_element_type + ": " +
basename(join(*pathcomponents)) + ", " +
rfp("test_directories", "examples_article", *pathcomponents))
diff --git a/unittests/test_variable_substitutions.py b/unittests/test_variable_substitutions.py
index b8c10d85..071bf464 100644
--- a/unittests/test_variable_substitutions.py
+++ b/unittests/test_variable_substitutions.py
@@ -2,9 +2,9 @@
# Tests for variable substitutions
# A. Schlemmer, 05/2022
-from newcrawler import Crawler
-from newcrawler.structure_elements import File, DictTextElement, DictListElement
-from newcrawler.identifiable_adapters import IdentifiableAdapter, LocalStorageIdentifiableAdapter
+from caoscrawler import Crawler
+from caoscrawler.structure_elements import File, DictTextElement, DictListElement
+from caoscrawler.identifiable_adapters import IdentifiableAdapter, LocalStorageIdentifiableAdapter
from functools import partial
from copy import deepcopy
from unittest.mock import MagicMock, Mock
@@ -29,7 +29,7 @@ def dircheckstr(element_type, *pathcomponents):
"""
Return the debug tree identifier for a given path.
"""
- return "newcrawler.structure_elements." + element_type + ": " + basename(join(*pathcomponents)) + ", " + rfp("test_directories", "example_substitutions", *pathcomponents)
+ return "caoscrawler.structure_elements." + element_type + ": " + basename(join(*pathcomponents)) + ", " + rfp("test_directories", "example_substitutions", *pathcomponents)
@pytest.fixture
--
GitLab