diff --git a/src/caoscrawler/authorize.py b/src/caoscrawler/authorize.py index 6f1011b227881d4b73186996076abe20d94d52e5..f3deed4f8c78afa85fdd4471fe9383760b8c8b12 100644 --- a/src/caoscrawler/authorize.py +++ b/src/caoscrawler/authorize.py @@ -19,10 +19,10 @@ # along with this program. If not, see <https://www.gnu.org/licenses/>. # -from caosadvancedtools.crawler import Crawler as OldCrawler - import argparse +from caosadvancedtools.crawler import Crawler as OldCrawler + def parse_args(): parser = argparse.ArgumentParser() diff --git a/src/caoscrawler/converters/__init__.py b/src/caoscrawler/converters/__init__.py index 70fca6c44c90a3bbb44bd05c34e51cafae91a229..670d4e966c72c6bcf45d0d46c1db715fb79d8ab5 100644 --- a/src/caoscrawler/converters/__init__.py +++ b/src/caoscrawler/converters/__init__.py @@ -32,9 +32,8 @@ except ImportError as err: err=err) try: - from .rocrate import ROCrateEntityConverter - from .rocrate import ROCrateConverter - from .rocrate import ELNFileConverter + from .rocrate import (ELNFileConverter, ROCrateConverter, + ROCrateEntityConverter) except ImportError as err: ROCrateEntityConverter: type = utils.MissingImport( name="ROCrateEntityConverter", hint="Try installing with the `rocrate` extra option.", diff --git a/src/caoscrawler/converters/hdf5_converter.py b/src/caoscrawler/converters/hdf5_converter.py index a4d974bd53fc4b0e22d155f01a6a47295b79e984..97dac53d053dbcb87c48f0cfb59d4f09770b9710 100644 --- a/src/caoscrawler/converters/hdf5_converter.py +++ b/src/caoscrawler/converters/hdf5_converter.py @@ -28,16 +28,16 @@ except ModuleNotFoundError: "its optional `h5-crawler` dependency?" ) -import numpy as np - from typing import Union import linkahead as db +import numpy as np -from .converters import (convert_basic_element, Converter, DictElementConverter, - match_name_and_value, SimpleFileConverter) from ..stores import GeneralStore, RecordStore -from ..structure_elements import DictElement, File, FloatElement, IntegerElement, StructureElement +from ..structure_elements import (DictElement, File, FloatElement, + IntegerElement, StructureElement) +from .converters import (Converter, DictElementConverter, SimpleFileConverter, + convert_basic_element, match_name_and_value) def convert_attributes(elt: Union[h5py.File, h5py.Group, h5py.Dataset]): diff --git a/src/caoscrawler/converters/rocrate.py b/src/caoscrawler/converters/rocrate.py index 189d936cebe4bee6213005adbee50e36b63b2898..b84462acba2fdd7e60094e38edc38605c80deb11 100644 --- a/src/caoscrawler/converters/rocrate.py +++ b/src/caoscrawler/converters/rocrate.py @@ -33,7 +33,6 @@ from typing import Optional from zipfile import ZipFile import linkahead as db - import rocrate from rocrate.rocrate import ROCrate diff --git a/src/caoscrawler/converters/spss.py b/src/caoscrawler/converters/spss.py index b4f03aeaed6663be98487a4780bb96237e72e27e..00742e91506245435ed0c590f68ea9ffce65717a 100644 --- a/src/caoscrawler/converters/spss.py +++ b/src/caoscrawler/converters/spss.py @@ -22,17 +22,16 @@ from __future__ import annotations # Can be removed with 3.10. import argparse from collections import OrderedDict +from typing import Any, Optional import numpy as np import pandas as pd import pyreadstat import yaml -from . import converters from ..stores import GeneralStore -from ..structure_elements import (File, StructureElement) -from typing import Optional, Any - +from ..structure_elements import File, StructureElement +from . import converters READSTAT_TYPES = { "double": "DOUBLE", diff --git a/src/caoscrawler/converters/xml_converter.py b/src/caoscrawler/converters/xml_converter.py index bd3f6cf0fdcc5fed5b5452da8a17a8a877009b06..b9f7487ee633d0ba25a3b81b78b9a3561274edc9 100644 --- a/src/caoscrawler/converters/xml_converter.py +++ b/src/caoscrawler/converters/xml_converter.py @@ -22,17 +22,17 @@ from __future__ import annotations -import lxml.etree import re - from typing import Optional import linkahead as db +import lxml.etree -from .converters import SimpleFileConverter, ConverterValidationError, Converter from ..stores import GeneralStore, RecordStore -from ..structure_elements import (File, StructureElement, - XMLTagElement, XMLTextNode, XMLAttributeNode) +from ..structure_elements import (File, StructureElement, XMLAttributeNode, + XMLTagElement, XMLTextNode) +from .converters import (Converter, ConverterValidationError, + SimpleFileConverter) class XMLFileConverter(SimpleFileConverter): diff --git a/src/caoscrawler/crawl.py b/src/caoscrawler/crawl.py index 9e4e2a80da40599a51b674ad9a2001b8d9fa691b..a79e4434ee8f58fd1cc2646ced85c0d02d3fb66b 100644 --- a/src/caoscrawler/crawl.py +++ b/src/caoscrawler/crawl.py @@ -39,7 +39,6 @@ import sys import traceback import uuid import warnings - from argparse import RawTextHelpFormatter from copy import deepcopy from datetime import datetime @@ -52,13 +51,10 @@ from caosadvancedtools.cache import UpdateCache from caosadvancedtools.crawler import Crawler as OldCrawler from caosadvancedtools.serverside.helper import send_mail from caosadvancedtools.utils import create_entity_link -from linkahead.apiutils import (compare_entities, - merge_entities) +from linkahead.apiutils import compare_entities, merge_entities from linkahead.cached import cache_clear, cached_get_entity_by from linkahead.common.datatype import get_list_datatype, is_reference -from linkahead.exceptions import ( - TransactionError, -) +from linkahead.exceptions import TransactionError from linkahead.utils.escape import escape_squoted_text from .config import get_config_setting diff --git a/src/caoscrawler/debug_tree.py b/src/caoscrawler/debug_tree.py index 0d57040f5c20aca236a3c11531e8b7c45bad89ab..c154f5b91d850476be0c0610e5bb1dfcbf9866ab 100644 --- a/src/caoscrawler/debug_tree.py +++ b/src/caoscrawler/debug_tree.py @@ -29,35 +29,20 @@ A structure containing debug tree information. from __future__ import annotations -import argparse -import importlib -import logging -import os -import sys -import warnings -import yaml - -from argparse import RawTextHelpFormatter from collections import defaultdict -from copy import deepcopy -from enum import Enum -from importlib_resources import files -from jsonschema import validate -from typing import Any, Optional, Type, Union import linkahead as db - -from caosadvancedtools.cache import UpdateCache, Cache -from caosadvancedtools.crawler import Crawler as OldCrawler -from linkahead.apiutils import (compare_entities, EntityMergeConflictError, +import yaml +from importlib_resources import files +from jsonschema import validate +from linkahead.apiutils import (EntityMergeConflictError, compare_entities, merge_entities) from linkahead.common.datatype import is_reference -from .converters import Converter, DirectoryConverter, ConverterValidationError - +from .converters import Converter, ConverterValidationError, DirectoryConverter from .macros import defmacro_constructor, macro_constructor -from .stores import Store, GeneralStore, RecordStore -from .structure_elements import StructureElement, Directory, NoneElement +from .stores import GeneralStore, RecordStore, Store +from .structure_elements import Directory, NoneElement, StructureElement from .version import check_cfood_version diff --git a/src/caoscrawler/identifiable_adapters.py b/src/caoscrawler/identifiable_adapters.py index 854ee614638712bdcf957c592ef2946dbdd43afc..592f603bef508771d734ff633f8cdb2c100742d5 100644 --- a/src/caoscrawler/identifiable_adapters.py +++ b/src/caoscrawler/identifiable_adapters.py @@ -36,12 +36,8 @@ import yaml from linkahead.cached import cached_get_entity_by, cached_query from linkahead.utils.escape import escape_squoted_text -from .exceptions import ( - InvalidIdentifiableYAML, - MissingIdentifyingProperty, - MissingRecordType, - MissingReferencingEntityError, -) +from .exceptions import (InvalidIdentifiableYAML, MissingIdentifyingProperty, + MissingRecordType, MissingReferencingEntityError) from .identifiable import Identifiable from .sync_node import SyncNode from .utils import has_parent diff --git a/src/caoscrawler/logging.py b/src/caoscrawler/logging.py index f22f9678de768586b04e6781e1448242802406ee..b57a067d8635a468df7345365fabbfae9ee0b22f 100644 --- a/src/caoscrawler/logging.py +++ b/src/caoscrawler/logging.py @@ -20,10 +20,10 @@ # along with this program. If not, see <https://www.gnu.org/licenses/>. import logging +import sys -from caosadvancedtools.webui_formatter import WebUI_Formatter from caosadvancedtools.serverside.helper import get_shared_filename -import sys +from caosadvancedtools.webui_formatter import WebUI_Formatter def configure_server_side_logging(max_log_level: int = logging.INFO): diff --git a/src/caoscrawler/macros/macro_yaml_object.py b/src/caoscrawler/macros/macro_yaml_object.py index d85883011db3cf651da0dda6c110015128fbe439..5d2bc1fe0775499fa8b40a65e115fb4569892e38 100644 --- a/src/caoscrawler/macros/macro_yaml_object.py +++ b/src/caoscrawler/macros/macro_yaml_object.py @@ -26,11 +26,10 @@ # A. Schlemmer, 05/2022 import re -from dataclasses import dataclass -from typing import Any, Dict from copy import deepcopy +from dataclasses import dataclass from string import Template - +from typing import Any, Dict _SAFE_SUBST_PAT = re.compile(r"^\$(?P<key>\w+)$") _SAFE_SUBST_PAT_BRACES = re.compile(r"^\$\{(?P<key>\w+)}$") diff --git a/src/caoscrawler/scanner.py b/src/caoscrawler/scanner.py index 27711e6a7c4e69df3c2d99aca7a427670b153765..89bd1c04411665bf4832d6bccce69bbe1b11cad1 100644 --- a/src/caoscrawler/scanner.py +++ b/src/caoscrawler/scanner.py @@ -39,7 +39,7 @@ import logging import os import warnings from collections.abc import Callable -from typing import Any, Optional, Type, Union +from typing import Any, Optional, Union import linkahead as db import yaml diff --git a/src/caoscrawler/scripts/generators.py b/src/caoscrawler/scripts/generators.py index ba8e6e39cc03e9be1923d72ec5c8d699c01fa8f9..2bf8a90f5af5086e23b7e7cc35d21a50d8cd511a 100644 --- a/src/caoscrawler/scripts/generators.py +++ b/src/caoscrawler/scripts/generators.py @@ -30,7 +30,6 @@ from typing import Optional import pandas as pd import yaml - DM_TEMPLATE = """# auto-generated data model from file "[]{infile}". # To insert a datamodel into LinkAhead, run: # diff --git a/src/caoscrawler/structure_elements/rocrate_structure_elements.py b/src/caoscrawler/structure_elements/rocrate_structure_elements.py index d39617432fcb63220d3acbb63a618b0445165388..66768ad800128297a27f47d672352f21310703e9 100644 --- a/src/caoscrawler/structure_elements/rocrate_structure_elements.py +++ b/src/caoscrawler/structure_elements/rocrate_structure_elements.py @@ -23,6 +23,7 @@ # from rocrate.model.entity import Entity + from .structure_elements import StructureElement diff --git a/src/caoscrawler/structure_elements/structure_elements.py b/src/caoscrawler/structure_elements/structure_elements.py index 67cd1056b382c92485deada2058526a03b6d8535..3b4c6e9b9d13c61a5924a12d23b11b62edff6924 100644 --- a/src/caoscrawler/structure_elements/structure_elements.py +++ b/src/caoscrawler/structure_elements/structure_elements.py @@ -24,6 +24,7 @@ # import warnings + import lxml.etree diff --git a/src/caoscrawler/sync_graph.py b/src/caoscrawler/sync_graph.py index 9c021a10f35e95ca56d45151b8d064ec905993ec..a05e6320892239cbe8d7f1d9fbd7949a57f9bccb 100644 --- a/src/caoscrawler/sync_graph.py +++ b/src/caoscrawler/sync_graph.py @@ -27,18 +27,17 @@ crawler. from __future__ import annotations import logging -from typing import Any, Optional, Union, Callable +import re +from typing import Any, Callable, Optional, Union import linkahead as db from linkahead.cached import cached_get_entity_by from linkahead.exceptions import EmptyUniqueQueryError -from .identifiable_adapters import IdentifiableAdapter from .identifiable import Identifiable +from .identifiable_adapters import IdentifiableAdapter from .sync_node import SyncNode, TempID -import re - logger = logging.getLogger(__name__) diff --git a/src/caoscrawler/sync_node.py b/src/caoscrawler/sync_node.py index ad0feb98d38750ec46eaced1074bb0fa499acae5..d912d6465a68270411c121f65b4c5a828c9c667e 100644 --- a/src/caoscrawler/sync_node.py +++ b/src/caoscrawler/sync_node.py @@ -22,12 +22,12 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Optional, Union +from typing import TYPE_CHECKING, Any, Optional +from warnings import warn import linkahead as db import yaml from linkahead.common.models import Parent, ParentList, PropertyList -from warnings import warn from .exceptions import ImpossibleMergeError diff --git a/src/caoscrawler/utils.py b/src/caoscrawler/utils.py index d9a5af839068a2582859aad1b51fbc8b9713d5d1..5f736d5ad7550e0b29cb629b2fa140a2f38d6f5f 100644 --- a/src/caoscrawler/utils.py +++ b/src/caoscrawler/utils.py @@ -26,7 +26,6 @@ # Some utility functions, e.g. for extending pylib. import sys - from posixpath import join as posixjoin from typing import Optional from urllib.parse import urljoin diff --git a/src/caoscrawler/version.py b/src/caoscrawler/version.py index 0b72dd65116fbc102a4dc2492d726698cad5a13b..4cd435486aca26e20e785bbbeb65c013d8e727cb 100644 --- a/src/caoscrawler/version.py +++ b/src/caoscrawler/version.py @@ -18,9 +18,10 @@ # along with this program. If not, see <https://www.gnu.org/licenses/>. # from importlib import metadata as importlib_metadata -from packaging.version import parse as parse_version from warnings import warn +from packaging.version import parse as parse_version + def get_caoscrawler_version(): """ Read in version of locally installed caoscrawler package""" diff --git a/src/doc/conf.py b/src/doc/conf.py index c52442aa358d7aa60085c48168d7a63798c967b0..54b2bed270d7f7b72d2f17e33572fbbaf69d8c28 100644 --- a/src/doc/conf.py +++ b/src/doc/conf.py @@ -21,11 +21,11 @@ # import os import sys + sys.path.insert(0, os.path.abspath('..')) import sphinx_rtd_theme # noqa: E402 - # -- Project information ----------------------------------------------------- project = 'caosdb-caoscrawler'