diff --git a/src/caoscrawler/authorize.py b/src/caoscrawler/authorize.py index 6f1011b227881d4b73186996076abe20d94d52e5..f3deed4f8c78afa85fdd4471fe9383760b8c8b12 100644 --- a/src/caoscrawler/authorize.py +++ b/src/caoscrawler/authorize.py @@ -19,10 +19,10 @@ # along with this program. If not, see <https://www.gnu.org/licenses/>. # -from caosadvancedtools.crawler import Crawler as OldCrawler - import argparse +from caosadvancedtools.crawler import Crawler as OldCrawler + def parse_args(): parser = argparse.ArgumentParser() diff --git a/src/caoscrawler/converters/__init__.py b/src/caoscrawler/converters/__init__.py index 70fca6c44c90a3bbb44bd05c34e51cafae91a229..670d4e966c72c6bcf45d0d46c1db715fb79d8ab5 100644 --- a/src/caoscrawler/converters/__init__.py +++ b/src/caoscrawler/converters/__init__.py @@ -32,9 +32,8 @@ except ImportError as err: err=err) try: - from .rocrate import ROCrateEntityConverter - from .rocrate import ROCrateConverter - from .rocrate import ELNFileConverter + from .rocrate import (ELNFileConverter, ROCrateConverter, + ROCrateEntityConverter) except ImportError as err: ROCrateEntityConverter: type = utils.MissingImport( name="ROCrateEntityConverter", hint="Try installing with the `rocrate` extra option.", diff --git a/src/caoscrawler/converters/hdf5_converter.py b/src/caoscrawler/converters/hdf5_converter.py index a4d974bd53fc4b0e22d155f01a6a47295b79e984..97dac53d053dbcb87c48f0cfb59d4f09770b9710 100644 --- a/src/caoscrawler/converters/hdf5_converter.py +++ b/src/caoscrawler/converters/hdf5_converter.py @@ -28,16 +28,16 @@ except ModuleNotFoundError: "its optional `h5-crawler` dependency?" ) -import numpy as np - from typing import Union import linkahead as db +import numpy as np -from .converters import (convert_basic_element, Converter, DictElementConverter, - match_name_and_value, SimpleFileConverter) from ..stores import GeneralStore, RecordStore -from ..structure_elements import DictElement, File, FloatElement, IntegerElement, StructureElement +from ..structure_elements import (DictElement, File, FloatElement, + IntegerElement, StructureElement) +from .converters import (Converter, DictElementConverter, SimpleFileConverter, + convert_basic_element, match_name_and_value) def convert_attributes(elt: Union[h5py.File, h5py.Group, h5py.Dataset]): diff --git a/src/caoscrawler/converters/rocrate.py b/src/caoscrawler/converters/rocrate.py index 286061ef6dbe9c7caf851fe32932dee848ac55d4..b84462acba2fdd7e60094e38edc38605c80deb11 100644 --- a/src/caoscrawler/converters/rocrate.py +++ b/src/caoscrawler/converters/rocrate.py @@ -26,22 +26,21 @@ This converter converts ro-crate files which may also be .eln-files. from __future__ import annotations +import os +import re +import tempfile from typing import Optional +from zipfile import ZipFile +import linkahead as db import rocrate from rocrate.rocrate import ROCrate -import linkahead as db - -from .converters import SimpleFileConverter, ConverterValidationError, Converter, convert_basic_element from ..stores import GeneralStore, RecordStore -from ..structure_elements import (File, Directory, StructureElement, ROCrateEntity) - -from zipfile import ZipFile - -import tempfile -import os -import re +from ..structure_elements import (Directory, File, ROCrateEntity, + StructureElement) +from .converters import (Converter, ConverterValidationError, + SimpleFileConverter, convert_basic_element) class ROCrateConverter(SimpleFileConverter): diff --git a/src/caoscrawler/converters/spss.py b/src/caoscrawler/converters/spss.py index b4f03aeaed6663be98487a4780bb96237e72e27e..00742e91506245435ed0c590f68ea9ffce65717a 100644 --- a/src/caoscrawler/converters/spss.py +++ b/src/caoscrawler/converters/spss.py @@ -22,17 +22,16 @@ from __future__ import annotations # Can be removed with 3.10. import argparse from collections import OrderedDict +from typing import Any, Optional import numpy as np import pandas as pd import pyreadstat import yaml -from . import converters from ..stores import GeneralStore -from ..structure_elements import (File, StructureElement) -from typing import Optional, Any - +from ..structure_elements import File, StructureElement +from . import converters READSTAT_TYPES = { "double": "DOUBLE", diff --git a/src/caoscrawler/converters/xml_converter.py b/src/caoscrawler/converters/xml_converter.py index bd3f6cf0fdcc5fed5b5452da8a17a8a877009b06..b9f7487ee633d0ba25a3b81b78b9a3561274edc9 100644 --- a/src/caoscrawler/converters/xml_converter.py +++ b/src/caoscrawler/converters/xml_converter.py @@ -22,17 +22,17 @@ from __future__ import annotations -import lxml.etree import re - from typing import Optional import linkahead as db +import lxml.etree -from .converters import SimpleFileConverter, ConverterValidationError, Converter from ..stores import GeneralStore, RecordStore -from ..structure_elements import (File, StructureElement, - XMLTagElement, XMLTextNode, XMLAttributeNode) +from ..structure_elements import (File, StructureElement, XMLAttributeNode, + XMLTagElement, XMLTextNode) +from .converters import (Converter, ConverterValidationError, + SimpleFileConverter) class XMLFileConverter(SimpleFileConverter): diff --git a/src/caoscrawler/crawl.py b/src/caoscrawler/crawl.py index 9e4e2a80da40599a51b674ad9a2001b8d9fa691b..a79e4434ee8f58fd1cc2646ced85c0d02d3fb66b 100644 --- a/src/caoscrawler/crawl.py +++ b/src/caoscrawler/crawl.py @@ -39,7 +39,6 @@ import sys import traceback import uuid import warnings - from argparse import RawTextHelpFormatter from copy import deepcopy from datetime import datetime @@ -52,13 +51,10 @@ from caosadvancedtools.cache import UpdateCache from caosadvancedtools.crawler import Crawler as OldCrawler from caosadvancedtools.serverside.helper import send_mail from caosadvancedtools.utils import create_entity_link -from linkahead.apiutils import (compare_entities, - merge_entities) +from linkahead.apiutils import compare_entities, merge_entities from linkahead.cached import cache_clear, cached_get_entity_by from linkahead.common.datatype import get_list_datatype, is_reference -from linkahead.exceptions import ( - TransactionError, -) +from linkahead.exceptions import TransactionError from linkahead.utils.escape import escape_squoted_text from .config import get_config_setting diff --git a/src/caoscrawler/debug_tree.py b/src/caoscrawler/debug_tree.py index 0d57040f5c20aca236a3c11531e8b7c45bad89ab..c154f5b91d850476be0c0610e5bb1dfcbf9866ab 100644 --- a/src/caoscrawler/debug_tree.py +++ b/src/caoscrawler/debug_tree.py @@ -29,35 +29,20 @@ A structure containing debug tree information. from __future__ import annotations -import argparse -import importlib -import logging -import os -import sys -import warnings -import yaml - -from argparse import RawTextHelpFormatter from collections import defaultdict -from copy import deepcopy -from enum import Enum -from importlib_resources import files -from jsonschema import validate -from typing import Any, Optional, Type, Union import linkahead as db - -from caosadvancedtools.cache import UpdateCache, Cache -from caosadvancedtools.crawler import Crawler as OldCrawler -from linkahead.apiutils import (compare_entities, EntityMergeConflictError, +import yaml +from importlib_resources import files +from jsonschema import validate +from linkahead.apiutils import (EntityMergeConflictError, compare_entities, merge_entities) from linkahead.common.datatype import is_reference -from .converters import Converter, DirectoryConverter, ConverterValidationError - +from .converters import Converter, ConverterValidationError, DirectoryConverter from .macros import defmacro_constructor, macro_constructor -from .stores import Store, GeneralStore, RecordStore -from .structure_elements import StructureElement, Directory, NoneElement +from .stores import GeneralStore, RecordStore, Store +from .structure_elements import Directory, NoneElement, StructureElement from .version import check_cfood_version diff --git a/src/caoscrawler/identifiable_adapters.py b/src/caoscrawler/identifiable_adapters.py index 854ee614638712bdcf957c592ef2946dbdd43afc..592f603bef508771d734ff633f8cdb2c100742d5 100644 --- a/src/caoscrawler/identifiable_adapters.py +++ b/src/caoscrawler/identifiable_adapters.py @@ -36,12 +36,8 @@ import yaml from linkahead.cached import cached_get_entity_by, cached_query from linkahead.utils.escape import escape_squoted_text -from .exceptions import ( - InvalidIdentifiableYAML, - MissingIdentifyingProperty, - MissingRecordType, - MissingReferencingEntityError, -) +from .exceptions import (InvalidIdentifiableYAML, MissingIdentifyingProperty, + MissingRecordType, MissingReferencingEntityError) from .identifiable import Identifiable from .sync_node import SyncNode from .utils import has_parent diff --git a/src/caoscrawler/logging.py b/src/caoscrawler/logging.py index f22f9678de768586b04e6781e1448242802406ee..b57a067d8635a468df7345365fabbfae9ee0b22f 100644 --- a/src/caoscrawler/logging.py +++ b/src/caoscrawler/logging.py @@ -20,10 +20,10 @@ # along with this program. If not, see <https://www.gnu.org/licenses/>. import logging +import sys -from caosadvancedtools.webui_formatter import WebUI_Formatter from caosadvancedtools.serverside.helper import get_shared_filename -import sys +from caosadvancedtools.webui_formatter import WebUI_Formatter def configure_server_side_logging(max_log_level: int = logging.INFO): diff --git a/src/caoscrawler/macros/macro_yaml_object.py b/src/caoscrawler/macros/macro_yaml_object.py index d85883011db3cf651da0dda6c110015128fbe439..5d2bc1fe0775499fa8b40a65e115fb4569892e38 100644 --- a/src/caoscrawler/macros/macro_yaml_object.py +++ b/src/caoscrawler/macros/macro_yaml_object.py @@ -26,11 +26,10 @@ # A. Schlemmer, 05/2022 import re -from dataclasses import dataclass -from typing import Any, Dict from copy import deepcopy +from dataclasses import dataclass from string import Template - +from typing import Any, Dict _SAFE_SUBST_PAT = re.compile(r"^\$(?P<key>\w+)$") _SAFE_SUBST_PAT_BRACES = re.compile(r"^\$\{(?P<key>\w+)}$") diff --git a/src/caoscrawler/scanner.py b/src/caoscrawler/scanner.py index 27711e6a7c4e69df3c2d99aca7a427670b153765..89bd1c04411665bf4832d6bccce69bbe1b11cad1 100644 --- a/src/caoscrawler/scanner.py +++ b/src/caoscrawler/scanner.py @@ -39,7 +39,7 @@ import logging import os import warnings from collections.abc import Callable -from typing import Any, Optional, Type, Union +from typing import Any, Optional, Union import linkahead as db import yaml diff --git a/src/caoscrawler/scripts/generators.py b/src/caoscrawler/scripts/generators.py index ba8e6e39cc03e9be1923d72ec5c8d699c01fa8f9..2bf8a90f5af5086e23b7e7cc35d21a50d8cd511a 100644 --- a/src/caoscrawler/scripts/generators.py +++ b/src/caoscrawler/scripts/generators.py @@ -30,7 +30,6 @@ from typing import Optional import pandas as pd import yaml - DM_TEMPLATE = """# auto-generated data model from file "[]{infile}". # To insert a datamodel into LinkAhead, run: # diff --git a/src/caoscrawler/structure_elements/rocrate_structure_elements.py b/src/caoscrawler/structure_elements/rocrate_structure_elements.py index d39617432fcb63220d3acbb63a618b0445165388..66768ad800128297a27f47d672352f21310703e9 100644 --- a/src/caoscrawler/structure_elements/rocrate_structure_elements.py +++ b/src/caoscrawler/structure_elements/rocrate_structure_elements.py @@ -23,6 +23,7 @@ # from rocrate.model.entity import Entity + from .structure_elements import StructureElement diff --git a/src/caoscrawler/structure_elements/structure_elements.py b/src/caoscrawler/structure_elements/structure_elements.py index 67cd1056b382c92485deada2058526a03b6d8535..3b4c6e9b9d13c61a5924a12d23b11b62edff6924 100644 --- a/src/caoscrawler/structure_elements/structure_elements.py +++ b/src/caoscrawler/structure_elements/structure_elements.py @@ -24,6 +24,7 @@ # import warnings + import lxml.etree diff --git a/src/caoscrawler/sync_graph.py b/src/caoscrawler/sync_graph.py index 9c021a10f35e95ca56d45151b8d064ec905993ec..a05e6320892239cbe8d7f1d9fbd7949a57f9bccb 100644 --- a/src/caoscrawler/sync_graph.py +++ b/src/caoscrawler/sync_graph.py @@ -27,18 +27,17 @@ crawler. from __future__ import annotations import logging -from typing import Any, Optional, Union, Callable +import re +from typing import Any, Callable, Optional, Union import linkahead as db from linkahead.cached import cached_get_entity_by from linkahead.exceptions import EmptyUniqueQueryError -from .identifiable_adapters import IdentifiableAdapter from .identifiable import Identifiable +from .identifiable_adapters import IdentifiableAdapter from .sync_node import SyncNode, TempID -import re - logger = logging.getLogger(__name__) diff --git a/src/caoscrawler/sync_node.py b/src/caoscrawler/sync_node.py index ad0feb98d38750ec46eaced1074bb0fa499acae5..d912d6465a68270411c121f65b4c5a828c9c667e 100644 --- a/src/caoscrawler/sync_node.py +++ b/src/caoscrawler/sync_node.py @@ -22,12 +22,12 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Optional, Union +from typing import TYPE_CHECKING, Any, Optional +from warnings import warn import linkahead as db import yaml from linkahead.common.models import Parent, ParentList, PropertyList -from warnings import warn from .exceptions import ImpossibleMergeError diff --git a/src/caoscrawler/utils.py b/src/caoscrawler/utils.py index d9a5af839068a2582859aad1b51fbc8b9713d5d1..5f736d5ad7550e0b29cb629b2fa140a2f38d6f5f 100644 --- a/src/caoscrawler/utils.py +++ b/src/caoscrawler/utils.py @@ -26,7 +26,6 @@ # Some utility functions, e.g. for extending pylib. import sys - from posixpath import join as posixjoin from typing import Optional from urllib.parse import urljoin diff --git a/src/caoscrawler/version.py b/src/caoscrawler/version.py index 0b72dd65116fbc102a4dc2492d726698cad5a13b..4cd435486aca26e20e785bbbeb65c013d8e727cb 100644 --- a/src/caoscrawler/version.py +++ b/src/caoscrawler/version.py @@ -18,9 +18,10 @@ # along with this program. If not, see <https://www.gnu.org/licenses/>. # from importlib import metadata as importlib_metadata -from packaging.version import parse as parse_version from warnings import warn +from packaging.version import parse as parse_version + def get_caoscrawler_version(): """ Read in version of locally installed caoscrawler package""" diff --git a/src/doc/conf.py b/src/doc/conf.py index c52442aa358d7aa60085c48168d7a63798c967b0..54b2bed270d7f7b72d2f17e33572fbbaf69d8c28 100644 --- a/src/doc/conf.py +++ b/src/doc/conf.py @@ -21,11 +21,11 @@ # import os import sys + sys.path.insert(0, os.path.abspath('..')) import sphinx_rtd_theme # noqa: E402 - # -- Project information ----------------------------------------------------- project = 'caosdb-caoscrawler' diff --git a/unittests/test_cfood_metadata.py b/unittests/test_cfood_metadata.py index 494bd383d95b4a845b5ea6f86ccff0f9a1db257f..c606a0a1afcc15d48164694768bae02adfb0fc0b 100644 --- a/unittests/test_cfood_metadata.py +++ b/unittests/test_cfood_metadata.py @@ -17,15 +17,13 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <https://www.gnu.org/licenses/>. # +from tempfile import NamedTemporaryFile +from unittest.mock import MagicMock, Mock, patch + import pytest import yaml -from tempfile import NamedTemporaryFile -from unittest.mock import patch -from unittest.mock import MagicMock, Mock - import caoscrawler - from caoscrawler.scanner import load_definition diff --git a/unittests/test_converters.py b/unittests/test_converters.py index 5b3c34cceea4e2be2b24a869cb3fc3de747ad740..6c7db6ed346fc5e6d0d286024e96ef8828c5c872 100644 --- a/unittests/test_converters.py +++ b/unittests/test_converters.py @@ -29,30 +29,29 @@ import importlib import json import logging import os -import pytest -import sys -import yaml - from itertools import product from pathlib import Path import linkahead as db +import pytest +import yaml from caoscrawler.converters import (Converter, ConverterValidationError, DateElementConverter, DictElementConverter, DictIntegerElementConverter, DirectoryConverter, FloatElementConverter, IntegerElementConverter, JSONFileConverter, - ListElementConverter, MarkdownFileConverter, + ListElementConverter, + MarkdownFileConverter, PropertiesFromDictConverter, - YAMLFileConverter, - handle_value, replace_variables) -from caoscrawler.converters.converters import _AbstractScalarValueElementConverter + YAMLFileConverter, handle_value, + replace_variables) +from caoscrawler.converters.converters import \ + _AbstractScalarValueElementConverter from caoscrawler.crawl import Crawler from caoscrawler.scanner import (_load_definition_from_yaml_dict, create_converter_registry, - create_transformer_registry, - load_definition, + create_transformer_registry, load_definition, scan_structure_elements) from caoscrawler.stores import GeneralStore, RecordStore from caoscrawler.structure_elements import (BooleanElement, DictElement, diff --git a/unittests/test_crawler.py b/unittests/test_crawler.py index aaddec9e8c6b17ad726808bc36b0784adbc3c36d..e88ce454061fb268fa49e986f8392f71296beb07 100644 --- a/unittests/test_crawler.py +++ b/unittests/test_crawler.py @@ -23,7 +23,6 @@ """ test the Crawler class """ -import json import logging import os import warnings @@ -33,12 +32,17 @@ from os.path import basename, dirname, join from pathlib import Path from unittest.mock import MagicMock, Mock, patch -import caoscrawler import linkahead as db import linkahead.common.models as dbmodels import pytest import yaml from caosadvancedtools.models.parser import parse_model_from_string +from linkahead.apiutils import compare_entities +from linkahead.cached import cache_clear +from linkahead.exceptions import EmptyUniqueQueryError +from pytest import raises + +import caoscrawler from caoscrawler.crawl import (Crawler, SecurityMode, _treat_deprecated_prefix, crawler_main, split_restricted_path) from caoscrawler.debug_tree import DebugTree @@ -55,10 +59,6 @@ from caoscrawler.stores import GeneralStore, RecordStore from caoscrawler.structure_elements import (DictElement, DictListElement, DictTextElement, File) from caoscrawler.sync_graph import SyncGraph -from linkahead.apiutils import compare_entities -from linkahead.cached import cache_clear -from linkahead.exceptions import EmptyUniqueQueryError -from pytest import raises UNITTESTDIR = Path(__file__).parent diff --git a/unittests/test_entity_comparison.py b/unittests/test_entity_comparison.py index 0f62475b6c61d82feb3e550cf5ab53e91183f80a..8543732fde4d584e2022dcf6432e9572ae625eb5 100644 --- a/unittests/test_entity_comparison.py +++ b/unittests/test_entity_comparison.py @@ -3,7 +3,6 @@ # A. Schlemmer, 06/2021 import linkahead as db - import pytest from pytest import raises diff --git a/unittests/test_h5_converter.py b/unittests/test_h5_converter.py index 95060451badb0523cf91c70e5be345e35ec3964d..9c1058812c75c6d1e5ee7028c8f6fccd7081a54c 100644 --- a/unittests/test_h5_converter.py +++ b/unittests/test_h5_converter.py @@ -17,22 +17,21 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <https://www.gnu.org/licenses/>. # -import numpy as np - from functools import partial from pathlib import Path -from pytest import fixture, importorskip import linkahead as db +import numpy as np +from pytest import fixture, importorskip +from utils import dircheckstr as dircheck_base +from caoscrawler.converters.hdf5_converter import ( + H5DatasetElement, H5GroupElement, H5NdarrayElement, + convert_basic_element_with_nd_array, convert_h5_element) from caoscrawler.debug_tree import DebugTree -from caoscrawler.converters.hdf5_converter import (convert_basic_element_with_nd_array, - convert_h5_element, H5GroupElement, - H5DatasetElement, H5NdarrayElement) from caoscrawler.scanner import scan_directory from caoscrawler.structure_elements import (FloatElement, ListElement, TextElement) -from utils import dircheckstr as dircheck_base # Skip the whole module if h5py hasn't been installed h5py = importorskip("h5py") diff --git a/unittests/test_identifiable.py b/unittests/test_identifiable.py index d94d852583523a3b3f29f002eaacb9ae0b616c4f..44aac6a3edd40e0df8558f68083e22245ff58127 100644 --- a/unittests/test_identifiable.py +++ b/unittests/test_identifiable.py @@ -26,6 +26,7 @@ test identifiable module import linkahead as db import pytest + from caoscrawler.identifiable import Identifiable from caoscrawler.sync_node import SyncNode diff --git a/unittests/test_identifiable_adapters.py b/unittests/test_identifiable_adapters.py index d2a5c3bb5d4007348b48d513e664c39592dc4c61..bdc0ab850d1a8253e876e8b1a6bc621327802f79 100644 --- a/unittests/test_identifiable_adapters.py +++ b/unittests/test_identifiable_adapters.py @@ -27,15 +27,14 @@ test identifiable_adapters module """ -import os from datetime import datetime -from unittest.mock import MagicMock, Mock, patch from pathlib import Path +from unittest.mock import MagicMock, Mock, patch import linkahead as db import pytest -from caoscrawler.exceptions import (InvalidIdentifiableYAML, - ) + +from caoscrawler.exceptions import InvalidIdentifiableYAML from caoscrawler.identifiable import Identifiable from caoscrawler.identifiable_adapters import (CaosDBIdentifiableAdapter, IdentifiableAdapter, diff --git a/unittests/test_issues.py b/unittests/test_issues.py index 1678280555e739bae55819fa7fe42a53c938c4e5..a6de65400f42018c3fdcde7b2f29d4fd200bf62b 100644 --- a/unittests/test_issues.py +++ b/unittests/test_issues.py @@ -22,11 +22,12 @@ from pytest import mark -from caoscrawler.converters import replace_variables, CrawlerTemplate +from caoscrawler.converters import CrawlerTemplate, replace_variables from caoscrawler.crawl import Crawler -from caoscrawler.structure_elements import DictElement +from caoscrawler.scanner import (create_converter_registry, + scan_structure_elements) from caoscrawler.stores import GeneralStore -from caoscrawler.scanner import create_converter_registry, scan_structure_elements +from caoscrawler.structure_elements import DictElement def test_issue_10(): diff --git a/unittests/test_json.py b/unittests/test_json.py index be65a26ea01e11e11968bd927c80513708e73850..5d145b38fd36fa2de4e4ab754cbadda0fff6eff7 100644 --- a/unittests/test_json.py +++ b/unittests/test_json.py @@ -26,18 +26,17 @@ """ test the JSON converter """ -import json import os - -from pytest import raises +from pathlib import Path import linkahead as db +from pytest import raises from caoscrawler.converters import JSONFileConverter -from pathlib import Path from caoscrawler.crawl import Crawler +from caoscrawler.scanner import (create_converter_registry, load_definition, + scan_structure_elements) from caoscrawler.structure_elements import File, JSONFile -from caoscrawler.scanner import load_definition, create_converter_registry, scan_structure_elements UNITTESTDIR = Path(__file__).parent diff --git a/unittests/test_macros.py b/unittests/test_macros.py index cfa405e5041fb4324b7de98ffcb942cf4b040715..a87b633e8585a03431575426733cae6ba31b7acf 100644 --- a/unittests/test_macros.py +++ b/unittests/test_macros.py @@ -22,15 +22,15 @@ # ** end header # -from caoscrawler.macros import defmacro_constructor, macro_constructor -from caoscrawler.macros.macro_yaml_object import macro_store -from caoscrawler.crawl import Crawler -from caoscrawler.scanner import load_definition - from tempfile import NamedTemporaryFile -import yaml import pytest +import yaml + +from caoscrawler.crawl import Crawler +from caoscrawler.macros import defmacro_constructor, macro_constructor +from caoscrawler.macros.macro_yaml_object import macro_store +from caoscrawler.scanner import load_definition @pytest.fixture diff --git a/unittests/test_rocrate_converter.py b/unittests/test_rocrate_converter.py index 16cfc3a3c3aec811219da7006d4722d9abf6dcf7..ef59a37c7a9ca91f85d3a62b4f5b6f5c12559575 100644 --- a/unittests/test_rocrate_converter.py +++ b/unittests/test_rocrate_converter.py @@ -24,33 +24,25 @@ test the XML converters """ import importlib -import json -import pytest -import sys -import yaml import os - -from lxml.etree import fromstring from pathlib import Path -from rocrate.rocrate import ROCrate -from rocrate.model.entity import Entity +import jsonschema +import linkahead as db +import pytest import rocrate +import yaml +from linkahead.high_level_api import convert_to_python_object +from lxml.etree import fromstring +from rocrate.model.entity import Entity +from rocrate.rocrate import ROCrate -from caoscrawler.converters import (ELNFileConverter, ROCrateEntityConverter) +from caoscrawler import scanner +from caoscrawler.converters import ELNFileConverter, ROCrateEntityConverter from caoscrawler.scanner import load_definition from caoscrawler.stores import GeneralStore -from caoscrawler.structure_elements import ROCrateEntity, File, TextElement, DictElement - -from caoscrawler import scanner -from caosadvancedtools.json_schema_exporter import recordtype_to_json_schema -from caosadvancedtools.models.parser import parse_model_from_yaml -from linkahead.high_level_api import convert_to_python_object -import json -import jsonschema - -import linkahead as db - +from caoscrawler.structure_elements import (DictElement, File, ROCrateEntity, + TextElement) UNITTESTDIR = Path(__file__).parent diff --git a/unittests/test_scalars_cfood.py b/unittests/test_scalars_cfood.py index 4375ba199d64c3a24d07b3ea1cc4d221d967954b..577fcd5f6c93bee2bc05451983d358aa2e07f798 100644 --- a/unittests/test_scalars_cfood.py +++ b/unittests/test_scalars_cfood.py @@ -2,10 +2,11 @@ # Tests for: # https://gitlab.com/caosdb/caosdb-crawler/-/issues/9 # A. Schlemmer, 06/2021 -import os from pathlib import Path import pytest +from utils import dircheckstr + # The main function that is affected by this issue: from caoscrawler.converters import handle_value from caoscrawler.crawl import Crawler @@ -14,8 +15,6 @@ from caoscrawler.scanner import scan_directory # We need the store for the above function from caoscrawler.stores import GeneralStore -from utils import dircheckstr - UNITTESTDIR = Path(__file__).parent diff --git a/unittests/test_scanner.py b/unittests/test_scanner.py index 680e4abe0da1896e63dba4636c05a38b90de2dc5..5cbbc63406ffb3f5ec1f9019ed7877d7880d7b69 100644 --- a/unittests/test_scanner.py +++ b/unittests/test_scanner.py @@ -34,16 +34,16 @@ from unittest.mock import MagicMock, Mock, patch import linkahead as db import pytest import yaml +from pytest import raises +from utils import dircheckstr as dircheck_base + from caoscrawler.crawl import Crawler from caoscrawler.debug_tree import DebugTree -from caoscrawler.scanner import (create_converter_registry, load_definition, - scan_directory, scan_structure_elements, - _load_definition_from_yaml_dict) +from caoscrawler.scanner import (_load_definition_from_yaml_dict, + create_converter_registry, load_definition, + scan_directory, scan_structure_elements) from caoscrawler.structure_elements import (DictElement, DictListElement, DictTextElement, File) -from pytest import raises - -from utils import dircheckstr as dircheck_base UNITTESTDIR = Path(__file__).parent diff --git a/unittests/test_schema.py b/unittests/test_schema.py index ea8549b0b8dfd1f1af35784082a9e46320cfcff4..96c388ac362583eda13ca368519467c34446868e 100644 --- a/unittests/test_schema.py +++ b/unittests/test_schema.py @@ -2,17 +2,15 @@ # Tests for schema validation # A. Schlemmer, 06/2021 -from importlib_resources import files -import linkahead as db - -from os.path import join, dirname -from caoscrawler import Crawler +from os.path import dirname, join +import linkahead as db import pytest -from pytest import raises - +from importlib_resources import files from jsonschema.exceptions import ValidationError +from pytest import raises +from caoscrawler import Crawler from caoscrawler.scanner import load_definition diff --git a/unittests/test_spss_converter.py b/unittests/test_spss_converter.py index 7ffc18dba43a6f7cd3c9fbc9273da349b4ec3c6e..59fe723849dadcda21a699416372f08f2756f4e1 100644 --- a/unittests/test_spss_converter.py +++ b/unittests/test_spss_converter.py @@ -20,16 +20,12 @@ import datetime import importlib -import re from pathlib import Path import numpy as np import pytest -from caoscrawler.converters import ( - ConverterValidationError, - SPSSConverter, -) +from caoscrawler.converters import ConverterValidationError, SPSSConverter from caoscrawler.structure_elements import (BooleanElement, DictElement, Directory, File, FloatElement, IntegerElement, ListElement, diff --git a/unittests/test_sync_graph.py b/unittests/test_sync_graph.py index 84451790ddd02f90c2a12a3ce7280b17d8f7c73b..06f0dfb9eb3d3536d26dcfd354ca27f08ef99a02 100644 --- a/unittests/test_sync_graph.py +++ b/unittests/test_sync_graph.py @@ -21,25 +21,21 @@ import logging from functools import partial +from itertools import product from unittest.mock import MagicMock, Mock, patch import linkahead as db import pytest from test_crawler import (basic_retrieve_by_name_mock_up, - mock_cached_only_rt_allow_empty, - mock_get_entity_by, - ) + mock_cached_only_rt_allow_empty, mock_get_entity_by) from caoscrawler.exceptions import (MissingIdentifyingProperty, - MissingRecordType, - ) + MissingRecordType) from caoscrawler.identifiable import Identifiable from caoscrawler.identifiable_adapters import CaosDBIdentifiableAdapter from caoscrawler.sync_graph import SyncGraph, _set_each_scalar_value from caoscrawler.sync_node import SyncNode, parent_in_list, property_in_list -from itertools import product - @pytest.fixture def simple_adapter(): diff --git a/unittests/test_sync_node.py b/unittests/test_sync_node.py index bd9e1a6ccbc2ac9ec9ccace96e0ec0422ba1d95b..1f95551d34f9e06ab3e2fc196e1e7809eabfa019 100644 --- a/unittests/test_sync_node.py +++ b/unittests/test_sync_node.py @@ -18,19 +18,18 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <https://www.gnu.org/licenses/>. # -from functools import partial from unittest.mock import MagicMock, Mock, patch import linkahead as db import pytest +from test_crawler import basic_retrieve_by_name_mock_up, mock_get_entity_by + from caoscrawler.exceptions import ImpossibleMergeError from caoscrawler.identifiable import Identifiable from caoscrawler.identifiable_adapters import CaosDBIdentifiableAdapter from caoscrawler.sync_graph import SyncGraph from caoscrawler.sync_node import SyncNode, parent_in_list, property_in_list -from test_crawler import basic_retrieve_by_name_mock_up, mock_get_entity_by - def assert_parents_equal(p1, p2): """Special assertion for comparing parents.""" diff --git a/unittests/test_table_converter.py b/unittests/test_table_converter.py index 3b563fd3179968fd90b1c92b9bc5bf0db9ed0858..c606c1d3cdf9a95f00728eaae88153631b08af53 100644 --- a/unittests/test_table_converter.py +++ b/unittests/test_table_converter.py @@ -28,12 +28,13 @@ test the converters module import importlib import math -import os from os.path import basename, dirname, join from pathlib import Path import linkahead as db import pytest +from utils import dircheckstr + from caoscrawler import Crawler from caoscrawler.converters import (Converter, ConverterValidationError, CSVTableConverter, DictConverter, @@ -48,8 +49,6 @@ from caoscrawler.structure_elements import (BooleanElement, DictElement, IntegerElement, ListElement, TextElement) -from utils import dircheckstr - UNITTESTDIR = Path(__file__).parent diff --git a/unittests/test_transformers.py b/unittests/test_transformers.py index 4ed12751d9052c839aa4db4abd586c419bed1018..0571dbd31de9b37230f0ee1d93c22c6df47c87e7 100644 --- a/unittests/test_transformers.py +++ b/unittests/test_transformers.py @@ -29,19 +29,18 @@ See: https://gitlab.indiscale.com/caosdb/src/caosdb-crawler/-/issues/107 """ import importlib -from functools import partial from pathlib import Path -from tempfile import NamedTemporaryFile from unittest.mock import MagicMock, Mock, patch import linkahead as db import pytest import yaml +from pytest import raises + from caoscrawler.converters import Converter, ListElementConverter from caoscrawler.scanner import create_transformer_registry, scan_directory from caoscrawler.stores import GeneralStore from caoscrawler.transformer_functions import replace, split -from pytest import raises UNITTESTDIR = Path(__file__).parent diff --git a/unittests/test_utilities.py b/unittests/test_utilities.py index 15e84a609149ac602ee80b7357f7622566563792..463e304a99161f2294e5d202611dcf0b829e2045 100644 --- a/unittests/test_utilities.py +++ b/unittests/test_utilities.py @@ -22,7 +22,7 @@ import pytest from caoscrawler.crawl import split_restricted_path -from caoscrawler.utils import get_shared_resource_link, MissingImport +from caoscrawler.utils import MissingImport, get_shared_resource_link def test_split_restricted_path(): diff --git a/unittests/test_variable_substitutions.py b/unittests/test_variable_substitutions.py index 90d144b04a4e1271f74b769759e3f201007af705..c75e37956c1ec24e47ff9cbd9b03572ed4a0f80e 100644 --- a/unittests/test_variable_substitutions.py +++ b/unittests/test_variable_substitutions.py @@ -19,7 +19,6 @@ # along with this program. If not, see <https://www.gnu.org/licenses/>. # -from copy import deepcopy from functools import partial from os.path import basename, dirname, join from pathlib import Path @@ -28,6 +27,10 @@ from unittest.mock import MagicMock, Mock import linkahead as db import pytest import yaml +from linkahead.apiutils import compare_entities +from pytest import raises +from utils import dircheckstr as dircheckstr_base + from caoscrawler import Crawler from caoscrawler.debug_tree import DebugTree from caoscrawler.identifiable_adapters import (IdentifiableAdapter, @@ -35,10 +38,6 @@ from caoscrawler.identifiable_adapters import (IdentifiableAdapter, from caoscrawler.scanner import scan_directory from caoscrawler.structure_elements import (DictListElement, DictTextElement, File) -from linkahead.apiutils import compare_entities -from pytest import raises - -from utils import dircheckstr as dircheckstr_base UNITTESTDIR = Path(__file__).parent dircheckstr = partial(dircheckstr_base, UNITTESTDIR / "test_directories" / diff --git a/unittests/test_xml_converter.py b/unittests/test_xml_converter.py index 9fc9749ccececd41d460fe297edfea72cc30a5ef..e8869ef6ffad511159a583a14fd49d2fad48766b 100644 --- a/unittests/test_xml_converter.py +++ b/unittests/test_xml_converter.py @@ -24,22 +24,18 @@ test the XML converters """ import importlib -import json +from pathlib import Path + import pytest -import sys import yaml - from lxml.etree import fromstring -from pathlib import Path -from caoscrawler.converters import (XMLTagConverter, - XMLAttributeNodeConverter, +from caoscrawler.converters import (XMLAttributeNodeConverter, XMLTagConverter, XMLTextNodeConverter) from caoscrawler.scanner import load_definition from caoscrawler.stores import GeneralStore from caoscrawler.structure_elements import XMLTagElement - UNITTESTDIR = Path(__file__).parent