diff --git a/src/caoscrawler/converters/rocrate.py b/src/caoscrawler/converters/rocrate.py index 286061ef6dbe9c7caf851fe32932dee848ac55d4..189d936cebe4bee6213005adbee50e36b63b2898 100644 --- a/src/caoscrawler/converters/rocrate.py +++ b/src/caoscrawler/converters/rocrate.py @@ -26,22 +26,22 @@ This converter converts ro-crate files which may also be .eln-files. from __future__ import annotations +import os +import re +import tempfile from typing import Optional +from zipfile import ZipFile + +import linkahead as db import rocrate from rocrate.rocrate import ROCrate -import linkahead as db - -from .converters import SimpleFileConverter, ConverterValidationError, Converter, convert_basic_element from ..stores import GeneralStore, RecordStore -from ..structure_elements import (File, Directory, StructureElement, ROCrateEntity) - -from zipfile import ZipFile - -import tempfile -import os -import re +from ..structure_elements import (Directory, File, ROCrateEntity, + StructureElement) +from .converters import (Converter, ConverterValidationError, + SimpleFileConverter, convert_basic_element) class ROCrateConverter(SimpleFileConverter): diff --git a/unittests/test_cfood_metadata.py b/unittests/test_cfood_metadata.py index 494bd383d95b4a845b5ea6f86ccff0f9a1db257f..c606a0a1afcc15d48164694768bae02adfb0fc0b 100644 --- a/unittests/test_cfood_metadata.py +++ b/unittests/test_cfood_metadata.py @@ -17,15 +17,13 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <https://www.gnu.org/licenses/>. # +from tempfile import NamedTemporaryFile +from unittest.mock import MagicMock, Mock, patch + import pytest import yaml -from tempfile import NamedTemporaryFile -from unittest.mock import patch -from unittest.mock import MagicMock, Mock - import caoscrawler - from caoscrawler.scanner import load_definition diff --git a/unittests/test_converters.py b/unittests/test_converters.py index 5b3c34cceea4e2be2b24a869cb3fc3de747ad740..6c7db6ed346fc5e6d0d286024e96ef8828c5c872 100644 --- a/unittests/test_converters.py +++ b/unittests/test_converters.py @@ -29,30 +29,29 @@ import importlib import json import logging import os -import pytest -import sys -import yaml - from itertools import product from pathlib import Path import linkahead as db +import pytest +import yaml from caoscrawler.converters import (Converter, ConverterValidationError, DateElementConverter, DictElementConverter, DictIntegerElementConverter, DirectoryConverter, FloatElementConverter, IntegerElementConverter, JSONFileConverter, - ListElementConverter, MarkdownFileConverter, + ListElementConverter, + MarkdownFileConverter, PropertiesFromDictConverter, - YAMLFileConverter, - handle_value, replace_variables) -from caoscrawler.converters.converters import _AbstractScalarValueElementConverter + YAMLFileConverter, handle_value, + replace_variables) +from caoscrawler.converters.converters import \ + _AbstractScalarValueElementConverter from caoscrawler.crawl import Crawler from caoscrawler.scanner import (_load_definition_from_yaml_dict, create_converter_registry, - create_transformer_registry, - load_definition, + create_transformer_registry, load_definition, scan_structure_elements) from caoscrawler.stores import GeneralStore, RecordStore from caoscrawler.structure_elements import (BooleanElement, DictElement, diff --git a/unittests/test_crawler.py b/unittests/test_crawler.py index aaddec9e8c6b17ad726808bc36b0784adbc3c36d..e88ce454061fb268fa49e986f8392f71296beb07 100644 --- a/unittests/test_crawler.py +++ b/unittests/test_crawler.py @@ -23,7 +23,6 @@ """ test the Crawler class """ -import json import logging import os import warnings @@ -33,12 +32,17 @@ from os.path import basename, dirname, join from pathlib import Path from unittest.mock import MagicMock, Mock, patch -import caoscrawler import linkahead as db import linkahead.common.models as dbmodels import pytest import yaml from caosadvancedtools.models.parser import parse_model_from_string +from linkahead.apiutils import compare_entities +from linkahead.cached import cache_clear +from linkahead.exceptions import EmptyUniqueQueryError +from pytest import raises + +import caoscrawler from caoscrawler.crawl import (Crawler, SecurityMode, _treat_deprecated_prefix, crawler_main, split_restricted_path) from caoscrawler.debug_tree import DebugTree @@ -55,10 +59,6 @@ from caoscrawler.stores import GeneralStore, RecordStore from caoscrawler.structure_elements import (DictElement, DictListElement, DictTextElement, File) from caoscrawler.sync_graph import SyncGraph -from linkahead.apiutils import compare_entities -from linkahead.cached import cache_clear -from linkahead.exceptions import EmptyUniqueQueryError -from pytest import raises UNITTESTDIR = Path(__file__).parent diff --git a/unittests/test_entity_comparison.py b/unittests/test_entity_comparison.py index 0f62475b6c61d82feb3e550cf5ab53e91183f80a..8543732fde4d584e2022dcf6432e9572ae625eb5 100644 --- a/unittests/test_entity_comparison.py +++ b/unittests/test_entity_comparison.py @@ -3,7 +3,6 @@ # A. Schlemmer, 06/2021 import linkahead as db - import pytest from pytest import raises diff --git a/unittests/test_h5_converter.py b/unittests/test_h5_converter.py index 95060451badb0523cf91c70e5be345e35ec3964d..9c1058812c75c6d1e5ee7028c8f6fccd7081a54c 100644 --- a/unittests/test_h5_converter.py +++ b/unittests/test_h5_converter.py @@ -17,22 +17,21 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <https://www.gnu.org/licenses/>. # -import numpy as np - from functools import partial from pathlib import Path -from pytest import fixture, importorskip import linkahead as db +import numpy as np +from pytest import fixture, importorskip +from utils import dircheckstr as dircheck_base +from caoscrawler.converters.hdf5_converter import ( + H5DatasetElement, H5GroupElement, H5NdarrayElement, + convert_basic_element_with_nd_array, convert_h5_element) from caoscrawler.debug_tree import DebugTree -from caoscrawler.converters.hdf5_converter import (convert_basic_element_with_nd_array, - convert_h5_element, H5GroupElement, - H5DatasetElement, H5NdarrayElement) from caoscrawler.scanner import scan_directory from caoscrawler.structure_elements import (FloatElement, ListElement, TextElement) -from utils import dircheckstr as dircheck_base # Skip the whole module if h5py hasn't been installed h5py = importorskip("h5py") diff --git a/unittests/test_identifiable.py b/unittests/test_identifiable.py index d94d852583523a3b3f29f002eaacb9ae0b616c4f..44aac6a3edd40e0df8558f68083e22245ff58127 100644 --- a/unittests/test_identifiable.py +++ b/unittests/test_identifiable.py @@ -26,6 +26,7 @@ test identifiable module import linkahead as db import pytest + from caoscrawler.identifiable import Identifiable from caoscrawler.sync_node import SyncNode diff --git a/unittests/test_identifiable_adapters.py b/unittests/test_identifiable_adapters.py index d2a5c3bb5d4007348b48d513e664c39592dc4c61..bdc0ab850d1a8253e876e8b1a6bc621327802f79 100644 --- a/unittests/test_identifiable_adapters.py +++ b/unittests/test_identifiable_adapters.py @@ -27,15 +27,14 @@ test identifiable_adapters module """ -import os from datetime import datetime -from unittest.mock import MagicMock, Mock, patch from pathlib import Path +from unittest.mock import MagicMock, Mock, patch import linkahead as db import pytest -from caoscrawler.exceptions import (InvalidIdentifiableYAML, - ) + +from caoscrawler.exceptions import InvalidIdentifiableYAML from caoscrawler.identifiable import Identifiable from caoscrawler.identifiable_adapters import (CaosDBIdentifiableAdapter, IdentifiableAdapter, diff --git a/unittests/test_issues.py b/unittests/test_issues.py index 1678280555e739bae55819fa7fe42a53c938c4e5..a6de65400f42018c3fdcde7b2f29d4fd200bf62b 100644 --- a/unittests/test_issues.py +++ b/unittests/test_issues.py @@ -22,11 +22,12 @@ from pytest import mark -from caoscrawler.converters import replace_variables, CrawlerTemplate +from caoscrawler.converters import CrawlerTemplate, replace_variables from caoscrawler.crawl import Crawler -from caoscrawler.structure_elements import DictElement +from caoscrawler.scanner import (create_converter_registry, + scan_structure_elements) from caoscrawler.stores import GeneralStore -from caoscrawler.scanner import create_converter_registry, scan_structure_elements +from caoscrawler.structure_elements import DictElement def test_issue_10(): diff --git a/unittests/test_json.py b/unittests/test_json.py index be65a26ea01e11e11968bd927c80513708e73850..5d145b38fd36fa2de4e4ab754cbadda0fff6eff7 100644 --- a/unittests/test_json.py +++ b/unittests/test_json.py @@ -26,18 +26,17 @@ """ test the JSON converter """ -import json import os - -from pytest import raises +from pathlib import Path import linkahead as db +from pytest import raises from caoscrawler.converters import JSONFileConverter -from pathlib import Path from caoscrawler.crawl import Crawler +from caoscrawler.scanner import (create_converter_registry, load_definition, + scan_structure_elements) from caoscrawler.structure_elements import File, JSONFile -from caoscrawler.scanner import load_definition, create_converter_registry, scan_structure_elements UNITTESTDIR = Path(__file__).parent diff --git a/unittests/test_macros.py b/unittests/test_macros.py index cfa405e5041fb4324b7de98ffcb942cf4b040715..a87b633e8585a03431575426733cae6ba31b7acf 100644 --- a/unittests/test_macros.py +++ b/unittests/test_macros.py @@ -22,15 +22,15 @@ # ** end header # -from caoscrawler.macros import defmacro_constructor, macro_constructor -from caoscrawler.macros.macro_yaml_object import macro_store -from caoscrawler.crawl import Crawler -from caoscrawler.scanner import load_definition - from tempfile import NamedTemporaryFile -import yaml import pytest +import yaml + +from caoscrawler.crawl import Crawler +from caoscrawler.macros import defmacro_constructor, macro_constructor +from caoscrawler.macros.macro_yaml_object import macro_store +from caoscrawler.scanner import load_definition @pytest.fixture diff --git a/unittests/test_rocrate_converter.py b/unittests/test_rocrate_converter.py index 16cfc3a3c3aec811219da7006d4722d9abf6dcf7..ef59a37c7a9ca91f85d3a62b4f5b6f5c12559575 100644 --- a/unittests/test_rocrate_converter.py +++ b/unittests/test_rocrate_converter.py @@ -24,33 +24,25 @@ test the XML converters """ import importlib -import json -import pytest -import sys -import yaml import os - -from lxml.etree import fromstring from pathlib import Path -from rocrate.rocrate import ROCrate -from rocrate.model.entity import Entity +import jsonschema +import linkahead as db +import pytest import rocrate +import yaml +from linkahead.high_level_api import convert_to_python_object +from lxml.etree import fromstring +from rocrate.model.entity import Entity +from rocrate.rocrate import ROCrate -from caoscrawler.converters import (ELNFileConverter, ROCrateEntityConverter) +from caoscrawler import scanner +from caoscrawler.converters import ELNFileConverter, ROCrateEntityConverter from caoscrawler.scanner import load_definition from caoscrawler.stores import GeneralStore -from caoscrawler.structure_elements import ROCrateEntity, File, TextElement, DictElement - -from caoscrawler import scanner -from caosadvancedtools.json_schema_exporter import recordtype_to_json_schema -from caosadvancedtools.models.parser import parse_model_from_yaml -from linkahead.high_level_api import convert_to_python_object -import json -import jsonschema - -import linkahead as db - +from caoscrawler.structure_elements import (DictElement, File, ROCrateEntity, + TextElement) UNITTESTDIR = Path(__file__).parent diff --git a/unittests/test_scalars_cfood.py b/unittests/test_scalars_cfood.py index 4375ba199d64c3a24d07b3ea1cc4d221d967954b..577fcd5f6c93bee2bc05451983d358aa2e07f798 100644 --- a/unittests/test_scalars_cfood.py +++ b/unittests/test_scalars_cfood.py @@ -2,10 +2,11 @@ # Tests for: # https://gitlab.com/caosdb/caosdb-crawler/-/issues/9 # A. Schlemmer, 06/2021 -import os from pathlib import Path import pytest +from utils import dircheckstr + # The main function that is affected by this issue: from caoscrawler.converters import handle_value from caoscrawler.crawl import Crawler @@ -14,8 +15,6 @@ from caoscrawler.scanner import scan_directory # We need the store for the above function from caoscrawler.stores import GeneralStore -from utils import dircheckstr - UNITTESTDIR = Path(__file__).parent diff --git a/unittests/test_scanner.py b/unittests/test_scanner.py index 680e4abe0da1896e63dba4636c05a38b90de2dc5..5cbbc63406ffb3f5ec1f9019ed7877d7880d7b69 100644 --- a/unittests/test_scanner.py +++ b/unittests/test_scanner.py @@ -34,16 +34,16 @@ from unittest.mock import MagicMock, Mock, patch import linkahead as db import pytest import yaml +from pytest import raises +from utils import dircheckstr as dircheck_base + from caoscrawler.crawl import Crawler from caoscrawler.debug_tree import DebugTree -from caoscrawler.scanner import (create_converter_registry, load_definition, - scan_directory, scan_structure_elements, - _load_definition_from_yaml_dict) +from caoscrawler.scanner import (_load_definition_from_yaml_dict, + create_converter_registry, load_definition, + scan_directory, scan_structure_elements) from caoscrawler.structure_elements import (DictElement, DictListElement, DictTextElement, File) -from pytest import raises - -from utils import dircheckstr as dircheck_base UNITTESTDIR = Path(__file__).parent diff --git a/unittests/test_schema.py b/unittests/test_schema.py index ea8549b0b8dfd1f1af35784082a9e46320cfcff4..96c388ac362583eda13ca368519467c34446868e 100644 --- a/unittests/test_schema.py +++ b/unittests/test_schema.py @@ -2,17 +2,15 @@ # Tests for schema validation # A. Schlemmer, 06/2021 -from importlib_resources import files -import linkahead as db - -from os.path import join, dirname -from caoscrawler import Crawler +from os.path import dirname, join +import linkahead as db import pytest -from pytest import raises - +from importlib_resources import files from jsonschema.exceptions import ValidationError +from pytest import raises +from caoscrawler import Crawler from caoscrawler.scanner import load_definition diff --git a/unittests/test_spss_converter.py b/unittests/test_spss_converter.py index 7ffc18dba43a6f7cd3c9fbc9273da349b4ec3c6e..59fe723849dadcda21a699416372f08f2756f4e1 100644 --- a/unittests/test_spss_converter.py +++ b/unittests/test_spss_converter.py @@ -20,16 +20,12 @@ import datetime import importlib -import re from pathlib import Path import numpy as np import pytest -from caoscrawler.converters import ( - ConverterValidationError, - SPSSConverter, -) +from caoscrawler.converters import ConverterValidationError, SPSSConverter from caoscrawler.structure_elements import (BooleanElement, DictElement, Directory, File, FloatElement, IntegerElement, ListElement, diff --git a/unittests/test_sync_graph.py b/unittests/test_sync_graph.py index 84451790ddd02f90c2a12a3ce7280b17d8f7c73b..06f0dfb9eb3d3536d26dcfd354ca27f08ef99a02 100644 --- a/unittests/test_sync_graph.py +++ b/unittests/test_sync_graph.py @@ -21,25 +21,21 @@ import logging from functools import partial +from itertools import product from unittest.mock import MagicMock, Mock, patch import linkahead as db import pytest from test_crawler import (basic_retrieve_by_name_mock_up, - mock_cached_only_rt_allow_empty, - mock_get_entity_by, - ) + mock_cached_only_rt_allow_empty, mock_get_entity_by) from caoscrawler.exceptions import (MissingIdentifyingProperty, - MissingRecordType, - ) + MissingRecordType) from caoscrawler.identifiable import Identifiable from caoscrawler.identifiable_adapters import CaosDBIdentifiableAdapter from caoscrawler.sync_graph import SyncGraph, _set_each_scalar_value from caoscrawler.sync_node import SyncNode, parent_in_list, property_in_list -from itertools import product - @pytest.fixture def simple_adapter(): diff --git a/unittests/test_sync_node.py b/unittests/test_sync_node.py index bd9e1a6ccbc2ac9ec9ccace96e0ec0422ba1d95b..1f95551d34f9e06ab3e2fc196e1e7809eabfa019 100644 --- a/unittests/test_sync_node.py +++ b/unittests/test_sync_node.py @@ -18,19 +18,18 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see <https://www.gnu.org/licenses/>. # -from functools import partial from unittest.mock import MagicMock, Mock, patch import linkahead as db import pytest +from test_crawler import basic_retrieve_by_name_mock_up, mock_get_entity_by + from caoscrawler.exceptions import ImpossibleMergeError from caoscrawler.identifiable import Identifiable from caoscrawler.identifiable_adapters import CaosDBIdentifiableAdapter from caoscrawler.sync_graph import SyncGraph from caoscrawler.sync_node import SyncNode, parent_in_list, property_in_list -from test_crawler import basic_retrieve_by_name_mock_up, mock_get_entity_by - def assert_parents_equal(p1, p2): """Special assertion for comparing parents.""" diff --git a/unittests/test_table_converter.py b/unittests/test_table_converter.py index 3b563fd3179968fd90b1c92b9bc5bf0db9ed0858..c606c1d3cdf9a95f00728eaae88153631b08af53 100644 --- a/unittests/test_table_converter.py +++ b/unittests/test_table_converter.py @@ -28,12 +28,13 @@ test the converters module import importlib import math -import os from os.path import basename, dirname, join from pathlib import Path import linkahead as db import pytest +from utils import dircheckstr + from caoscrawler import Crawler from caoscrawler.converters import (Converter, ConverterValidationError, CSVTableConverter, DictConverter, @@ -48,8 +49,6 @@ from caoscrawler.structure_elements import (BooleanElement, DictElement, IntegerElement, ListElement, TextElement) -from utils import dircheckstr - UNITTESTDIR = Path(__file__).parent diff --git a/unittests/test_transformers.py b/unittests/test_transformers.py index 4ed12751d9052c839aa4db4abd586c419bed1018..0571dbd31de9b37230f0ee1d93c22c6df47c87e7 100644 --- a/unittests/test_transformers.py +++ b/unittests/test_transformers.py @@ -29,19 +29,18 @@ See: https://gitlab.indiscale.com/caosdb/src/caosdb-crawler/-/issues/107 """ import importlib -from functools import partial from pathlib import Path -from tempfile import NamedTemporaryFile from unittest.mock import MagicMock, Mock, patch import linkahead as db import pytest import yaml +from pytest import raises + from caoscrawler.converters import Converter, ListElementConverter from caoscrawler.scanner import create_transformer_registry, scan_directory from caoscrawler.stores import GeneralStore from caoscrawler.transformer_functions import replace, split -from pytest import raises UNITTESTDIR = Path(__file__).parent diff --git a/unittests/test_utilities.py b/unittests/test_utilities.py index 15e84a609149ac602ee80b7357f7622566563792..463e304a99161f2294e5d202611dcf0b829e2045 100644 --- a/unittests/test_utilities.py +++ b/unittests/test_utilities.py @@ -22,7 +22,7 @@ import pytest from caoscrawler.crawl import split_restricted_path -from caoscrawler.utils import get_shared_resource_link, MissingImport +from caoscrawler.utils import MissingImport, get_shared_resource_link def test_split_restricted_path(): diff --git a/unittests/test_variable_substitutions.py b/unittests/test_variable_substitutions.py index 90d144b04a4e1271f74b769759e3f201007af705..c75e37956c1ec24e47ff9cbd9b03572ed4a0f80e 100644 --- a/unittests/test_variable_substitutions.py +++ b/unittests/test_variable_substitutions.py @@ -19,7 +19,6 @@ # along with this program. If not, see <https://www.gnu.org/licenses/>. # -from copy import deepcopy from functools import partial from os.path import basename, dirname, join from pathlib import Path @@ -28,6 +27,10 @@ from unittest.mock import MagicMock, Mock import linkahead as db import pytest import yaml +from linkahead.apiutils import compare_entities +from pytest import raises +from utils import dircheckstr as dircheckstr_base + from caoscrawler import Crawler from caoscrawler.debug_tree import DebugTree from caoscrawler.identifiable_adapters import (IdentifiableAdapter, @@ -35,10 +38,6 @@ from caoscrawler.identifiable_adapters import (IdentifiableAdapter, from caoscrawler.scanner import scan_directory from caoscrawler.structure_elements import (DictListElement, DictTextElement, File) -from linkahead.apiutils import compare_entities -from pytest import raises - -from utils import dircheckstr as dircheckstr_base UNITTESTDIR = Path(__file__).parent dircheckstr = partial(dircheckstr_base, UNITTESTDIR / "test_directories" / diff --git a/unittests/test_xml_converter.py b/unittests/test_xml_converter.py index 9fc9749ccececd41d460fe297edfea72cc30a5ef..e8869ef6ffad511159a583a14fd49d2fad48766b 100644 --- a/unittests/test_xml_converter.py +++ b/unittests/test_xml_converter.py @@ -24,22 +24,18 @@ test the XML converters """ import importlib -import json +from pathlib import Path + import pytest -import sys import yaml - from lxml.etree import fromstring -from pathlib import Path -from caoscrawler.converters import (XMLTagConverter, - XMLAttributeNodeConverter, +from caoscrawler.converters import (XMLAttributeNodeConverter, XMLTagConverter, XMLTextNodeConverter) from caoscrawler.scanner import load_definition from caoscrawler.stores import GeneralStore from caoscrawler.structure_elements import XMLTagElement - UNITTESTDIR = Path(__file__).parent