From 3a40e9a2f071aee279e03d1fe7ee4b469b48d552 Mon Sep 17 00:00:00 2001
From: florian <f.spreckelsen@inidscale.com>
Date: Fri, 18 Nov 2022 17:55:31 +0100
Subject: [PATCH] MAINT: Use new error and reorganize imports
---
src/caoscrawler/crawl.py | 47 +++++++++++++++++++++-------------------
1 file changed, 25 insertions(+), 22 deletions(-)
diff --git a/src/caoscrawler/crawl.py b/src/caoscrawler/crawl.py
index be6e00ce..bc482c31 100644
--- a/src/caoscrawler/crawl.py
+++ b/src/caoscrawler/crawl.py
@@ -29,35 +29,40 @@ the acuired data with CaosDB.
"""
from __future__ import annotations
+
+import argparse
import importlib
-from caosadvancedtools.cache import UpdateCache, Cache
-import uuid
-import sys
+import logging
import os
+import sys
+import uuid
+import warnings
import yaml
+
+from argparse import RawTextHelpFormatter
+from collections import defaultdict
+from copy import deepcopy
from enum import Enum
-import logging
from importlib_resources import files
-import argparse
-from argparse import RawTextHelpFormatter
+from jsonschema import validate
+from typing import Any, Optional, Type, Union
+
import caosdb as db
+
+from caosadvancedtools.cache import UpdateCache, Cache
from caosadvancedtools.crawler import Crawler as OldCrawler
-import warnings
+from caosdb.apiutils import (compare_entities, EntityMergeConflictError,
+ merge_entities)
from caosdb.common.datatype import is_reference
-from .stores import GeneralStore, RecordStore
-from .identified_cache import IdentifiedCache
-from .structure_elements import StructureElement, Directory
+
from .converters import Converter, DirectoryConverter
from .identifiable_adapters import (IdentifiableAdapter,
LocalStorageIdentifiableAdapter,
CaosDBIdentifiableAdapter)
-from collections import defaultdict
-from typing import Any, Optional, Type, Union
-from caosdb.apiutils import compare_entities, merge_entities
-from copy import deepcopy
-from jsonschema import validate
-
+from .identified_cache import IdentifiedCache
from .macros import defmacro_constructor, macro_constructor
+from .stores import GeneralStore, RecordStore
+from .structure_elements import StructureElement, Directory
logger = logging.getLogger(__name__)
@@ -744,11 +749,8 @@ class Crawler(object):
newrecord = self.get_from_any_cache(record)
try:
merge_entities(newrecord, record)
- except RuntimeError as re:
- if "Merge conflict" in str(re):
- continue
- else:
- raise re
+ except EntityMergeConflictError:
+ continue
Crawler.bend_references_to_new_object(
old=record, new=newrecord, entities=flat + to_be_updated + to_be_inserted)
@@ -986,7 +988,8 @@ class Crawler(object):
if self.identifiableAdapter is None:
raise RuntimeError("Should not happen.")
- to_be_inserted, to_be_updated = self.split_into_inserts_and_updates(crawled_data)
+ to_be_inserted, to_be_updated = self.split_into_inserts_and_updates(
+ crawled_data)
# TODO: refactoring of typo
for el in to_be_updated:
--
GitLab