From 27c767e2edeee14e366d5c9e6dd91157aea81cfb Mon Sep 17 00:00:00 2001
From: Timm Fitschen <t.fitschen@indiscale.com>
Date: Fri, 25 Nov 2022 14:26:42 +0100
Subject: [PATCH] TST: fix integration tests
---
src/caosdb/common/models.py | 131 +++++++++++++++++++-----------------
1 file changed, 68 insertions(+), 63 deletions(-)
diff --git a/src/caosdb/common/models.py b/src/caosdb/common/models.py
index 49315031..0c196f9c 100644
--- a/src/caosdb/common/models.py
+++ b/src/caosdb/common/models.py
@@ -46,6 +46,7 @@ from os.path import isdir
from random import randint
from tempfile import NamedTemporaryFile
from warnings import warn
+from lxml import etree
from caosdb.common.datatype import (BOOLEAN, DATETIME, DOUBLE, INTEGER, TEXT,
is_list_datatype, is_reference)
@@ -66,7 +67,6 @@ from caosdb.exceptions import (AmbiguousEntityError, AuthorizationError,
TransactionError, UniqueNamesError,
UnqualifiedParentsError,
UnqualifiedPropertiesError)
-from lxml import etree
_ENTITY_URI_SEGMENT = "Entity"
@@ -198,6 +198,10 @@ class Entity(object):
return self._wrapped_entity.size
+ @size.setter
+ def size(self, new_size):
+ self._size = new_size if new_size is None else int(new_size)
+
@property
def id(self):
if self.__id is not None:
@@ -246,14 +250,24 @@ class Entity(object):
return self._wrapped_entity.description
- @property
- def checksum(self):
- return self._checksum
-
@description.setter
def description(self, new_description):
self.__description = new_description
+ @property
+ def checksum(self):
+ if self._checksum is not None:
+ return self._checksum
+
+ if self._wrapped_entity is None:
+ return None
+
+ return self._wrapped_entity._checksum
+
+ @checksum.setter
+ def checksum(self, new_checksum):
+ self._checksum = new_checksum
+
@property
def unit(self):
if self.__unit is not None or self._wrapped_entity is None:
@@ -1049,8 +1063,8 @@ class Entity(object):
if self.file is not None and local_serialization:
xml.set("file", self.file)
- if self._checksum is not None:
- xml.set("checksum", self._checksum)
+ if self.checksum is not None:
+ xml.set("checksum", self.checksum)
if self.size is not None:
xml.set("size", str(self.size))
@@ -1818,19 +1832,26 @@ class File(Record):
look at `test_files.py` in the Python integration tests of the
`load_files.py` script in the advanced user tools.
- @param name: A name for this file record (That's an entity name - not to be
- confused with the last segment of the files path).
- @param id: An ID.
- @param description: A description for this file record.
- @param path: The complete path, including the file name, of the file in the
- server's "caosroot" file system.
- @param file: A local path or python file object. The file designated by
- this argument will be uploaded to the server via HTTP.
- @param pickup: deprecated
- @param thumbnail: deprecated.
- @param properties: A list of properties for this file record. @todo is this
- implemented?
-
+ Parameters
+ ----------
+ name : str
+ A name for this file record (That's an entity name - not to be confused
+ with the last segment of the files path).
+ id : int
+ An ID.
+ description : str
+ A description for this file record.
+ path : str
+ The complete path, including the file name, of the file in the server's
+ virtual file system.
+ file : str or readable
+ A local path or python file object. The file designated by this
+ argument will be uploaded to the server via HTTP.
+ pickup : str
+ A file/folder in the DropOffBox (the server will move that file into
+ its "caosroot" file system). (DEPRECATED, use import feature)
+ import_file : bool
+ Import the file (don't upload it, its already there). Default: `False`
"""
def __init__(self, name=None, id=None, description=None,
@@ -1867,7 +1888,7 @@ class File(Record):
return Entity.to_xml(self, xml=xml, add_properties=add_properties,
local_serialization=local_serialization)
- def download(self, target=None):
+ def download(self, target=None, check_hash=True):
"""Download this file-entity's actual file from the file server. It
will be stored to the target or will be hold as a temporary file.
@@ -1880,16 +1901,16 @@ class File(Record):
file_ = open(target, 'wb')
else:
file_ = NamedTemporaryFile(mode='wb', delete=False)
- checksum = File.download_from_path(file_, self.path)
+ checksum = File._download_from_path(file_, self.path)
- if self._checksum is not None and self._checksum.lower() != checksum.hexdigest().lower():
+ if check_hash and self.checksum is not None and self.checksum.lower() != checksum:
raise ConsistencyError(
"The downloaded file had an invalid checksum. Maybe the download did not finish?")
return file_.name
@staticmethod
- def download_from_path(target_file, path):
+ def _download_from_path(target_file, path):
_log_request("GET (download): " + path)
response = get_connection().download_file(path)
@@ -1903,46 +1924,27 @@ class File(Record):
data = response.read(8000)
target_file.close()
- return checksum
+ return checksum.hexdigest().lower()
@staticmethod
- def _get_checksum(files):
- import locale
-
- if hasattr(files, "name"):
- return File._get_checksum_single_file(files.name)
+ def _get_checksum(file):
+ if hasattr(file, "name"):
+ file_name = file.name
else:
- if isdir(files):
- checksumappend = ""
-
- for child in sorted(listdir(files),
- key=cmp_to_key(locale.strcoll)):
-
- if isdir(files + '/' + child):
- checksumappend += child
- checksumappend += File._get_checksum(files + "/" + child)
- checksum = sha512()
- checksum.update(checksumappend.encode('utf-8'))
-
- return checksum.hexdigest()
- else:
- return File._get_checksum_single_file(files)
+ file_name = file
- @staticmethod
- def _get_checksum_single_file(single_file):
- _file = open(single_file, 'rb')
- data = _file.read(1000)
checksum = sha512()
-
- while data:
- checksum.update(data)
+ with open(file_name, 'rb') as _file:
data = _file.read(1000)
- _file.close()
- return checksum.hexdigest()
+ while data:
+ checksum.update(data)
+ data = _file.read(1000)
+
+ return checksum.hexdigest().lower()
def add_property(self, property=None, id=None, name=None, description=None, datatype=None,
- value=None, unit=None, importance=FIX, inheritance=FIX): # @ReservedAssignment
+ value=None, unit=None, importance=FIX, inheritance=FIX):
return super().add_property(
property=property, id=id, name=name, description=description, datatype=datatype,
@@ -3323,10 +3325,10 @@ class Container(list):
uri1, uri2 = Container._split_uri_string(entities)
except ValueError as val_e:
raise uri_e from val_e
- c1 = self._retrieve(entities=uri1, flags=flags)
- c2 = self._retrieve(entities=uri2, flags=flags)
- c1.extend(c2)
- c1.messages.extend(c2.messages)
+ c1 = self._retrieve(entities=uri1, flags=flags)
+ c2 = self._retrieve(entities=uri2, flags=flags)
+ c1.extend(c2)
+ c1.messages.extend(c2.messages)
return c1
@@ -3473,15 +3475,18 @@ class Container(list):
part.filename = entity._upload
http_parts.append(part)
+ elif isinstance(entity, File) and hasattr(entity, "file") and entity.file is None:
+ entity._upload = None
else:
+ entity._upload = None
entity._checksum = None
def insert(self, strict=False, raise_exception_on_error=True,
unique=True, sync=True, flags=None):
- """Insert this file entity into CaosDB. A successful insertion will
- generate a new persistent ID for this entity. This entity can be
- identified, retrieved, updated, and deleted via this ID until it has
- been deleted.
+ """Insert these entities into CaosDB. A successful insertion will
+ generate a persistent ID for the new entities. These entities can be
+ identified, retrieved, updated, and deleted via their ID until they
+ have been deleted.
If the insertion fails, a CaosDBException will be raised. The server will have returned at
least one error-message describing the reason why it failed in that case (call
--
GitLab