From dcd8e6e8f658b50c3075f858a4b1db0e4a8d4ca6 Mon Sep 17 00:00:00 2001
From: Timm Fitschen <t.fitschen@indiscale.com>
Date: Fri, 24 Jun 2022 21:43:38 +0200
Subject: [PATCH] Add import and recursive import flags
---
CHANGELOG.md | 9 +++
src/caosdb/common/models.py | 110 +++++++++++++-----------------------
2 files changed, 48 insertions(+), 71 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5ad9b14d..9e15263b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,12 +9,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added ###
+* Directory (EXPERIMENTAL)
+
### Changed ###
+* It is not possible anymore to upload complete directories in one go by just
+ using a File entity with `file=/my/dir/`. The functionality of creating
+ directory entities is handled be the new Directory class.
+
### Deprecated ###
+* Dropoffbox functionality is deprecated in the caosdb server as well.
+
### Removed ###
+* Anything thumbnail-related (File entities)
* Support for Python 3.6 and Python 3.7
### Fixed ###
diff --git a/src/caosdb/common/models.py b/src/caosdb/common/models.py
index baed2928..ee7a69e0 100644
--- a/src/caosdb/common/models.py
+++ b/src/caosdb/common/models.py
@@ -301,17 +301,6 @@ class Entity(object):
def path(self, new_path):
self.__path = new_path
- @property
- def thumbnail(self):
- if self.__thumbnail is not None or self._wrapped_entity is None:
- return self.__thumbnail
-
- return self._wrapped_entity.thumbnail
-
- @thumbnail.setter
- def thumbnail(self, new_thumbnail):
- self.__thumbnail = new_thumbnail
-
@property
def file(self):
if self.__file is not None or self._wrapped_entity is None:
@@ -1824,7 +1813,8 @@ class Record(Entity):
class Directory(Record):
"""This class represents CaosDB's directory entities."""
- def __init__(self, name=None, id=None, description=None, path=None):
+ def __init__(self, name=None, id=None, description=None, path=None,
+ import_file=False, recursive_import=False):
Record.__init__(self, id=id, name=name, description=description)
self.role = "Directory"
self.datatype = None
@@ -1832,6 +1822,9 @@ class Directory(Record):
# location in the fileserver
self.path = path
+ self.recursive_import = recursive_import
+ self.import_file = import_file or recursive_import
+
def to_xml(self, xml=None, add_properties=ALL, local_serialization=False):
"""Convert this file to an xml element.
@@ -1845,7 +1838,7 @@ class Directory(Record):
local_serialization=local_serialization)
def add_property(self, property=None, id=None, name=None, description=None, datatype=None,
- value=None, unit=None, importance=FIX, inheritance=FIX): # @ReservedAssignment
+ value=None, unit=None, importance=FIX, inheritance=FIX):
return super().add_property(
property=property, id=id, name=name, description=description, datatype=datatype,
@@ -1865,26 +1858,30 @@ class File(Record):
look at `test_files.py` in the Python integration tests of the
`load_files.py` script in the advanced user tools.
- @param name: A name for this file record (That's an entity name - not to be
- confused with the last segment of the files path).
- @param id: An ID.
- @param description: A description for this file record.
- @param path: The complete path, including the file name, of the file in the
- server's "caosroot" file system.
- @param file: A local path or python file object. The file designated by
- this argument will be uploaded to the server via HTTP.
- @param pickup: A file/folder in the DropOffBox (the server will move that
- file into its "caosroot" file system).
- @param thumbnail: (Local) filename to a thumbnail for this file.
- @param properties: A list of properties for this file record. @todo is this
- implemented?
- @param from_location: Deprecated, use `pickup` instead.
-
+ Parameters
+ ----------
+ name : str
+ A name for this file record (That's an entity name - not to be confused
+ with the last segment of the files path).
+ id : int
+ An ID.
+ description : str
+ A description for this file record.
+ path : str
+ The complete path, including the file name, of the file in the server's
+ virtual file system.
+ file : str or readable
+ A local path or python file object. The file designated by this
+ argument will be uploaded to the server via HTTP.
+ pickup : str
+ A file/folder in the DropOffBox (the server will move that file into
+ its "caosroot" file system). (DEPRECATED, use import feature)
+ import_file : bool
+ Import the file (don't upload it, its already there). Default: `False`
"""
- def __init__(self, name=None, id=None, description=None, # @ReservedAssignment
- path=None, file=None, pickup=None, # @ReservedAssignment
- thumbnail=None, from_location=None):
+ def __init__(self, name=None, id=None, description=None,
+ path=None, file=None, pickup=None, import_file=False):
Record.__init__(self, id=id, name=name, description=description)
self.role = "File"
self.datatype = None
@@ -1894,16 +1891,9 @@ class File(Record):
# local file path or pointer to local file
self.file = file
- self.thumbnail = thumbnail
self.pickup = pickup
-
- if from_location is not None:
- warn(DeprecationWarning(
- "Param `from_location` is deprecated, use `pickup instead`."))
-
- if self.pickup is None:
- self.pickup = from_location
+ self.import_file = import_file
def to_xml(self, xml=None, add_properties=ALL, local_serialization=False):
"""Convert this file to an xml element.
@@ -1930,7 +1920,7 @@ class File(Record):
file_ = open(target, 'wb')
else:
file_ = NamedTemporaryFile(mode='wb', delete=False)
- checksum = File.download_from_path(file_, self.path)
+ checksum = File._download_from_path(file_, self.path)
if check_hash and self.checksum is not None and self.checksum.lower() != checksum:
raise ConsistencyError(
@@ -1939,7 +1929,7 @@ class File(Record):
return file_.name
@staticmethod
- def download_from_path(target_file, path):
+ def _download_from_path(target_file, path):
_log_request("GET (download): " + path)
response = get_connection().download_file(path)
@@ -1959,29 +1949,6 @@ class File(Record):
def sha512(file):
return File._get_checksum_single_file(file)
- @staticmethod
- def _get_checksum(files):
- import locale
-
- if hasattr(files, "name"):
- return File._get_checksum_single_file(files.name)
- else:
- if isdir(files):
- checksumappend = ""
-
- for child in sorted(listdir(files),
- key=cmp_to_key(locale.strcoll)):
-
- if isdir(files + '/' + child):
- checksumappend += child
- checksumappend += File._get_checksum(files + "/" + child)
- checksum = sha512()
- checksum.update(checksumappend.encode('utf-8'))
-
- return checksum.hexdigest()
- else:
- return File._get_checksum_single_file(files)
-
@staticmethod
def _get_checksum_single_file(single_file):
_file = open(single_file, 'rb')
@@ -1996,7 +1963,7 @@ class File(Record):
return checksum.hexdigest()
def add_property(self, property=None, id=None, name=None, description=None, datatype=None,
- value=None, unit=None, importance=FIX, inheritance=FIX): # @ReservedAssignment
+ value=None, unit=None, importance=FIX, inheritance=FIX):
return super().add_property(
property=property, id=id, name=name, description=description, datatype=datatype,
@@ -3464,6 +3431,8 @@ class Container(list):
entity_xml.set("upload", entity._upload)
elif hasattr(entity, 'pickup') and entity.pickup is not None:
entity_xml.set("pickup", entity.pickup)
+ elif hasattr(entity, 'import_file') and entity.import_file is True:
+ entity_xml.set("import", "true")
insert_xml.append(entity_xml)
@@ -3504,7 +3473,7 @@ class Container(list):
def _process_file_if_present_and_add_to_http_parts(http_parts, entity):
if isinstance(entity, File) and hasattr(
entity, 'file') and entity.file is not None:
- new_checksum = File._get_checksum(entity.file)
+ new_checksum = File.sha512(entity.file)
# do not transfer unchanged files.
@@ -3534,11 +3503,6 @@ class Container(list):
part.filename = entity._upload
http_parts.append(part)
- if entity.thumbnail is not None:
- part = MultipartParam.from_file(paramname=hex(
- randint(0, sys.maxsize)), filename=entity.thumbnail)
- part.filename = entity._upload + ".thumbnail"
- http_parts.append(part)
elif isinstance(entity, File) and hasattr(entity, "file") and entity.file is None:
entity._upload = None
else:
@@ -3617,6 +3581,10 @@ class Container(list):
entity_xml.set("upload", entity._upload)
elif hasattr(entity, 'pickup') and entity.pickup is not None:
entity_xml.set("pickup", entity.pickup)
+ elif hasattr(entity, 'recursive_import') and entity.recursive_import is True:
+ entity_xml.set("recursive_import", "true")
+ elif hasattr(entity, 'import_file') and entity.import_file is True:
+ entity_xml.set("import", "true")
insert_xml.append(entity_xml)
if len(self) > 0 and len(insert_xml) < 1:
--
GitLab