Skip to content
Snippets Groups Projects

F filesystem

Closed Timm Fitschen requested to merge f-filesystem into dev
Files
4
+ 99
137
@@ -46,6 +46,7 @@ from os.path import isdir
@@ -46,6 +46,7 @@ from os.path import isdir
from random import randint
from random import randint
from tempfile import NamedTemporaryFile
from tempfile import NamedTemporaryFile
from warnings import warn
from warnings import warn
 
from lxml import etree
from caosdb.common.datatype import (BOOLEAN, DATETIME, DOUBLE, INTEGER, TEXT,
from caosdb.common.datatype import (BOOLEAN, DATETIME, DOUBLE, INTEGER, TEXT,
is_list_datatype, is_reference)
is_list_datatype, is_reference)
@@ -66,7 +67,6 @@ from caosdb.exceptions import (AmbiguousEntityError, AuthorizationError,
@@ -66,7 +67,6 @@ from caosdb.exceptions import (AmbiguousEntityError, AuthorizationError,
TransactionError, UniqueNamesError,
TransactionError, UniqueNamesError,
UnqualifiedParentsError,
UnqualifiedParentsError,
UnqualifiedPropertiesError)
UnqualifiedPropertiesError)
from lxml import etree
_ENTITY_URI_SEGMENT = "Entity"
_ENTITY_URI_SEGMENT = "Entity"
@@ -198,6 +198,10 @@ class Entity(object):
@@ -198,6 +198,10 @@ class Entity(object):
return self._wrapped_entity.size
return self._wrapped_entity.size
 
@size.setter
 
def size(self, new_size):
 
self._size = new_size if new_size is None else int(new_size)
 
@property
@property
def id(self):
def id(self):
if self.__id is not None:
if self.__id is not None:
@@ -246,14 +250,24 @@ class Entity(object):
@@ -246,14 +250,24 @@ class Entity(object):
return self._wrapped_entity.description
return self._wrapped_entity.description
@property
def checksum(self):
return self._checksum
@description.setter
@description.setter
def description(self, new_description):
def description(self, new_description):
self.__description = new_description
self.__description = new_description
 
@property
 
def checksum(self):
 
if self._checksum is not None:
 
return self._checksum
 
 
if self._wrapped_entity is None:
 
return None
 
 
return self._wrapped_entity._checksum
 
 
@checksum.setter
 
def checksum(self, new_checksum):
 
self._checksum = new_checksum
 
@property
@property
def unit(self):
def unit(self):
if self.__unit is not None or self._wrapped_entity is None:
if self.__unit is not None or self._wrapped_entity is None:
@@ -289,14 +303,16 @@ class Entity(object):
@@ -289,14 +303,16 @@ class Entity(object):
@property
@property
def thumbnail(self):
def thumbnail(self):
if self.__thumbnail is not None or self._wrapped_entity is None:
warn(DeprecationWarning(
return self.__thumbnail
"The thumbnail feature has been removed from the CaosDB server "
"API"))
return self._wrapped_entity.thumbnail
return None
@thumbnail.setter
@thumbnail.setter
def thumbnail(self, new_thumbnail):
def thumbnail(self, new_thumbnail):
self.__thumbnail = new_thumbnail
warn(DeprecationWarning(
 
"The thumbnail feature has been removed from the CaosDB server "
 
"API"))
@property
@property
def file(self):
def file(self):
@@ -311,14 +327,16 @@ class Entity(object):
@@ -311,14 +327,16 @@ class Entity(object):
@property
@property
def pickup(self):
def pickup(self):
if self.__pickup is not None or self._wrapped_entity is None:
warn(DeprecationWarning(
return self.__pickup
"The drop-off/pickup feature has been removed from the CaosDB "
"server API"))
return self._wrapped_entity.pickup
return None
@pickup.setter
@pickup.setter
def pickup(self, new_pickup):
def pickup(self, new_pickup):
self.__pickup = new_pickup
warn(DeprecationWarning(
 
"The drop-off/pickup feature has been removed from the CaosDB "
 
"server API"))
def grant(self, realm=None, username=None, role=None,
def grant(self, realm=None, username=None, role=None,
permission=None, priority=False, revoke_denial=True):
permission=None, priority=False, revoke_denial=True):
@@ -1045,8 +1063,8 @@ class Entity(object):
@@ -1045,8 +1063,8 @@ class Entity(object):
if self.file is not None and local_serialization:
if self.file is not None and local_serialization:
xml.set("file", self.file)
xml.set("file", self.file)
if self._checksum is not None:
if self.checksum is not None:
xml.set("checksum", self._checksum)
xml.set("checksum", self.checksum)
if self.size is not None:
if self.size is not None:
xml.set("size", str(self.size))
xml.set("size", str(self.size))
@@ -1805,35 +1823,33 @@ class File(Record):
@@ -1805,35 +1823,33 @@ class File(Record):
"""This class represents CaosDB's file entities.
"""This class represents CaosDB's file entities.
For inserting a new file to the server, `path` gives the new location, and
For inserting a new file to the server or updating an existent one.
(exactly?) one of `file` and `pickup` should (must?) be given to specify the
`path` gives the new location. `file` specifies the (local) file which is
source of the file.
to be uploaded. It is mandatory for insertions, but optionally for updates.
Symlinking from the "extroot" file system is not supported by this API yet,
it can be done manually using the `InsertFilesInDir` flag. For sample code,
look at `test_files.py` in the Python integration tests of the
`load_files.py` script in the advanced user tools.
@param name: A name for this file record (That's an entity name - not to be
confused with the last segment of the files path).
@param id: An ID.
@param description: A description for this file record.
@param path: The complete path, including the file name, of the file in the
server's "caosroot" file system.
@param file: A local path or python file object. The file designated by
this argument will be uploaded to the server via HTTP.
@param pickup: A file/folder in the DropOffBox (the server will move that
file into its "caosroot" file system).
@param thumbnail: (Local) filename to a thumbnail for this file.
@param properties: A list of properties for this file record. @todo is this
implemented?
@param from_location: Deprecated, use `pickup` instead.
 
Parameters
 
----------
 
name : str
 
A name for this file record (That's an entity name - not to be confused
 
with the last segment of the files path).
 
id : int
 
An ID.
 
description : str
 
A description for this file record.
 
path : str
 
The complete path, including the file name, of the file in the server's
 
virtual file system.
 
file : str or readable
 
A local path or python file object. The file designated by this
 
argument will be uploaded to the server via HTTP.
 
pickup : str
 
Deprecated.
 
thumbnail : str
 
Deprecated.
"""
"""
def __init__(self, name=None, id=None, description=None,
def __init__(self, name=None, id=None, description=None, # @ReservedAssignment
path=None, file=None, pickup=None,
path=None, file=None, pickup=None, # @ReservedAssignment
thumbnail=None):
thumbnail=None, from_location=None):
Record.__init__(self, id=id, name=name, description=description)
Record.__init__(self, id=id, name=name, description=description)
self.role = "File"
self.role = "File"
self.datatype = None
self.datatype = None
@@ -1843,16 +1859,15 @@ class File(Record):
@@ -1843,16 +1859,15 @@ class File(Record):
# local file path or pointer to local file
# local file path or pointer to local file
self.file = file
self.file = file
self.thumbnail = thumbnail
if thumbnail is not None:
self.pickup = pickup
if from_location is not None:
warn(DeprecationWarning(
warn(DeprecationWarning(
"Param `from_location` is deprecated, use `pickup instead`."))
"The thumbnail feature has been removed from the CaosDB "
 
"server API"))
if self.pickup is None:
if pickup is not None:
self.pickup = from_location
warn(DeprecationWarning(
 
"The drop-off/pickup feature has been removed from the CaosDB "
 
"server API"))
def to_xml(self, xml=None, add_properties=ALL, local_serialization=False):
def to_xml(self, xml=None, add_properties=ALL, local_serialization=False):
"""Convert this file to an xml element.
"""Convert this file to an xml element.
@@ -1866,7 +1881,7 @@ class File(Record):
@@ -1866,7 +1881,7 @@ class File(Record):
return Entity.to_xml(self, xml=xml, add_properties=add_properties,
return Entity.to_xml(self, xml=xml, add_properties=add_properties,
local_serialization=local_serialization)
local_serialization=local_serialization)
def download(self, target=None):
def download(self, target=None, check_hash=True):
"""Download this file-entity's actual file from the file server. It
"""Download this file-entity's actual file from the file server. It
will be stored to the target or will be hold as a temporary file.
will be stored to the target or will be hold as a temporary file.
@@ -1879,16 +1894,16 @@ class File(Record):
@@ -1879,16 +1894,16 @@ class File(Record):
file_ = open(target, 'wb')
file_ = open(target, 'wb')
else:
else:
file_ = NamedTemporaryFile(mode='wb', delete=False)
file_ = NamedTemporaryFile(mode='wb', delete=False)
checksum = File.download_from_path(file_, self.path)
checksum = File._download_from_path(file_, self.path)
if self._checksum is not None and self._checksum.lower() != checksum.hexdigest().lower():
if check_hash and self.checksum is not None and self.checksum.lower() != checksum:
raise ConsistencyError(
raise ConsistencyError(
"The downloaded file had an invalid checksum. Maybe the download did not finish?")
"The downloaded file had an invalid checksum. Maybe the download did not finish?")
return file_.name
return file_.name
@staticmethod
@staticmethod
def download_from_path(target_file, path):
def _download_from_path(target_file, path):
_log_request("GET (download): " + path)
_log_request("GET (download): " + path)
response = get_connection().download_file(path)
response = get_connection().download_file(path)
@@ -1902,46 +1917,27 @@ class File(Record):
@@ -1902,46 +1917,27 @@ class File(Record):
data = response.read(8000)
data = response.read(8000)
target_file.close()
target_file.close()
return checksum
return checksum.hexdigest().lower()
@staticmethod
@staticmethod
def _get_checksum(files):
def _get_checksum(file):
import locale
if hasattr(file, "name"):
file_name = file.name
if hasattr(files, "name"):
return File._get_checksum_single_file(files.name)
else:
else:
if isdir(files):
file_name = file
checksumappend = ""
for child in sorted(listdir(files),
key=cmp_to_key(locale.strcoll)):
if isdir(files + '/' + child):
checksumappend += child
checksumappend += File._get_checksum(files + "/" + child)
checksum = sha512()
checksum.update(checksumappend.encode('utf-8'))
return checksum.hexdigest()
else:
return File._get_checksum_single_file(files)
@staticmethod
def _get_checksum_single_file(single_file):
_file = open(single_file, 'rb')
data = _file.read(1000)
checksum = sha512()
checksum = sha512()
with open(file_name, 'rb') as _file:
while data:
checksum.update(data)
data = _file.read(1000)
data = _file.read(1000)
_file.close()
return checksum.hexdigest()
while data:
 
checksum.update(data)
 
data = _file.read(1000)
 
 
return checksum.hexdigest().lower()
def add_property(self, property=None, id=None, name=None, description=None, datatype=None,
def add_property(self, property=None, id=None, name=None, description=None, datatype=None,
value=None, unit=None, importance=FIX, inheritance=FIX): # @ReservedAssignment
value=None, unit=None, importance=FIX, inheritance=FIX):
return super().add_property(
return super().add_property(
property=property, id=id, name=name, description=description, datatype=datatype,
property=property, id=id, name=name, description=description, datatype=datatype,
@@ -3048,6 +3044,11 @@ class Container(list):
@@ -3048,6 +3044,11 @@ class Container(list):
sync_remote_entities = []
sync_remote_entities = []
for remote_entity in remote_container:
for remote_entity in remote_container:
 
 
# legacy API - remove new directories.
 
if unique and remote_entity.role is not None and remote_entity.role.lower() == "directory":
 
continue
 
if not (remote_entity in used_remote_entities):
if not (remote_entity in used_remote_entities):
sync_remote_entities.append(remote_entity)
sync_remote_entities.append(remote_entity)
@@ -3322,10 +3323,10 @@ class Container(list):
@@ -3322,10 +3323,10 @@ class Container(list):
uri1, uri2 = Container._split_uri_string(entities)
uri1, uri2 = Container._split_uri_string(entities)
except ValueError as val_e:
except ValueError as val_e:
raise uri_e from val_e
raise uri_e from val_e
c1 = self._retrieve(entities=uri1, flags=flags)
c1 = self._retrieve(entities=uri1, flags=flags)
c2 = self._retrieve(entities=uri2, flags=flags)
c2 = self._retrieve(entities=uri2, flags=flags)
c1.extend(c2)
c1.extend(c2)
c1.messages.extend(c2.messages)
c1.messages.extend(c2.messages)
return c1
return c1
@@ -3402,8 +3403,6 @@ class Container(list):
@@ -3402,8 +3403,6 @@ class Container(list):
if hasattr(entity, '_upload') and entity._upload is not None:
if hasattr(entity, '_upload') and entity._upload is not None:
entity_xml.set("upload", entity._upload)
entity_xml.set("upload", entity._upload)
elif hasattr(entity, 'pickup') and entity.pickup is not None:
entity_xml.set("pickup", entity.pickup)
insert_xml.append(entity_xml)
insert_xml.append(entity_xml)
@@ -3474,20 +3473,18 @@ class Container(list):
@@ -3474,20 +3473,18 @@ class Container(list):
part.filename = entity._upload
part.filename = entity._upload
http_parts.append(part)
http_parts.append(part)
if entity.thumbnail is not None:
elif isinstance(entity, File) and hasattr(entity, "file") and entity.file is None:
part = MultipartParam.from_file(paramname=hex(
entity._upload = None
randint(0, sys.maxsize)), filename=entity.thumbnail)
part.filename = entity._upload + ".thumbnail"
http_parts.append(part)
else:
else:
 
entity._upload = None
entity._checksum = None
entity._checksum = None
def insert(self, strict=False, raise_exception_on_error=True,
def insert(self, strict=False, raise_exception_on_error=True,
unique=True, sync=True, flags=None):
unique=True, sync=True, flags=None):
"""Insert this file entity into CaosDB. A successful insertion will
"""Insert these entities into CaosDB. A successful insertion will
generate a new persistent ID for this entity. This entity can be
generate a persistent ID for the new entities. These entities can be
identified, retrieved, updated, and deleted via this ID until it has
identified, retrieved, updated, and deleted via their ID until they
been deleted.
have been deleted.
If the insertion fails, a CaosDBException will be raised. The server will have returned at
If the insertion fails, a CaosDBException will be raised. The server will have returned at
least one error-message describing the reason why it failed in that case (call
least one error-message describing the reason why it failed in that case (call
@@ -3552,8 +3549,6 @@ class Container(list):
@@ -3552,8 +3549,6 @@ class Container(list):
if hasattr(entity, '_upload') and entity._upload is not None:
if hasattr(entity, '_upload') and entity._upload is not None:
entity_xml.set("upload", entity._upload)
entity_xml.set("upload", entity._upload)
elif hasattr(entity, 'pickup') and entity.pickup is not None:
entity_xml.set("pickup", entity.pickup)
insert_xml.append(entity_xml)
insert_xml.append(entity_xml)
if len(self) > 0 and len(insert_xml) < 1:
if len(self) > 0 and len(insert_xml) < 1:
@@ -4265,39 +4260,6 @@ def execute_query(q, unique=False, raise_exception_on_error=True, cache=True, fl
@@ -4265,39 +4260,6 @@ def execute_query(q, unique=False, raise_exception_on_error=True, cache=True, fl
cache=cache)
cache=cache)
class DropOffBox(list):
path = None
def sync(self):
c = get_connection()
_log_request("GET: Info")
http_response = c.retrieve(["Info"])
body = http_response.read()
_log_response(body)
xml = etree.fromstring(body)
for child in xml:
if child.tag.lower() == "stats":
infoelem = child
break
for child in infoelem:
if child.tag.lower() == "dropoffbox":
dropoffboxelem = child
break
del self[:]
self.path = dropoffboxelem.get('path')
for f in dropoffboxelem:
self.append(f.get('path'))
return self
class UserInfo():
class UserInfo():
def __init__(self, xml):
def __init__(self, xml):
Loading