diff --git a/src/caosdb/common/models.py b/src/caosdb/common/models.py
index df77bb7311a86abc8a78715e082f115c6a3efc2b..5d026e214f83d2e54f92fa5cda511259467f64a8 100644
--- a/src/caosdb/common/models.py
+++ b/src/caosdb/common/models.py
@@ -64,7 +64,7 @@ from caosdb.exceptions import (AmbiguousEntityError, AuthorizationError,
EntityHasNoDatatypeError, HTTPURITooLongError,
MismatchingEntitiesError, QueryNotUniqueError,
TransactionError, UniqueNamesError,
- UnqualifiedParentsError,
+ UnqualifiedParentsError, PagingConsistencyError,
UnqualifiedPropertiesError)
from lxml import etree
@@ -4348,7 +4348,34 @@ class Query():
else:
self.q = q
- def execute(self, unique=False, raise_exception_on_error=True, cache=True):
+ def _query_request(self, query_dict):
+ """Used internally to execute the query request..."""
+ _log_request("GET Entity?" + str(query_dict), None)
+ connection = get_connection()
+ http_response = connection.retrieve(
+ entity_uri_segments=["Entity"],
+ query_dict=query_dict)
+ cresp = Container._response_to_entities(http_response)
+ return cresp
+
+ def _paging_generator(self, first_page, query_dict, page_length):
+ """Used internally to create a generator of pages instead instead of a
+ container which contais all the results."""
+ if len(first_page) == 0:
+ return # empty page
+ yield first_page
+ index = page_length
+ while self.results > index:
+ query_dict["P"] = f"{index}L{page_length}"
+ next_page = self._query_request(query_dict)
+ etag = next_page.query.etag
+ if etag is not None and etag != self.etag:
+ raise PagingConsistencyError("The database state changed while retrieving the pages")
+ yield next_page
+ index += page_length
+
+ def execute(self, unique=False, raise_exception_on_error=True, cache=True,
+ page_length=None):
"""Execute a query (via a server-requests) and return the results.
Parameters
@@ -4361,8 +4388,24 @@ class Query():
Whether an exception should be raises when there are errors in the
resulting entities. Defaults to True.
cache : bool
- Whether to use the query cache (equivalent to adding a "cache"
- flag) to the Query object. Defaults to True.
+ Whether to use the server-side query cache (equivalent to adding a
+ "cache" flag) to the Query object. Defaults to True.
+ page_length : int
+ Whether to use paging. If page_length > 0 this method returns a
+ generator (to be used in a for-loop or with list-comprehension).
+ The generator yields containers with up to page_length entities.
+ Otherwise, paging is disabled, as well as for count queries and
+ when unique is True. Defaults to None.
+
+ Raises:
+ -------
+ PagingConsistencyError
+ If the database state changed between paged requests.
+
+ Yields
+ ------
+ page : Container
+ Returns a container with the next `page_length` resulting entities.
Returns
-------
@@ -4370,8 +4413,6 @@ class Query():
Returns an integer when it was a `COUNT` query. Otherwise, returns a
Container with the resulting entities.
"""
- connection = get_connection()
-
flags = self.flags
if cache is False:
@@ -4379,16 +4420,21 @@ class Query():
query_dict = dict(flags)
query_dict["query"] = str(self.q)
- _log_request("GET Entity?" + str(query_dict), None)
- http_response = connection.retrieve(
- entity_uri_segments=["Entity"],
- query_dict=query_dict)
- cresp = Container._response_to_entities(http_response)
+ has_paging = False
+ is_count_query = self.q.lower().startswith('count')
+
+ if not unique and not is_count_query and page_length is not None and page_length > 0:
+ has_paging = True
+ query_dict["P"] = f"0L{page_length}"
+
+ # retreive first/only page
+ cresp = self._query_request(query_dict)
+
self.results = cresp.query.results
self.cached = cresp.query.cached
self.etag = cresp.query.etag
- if self.q.lower().startswith('count') and len(cresp) == 0:
+ if is_count_query and len(cresp) == 0:
# this was a count query
return self.results
@@ -4412,10 +4458,14 @@ class Query():
return r
self.messages = cresp.messages
- return cresp
+ if has_paging:
+ return self._paging_generator(cresp, query_dict, page_length)
+ else:
+ return cresp
-def execute_query(q, unique=False, raise_exception_on_error=True, cache=True, flags=None):
+def execute_query(q, unique=False, raise_exception_on_error=True, cache=True,
+ flags=None, page_length=None):
"""Execute a query (via a server-requests) and return the results.
Parameters
@@ -4434,6 +4484,22 @@ def execute_query(q, unique=False, raise_exception_on_error=True, cache=True, fl
Defaults to True.
flags : dict of str
Flags to be added to the request.
+ page_length : int
+ Whether to use paging. If page_length > 0 this method returns a
+ generator (to be used in a for-loop or with list-comprehension).
+ The generator yields containers with up to page_length entities.
+ Otherwise, paging is disabled, as well as for count queries and
+ when unique is True. Defaults to None.
+
+ Raises:
+ -------
+ PagingConsistencyError
+ If the database state changed between paged requests.
+
+ Yields
+ ------
+ page : Container
+ Returns a container with the next `page_length` resulting entities.
Returns
-------
@@ -4448,7 +4514,7 @@ def execute_query(q, unique=False, raise_exception_on_error=True, cache=True, fl
return query.execute(unique=unique,
raise_exception_on_error=raise_exception_on_error,
- cache=cache)
+ cache=cache, page_length=page_length)
class DropOffBox(list):
diff --git a/src/caosdb/connection/authentication/auth_token.py b/src/caosdb/connection/authentication/auth_token.py
index 688123867f68153d3631bb8559baa235f6f02da5..00f614fd3b01ee26c5a714365b7b148ac1bff8a0 100644
--- a/src/caosdb/connection/authentication/auth_token.py
+++ b/src/caosdb/connection/authentication/auth_token.py
@@ -62,12 +62,20 @@ class AuthTokenAuthenticator(AbstractAuthenticator):
def __init__(self):
super(AuthTokenAuthenticator, self).__init__()
self.auth_token = None
+ self._initial_one_time_token = None
self._connection = None
def login(self):
self._login()
def _login(self):
+ if self._initial_one_time_token is not None:
+ # retry first (this could work if maxReplay > 1)
+ self.auth_token = self._initial_one_time_token
+ headers = {'Cookie': auth_token_to_cookie(self.auth_token)}
+ self._connection.request(method="DELETE", path="logout",
+ headers=headers)
+
raise LoginFailedError("The authentication token is expired or you "
"have been logged out otherwise. The "
"auth_token authenticator cannot log in "
@@ -87,6 +95,7 @@ class AuthTokenAuthenticator(AbstractAuthenticator):
def configure(self, **config):
if "auth_token" in config:
+ self._initial_one_time_token = config.["auth_token"]
self.auth_token = config["auth_token"]
if "connection" in config:
self._connection = config["connection"]
diff --git a/src/caosdb/exceptions.py b/src/caosdb/exceptions.py
index fdd2e11f1dfb8857f86942df2534d732bad9a793..ebbb52565ca7e95b064664da22797489c0d4d422 100644
--- a/src/caosdb/exceptions.py
+++ b/src/caosdb/exceptions.py
@@ -142,11 +142,15 @@ class MismatchingEntitiesError(CaosDBException):
# ######################### Bad query errors ###########################
-
class BadQueryError(CaosDBException):
"""Base class for query errors that are not transaction errors."""
+class PagingConsistencyError(BadQueryError):
+ """The database state changed between two consecutive paged requests of the
+ same query."""
+
+
class QueryNotUniqueError(BadQueryError):
"""A unique query or retrieve found more than one entity."""