From 5d81584cd98766237440deb451359d37f9c14fb9 Mon Sep 17 00:00:00 2001
From: Timm Fitschen <t.fitschen@indiscale.com>
Date: Sun, 8 Aug 2021 21:55:31 +0200
Subject: [PATCH] WIP: files
---
Makefile | 3 +
proto/caosdb/entity/v1alpha1/main.proto | 368 +++++++++++++-----------
2 files changed, 208 insertions(+), 163 deletions(-)
diff --git a/Makefile b/Makefile
index 99bee6f..5c2bedc 100644
--- a/Makefile
+++ b/Makefile
@@ -3,3 +3,6 @@ ALL:
formatting:
clang-format -i $$(find proto/ -type f -iname "*.proto")
+
+linting:
+ buf lint
diff --git a/proto/caosdb/entity/v1alpha1/main.proto b/proto/caosdb/entity/v1alpha1/main.proto
index 2f82b91..e19ff71 100644
--- a/proto/caosdb/entity/v1alpha1/main.proto
+++ b/proto/caosdb/entity/v1alpha1/main.proto
@@ -329,21 +329,6 @@ message Entity {
repeated Property properties = 9;
// Parents of this entity.
repeated Parent parents = 10;
- // Errors for this entity which occurred during the current transaction. The
- // presence of errors indicates that a transaction was not successful and the
- // errors describe why this is the case.
- repeated Message errors = 11;
- // Warnings for this entity which occurred during the current transaction,
- // but the transaction was nevertheless successful. The presence of warnings
- // indicates that the entity is in an undesired or non-standard state itself
- // or in an undesired or non-standard relation to other entities or the
- // server's state or configuration. This might cause errors in the future
- // and should be remedied if possible.
- repeated Message warnings = 12;
- // Info messages for this entity which may inform the client about anything
- // related to the transaction or the new state of the entity itself or in
- // relation to other entities or the server's state or configuration.
- repeated Message infos = 13;
// File meta data. The actual binary data has to uploaded or downloaded using
// the File message.
@@ -359,7 +344,7 @@ message FileDescriptor {
string path = 2;
// The size of the file
int64 size = 3;
- // Hash sums for consistency checks
+ // Hash sums for consistency checks.
repeated Hash hashes = 4;
}
@@ -371,15 +356,6 @@ message Hash {
string value = 2;
}
-// Wraps an entity a associates it with a file_upload
-message EntityRequest {
- // The entity.
- Entity entity = 1;
- // The transmission id (if any). Note: upload_ids are only meaningful for
- // File entities.
- FileTransmissionId upload_id = 2;
-}
-
// Property of an entity. Don't confuse with an Entity with role="Property"
message Property {
// ID of the implemented Entity
@@ -420,26 +396,81 @@ message Parent {
repeated Message infos = 6;
}
-// Response to a retrieve request
-message RetrieveResponse {
- // Payload of the retrieval
- oneof query_response {
- // Result of a FIND query or a request by id.
- Entity entity = 1;
- // Result of a SELECT query
- SelectQueryResult select_result = 3;
- // Result of a COUNT query
- int64 count_result = 4;
- }
+// Wraps an entity and associates it with a previous file upload via a
+// transmission id.
+message EntityRequest {
+ // The entity.
+ Entity entity = 1;
+ // The transmission id (if any). Note: upload_ids are only meaningful for
+ // File entities.
+ FileTransmissionId upload_id = 2;
}
-// CaosDB Query
+// Wraps an entity and associates it with a transimission id for a file
+// download
+message EntityResponse {
+ // The entity.
+ Entity entity = 1;
+ // The transmission id (if any). Note: download_ids are only meaningful for
+ // File entities.
+ FileTransmissionId download_id = 2;
+ // Errors for this entity which occurred during the current transaction. The
+ // presence of errors indicates that a transaction was not successful and the
+ // errors describe why this is the case.
+ repeated Message errors = 3;
+ // Warnings for this entity which occurred during the current transaction,
+ // but the transaction was nevertheless successful. The presence of warnings
+ // indicates that the entity is in an undesired or non-standard state itself
+ // or in an undesired or non-standard relation to other entities or the
+ // server's state or configuration. This might cause errors in the future
+ // and should be remedied if possible.
+ repeated Message warnings = 4;
+ // Info messages for this entity which may inform the client about anything
+ // related to the transaction or the new state of the entity itself or in
+ // relation to other entities or the server's state or configuration.
+ repeated Message infos = 5;
+}
+
+// A response containing only the id (of a deleted, inserted, or updated
+// entity) and the usual trias of entity messages.
+message IdResponse {
+ // The entity id.
+ string id = 1;
+ // Errors for this entity which occurred during the current transaction. The
+ // presence of errors indicates that a transaction was not successful and the
+ // errors describe why this is the case.
+ repeated Message errors = 2;
+ // Warnings for this entity which occurred during the current transaction,
+ // but the transaction was nevertheless successful. The presence of warnings
+ // indicates that the entity is in an undesired or non-standard state itself
+ // or in an undesired or non-standard relation to other entities or the
+ // server's state or configuration. This might cause errors in the future
+ // and should be remedied if possible.
+ repeated Message warnings = 3;
+ // Info messages for this entity which may inform the client about anything
+ // related to the transaction or the new state of the entity itself or in
+ // relation to other entities or the server's state or configuration.
+ repeated Message infos = 4;
+}
+
+///////////////////////////////////////////////////////////////////////////
+// Query stuff
+///////////////////////////////////////////////////////////////////////////
+
+// Query message. Currently this messge only wraps the query string. This might
+// be extended in the future.
message Query {
// The string representation of the query, e.g. "FIND Person WITH name =
// Aisha"
string query = 1;
}
+// Result of a FIND query.
+message FindQueryResult {
+ // Entities in the result set.
+ repeated EntityResponse result_set = 1;
+}
+
// A single result row of a SELECT query
message SelectQueryRows {
// The cells of the table row.
@@ -454,51 +485,134 @@ message SelectQueryResult {
repeated SelectQueryRows data_rows = 2;
}
-// Request which specifies one or multiple entities by query or by id.
-message QueryOrIdRequest {
- // We wrap these two together because this makes it easier to add further
- // requests (like "name", or "path") in the future.
+///////////////////////////////////////////////////////////////////////////
+// Request and Responses for single CRUD-Transaction
+///////////////////////////////////////////////////////////////////////////
+
+// Single retrieve request (by query or by id)
+message RetrieveRequest {
+ // specify be query or by id
oneof wrapped_request {
// A single id
string id = 1;
// A query
Query query = 2;
}
+ // Is the client requesting to download the file associated with the
+ // retrieved entity or query results?
+ bool register_file_download = 3;
+ // Settings for the download. Ignored if register_file_download is false.
+ FileTransmissionSettings download_settings = 4;
}
-// Single response containing either an id, a COUNT query result, a SELECT-table
-// or complete entities.
-message TransactionResponse {
- // We wrap these together in order to make this a repeatable field in the
- // MultiTransactionResponse.
+// Response to a retrieve request
+message RetrieveResponse {
+ // Payload of the retrieval
oneof wrapped_response {
- // Single response to a delete transaction.
- IdResponse delete_response = 1;
- // Single response to an update transaction.
- IdResponse update_response = 2;
- // Single response to a retrieve transaction.
- RetrieveResponse retrieve_response = 3;
- // Single response to an insert transaction.
- IdResponse insert_response = 4;
+ // Result of a request by id or any other unique identifier.
+ EntityResponse entity = 1;
+ // Result of a FIND query
+ FindQueryResult find_result = 2;
+ // Result of a SELECT query
+ SelectQueryResult select_result = 3;
+ // Result of a COUNT query
+ int64 count_result = 4;
+ }
+ // Response to `register_file_download = true` in the RetrieveRequest.
+ string registration_id = 5;
+}
+
+// Single delete request (by id)
+message DeleteRequest {
+ // Wrapped to make it easier to add delete-by-query in the future.
+ oneof wrapped_request {
+ // A single id
+ string id = 1;
}
}
+// Response to a single delete request.
+message DeleteResponse {
+ // Wrapped to make it easier to optionally return the full entity in the future.
+ oneof wrapped_response {
+ // Id and messages of the deleted entity.
+ IdResponse id_response = 1;
+ }
+}
+
+// Request to update a single Entity
+message UpdateRequest {
+ // The entity which is to be updated.
+ EntityRequest entity_request = 1;
+}
+
+// Response to a single update request.
+message UpdateResponse {
+ // Wraps an IdResponse. Might be extended in the future to optionally return the full entity.
+ oneof wrapped_response {
+ // Id and messages of the updated entity.
+ IdResponse id_response = 1;
+ }
+}
+
+// Request to insert a single entity.
+message InsertRequest {
+ // The new entity.
+ EntityRequest entity_request = 1;
+}
+
+// Response to a single insert request.
+message InsertResponse {
+ // Wraps an IdResponse. Might be extended in the future to optionally return also the full entity.
+ oneof wrapped_response {
+ // Id and messages of the inserted entity.
+ IdResponse id_response = 1;
+ }
+}
+
+///////////////////////////////////////////////////////////////////////////
+// Highlevel Requests and Responses for multiple mixed CRUD transactions.
+///////////////////////////////////////////////////////////////////////////
+
// Single request containing either a query, an id or a complete entity.
message TransactionRequest {
// We wrap these together in order to make this a repeatable field in the
// MultiTransactionRequest.
oneof wrapped_requests {
// Single request for a retrieve transaction.
- QueryOrIdRequest retrieve_request = 1;
+ RetrieveRequest retrieve_request = 1;
// Single request for an update transaction.
- EntityRequest update_request = 2;
+ UpdateRequest update_request = 2;
// Single request for an insert transaction.
- EntityRequest insert_request = 3;
+ InsertRequest insert_request = 3;
// Single request for a delete transaction.
- QueryOrIdRequest delete_request = 4;
+ DeleteRequest delete_request = 4;
}
}
+// Single response containing either an id, a COUNT query result, a SELECT-table
+// or complete entities.
+message TransactionResponse {
+ // We wrap these together in order to make this a repeatable field in the
+ // MultiTransactionResponse.
+ oneof wrapped_response {
+ // Single response to a delete transaction.
+ DeleteResponse delete_response = 1;
+ // Single response to an update transaction.
+ UpdateResponse update_response = 2;
+ // Single response to a retrieve transaction.
+ RetrieveResponse retrieve_response = 3;
+ // Single response to an insert transaction.
+ InsertResponse insert_response = 4;
+ }
+}
+
+// Wraps multiple sub-transaction requests of mixed types.
+message MultiTransactionRequest {
+ // The actual payload.
+ repeated TransactionRequest requests = 1;
+}
+
// Wraps the reponses to multiple sub-transactions of mixed types.
message MultiTransactionResponse {
// The actual payload.
@@ -511,25 +625,6 @@ message MultiTransactionResponse {
repeated Message transaction_infos = 4;
}
-// Wraps multiple sub-transaction requests of mixed types.
-message MultiTransactionRequest {
- // The actual payload.
- repeated TransactionRequest requests = 1;
-}
-
-// A response containing only the id (of a deleted, inserted, or updated
-// entity) and the usual trias of entity messages.
-message IdResponse {
- // The entity id.
- string id = 1;
- // Entity errors.
- repeated Message entity_errors = 2;
- // Entity warnings.
- repeated Message entity_warnings = 3;
- // Entity info messages.
- repeated Message entity_infos = 4;
-}
-
// Entity Transaction Service
service EntityTransactionService {
// Issue multiple mixed requests
@@ -537,9 +632,36 @@ service EntityTransactionService {
returns (MultiTransactionResponse);
}
-////////////////////////////////////////////////////////////////////////////
-//////////// FILE UPLOAD/DOWNLOAD
-////////////////////////////////////////////////////////////////////////////
+///////////////////////////////////////////////////////////////////////////
+// File transmission
+///////////////////////////////////////////////////////////////////////////
+
+// Stores a single chunk of a file
+message FileChunk {
+ // Temporary identifier containing the file and registration_id.
+ FileTransmissionId file_transmission_id = 1;
+ // Binary data of a chunk.
+ bytes data = 2;
+}
+
+// Temporary identifier of a single file during a transmission.
+message FileTransmissionId {
+ // The registration id which has been issued by the target of the trans
+ string registration_id = 1;
+ // A temporary identifier which identifies the file of this chunk. The
+ // file_id is also used by transaction to associate entities (which are to be
+ // inserted or updated) with a binary blob.
+ string file_id = 2;
+}
+
+// Settings for the the file transmission.
+message FileTransmissionSettings {
+ // The maximum chunk size.
+ int64 max_chunk_size = 1;
+ // The maximum file size.
+ int64 max_file_size = 2;
+}
+
// Indicates whether a registration (for upload or download) has been accepted
// or rejected.
@@ -577,84 +699,7 @@ message RegisterFileUploadResponse {
// The registration id is used to identify chunks and files which belong to
// the same upload .
string registration_id = 2;
- // The server's maximum chunk size. Submitting bigger chunks has undefined
- // behavior.
- int64 max_chunk_size = 4;
- // The server's maximum file size. Submitting bigger files has undefined
- // behavior.
- int64 max_file_size = 5;
-}
-
-// Represents a download request.
-message RegisterFileDownloadRequest {
- // The files requested for download.
- repeated FileIdentifier files = 1;
- // The server's maximum chunk size. Submitting bigger chunks has undefined
- // behavior.
- int64 max_chunk_size = 4;
- // The server's maximum file size. Submitting bigger files has undefined
- // behavior.
- int64 max_file_size = 5;
-}
-
-// Identifies a file which is to be downloaded.
-message FileIdentifier {
- // Wraps several identifiers in order to have them as a repeatable field in
- // other messages.
- oneof wrapped_identifier {
- // An entity id.
- string entity_id = 1;
- // A path in the internal file system
- string path = 2;
- }
-}
-
-// Metadata and status information of a file which is about to be downloaded.
-message FileDownloadHeader {
- // The file identifier exactly as it has been requested during a
- // RegisterFileDownloadRequest.
- FileIdentifier id = 1;
- // The file descriptor of the file.
- FileDescriptor file_descriptor = 2;
- // This is the transmission id which can be used to download the actual blob
- // in the FileDownloadRequest.
- FileTransmissionId file_transmission_id = 3;
- // Errors for this file (e.g. MESSAGE_CODE_ENTITY_DOES_NOT_EXIST, or
- // MESSAGE_CODE_FILE_NOT_FOUND).
- repeated Message errors = 4;
- // Warnings for this file.
- repeated Message warnings = 5;
- // Info messages for this file.
- repeated Message infos = 6;
-}
-
-// Response to a download registration request.
-message RegisterFileDownloadResponse {
- // Whether the server accepted or rejected the registration.
- RegistrationStatus status = 1;
- // The registration id is used to identify chunks and files which belong to
- // the same upload .
- string registration_id = 2;
- // Metadata of the files which are to be downloaded.
- repeated FileDownloadHeader files = 4;
-}
-
-// Stores a single chunk of a file
-message FileChunk {
- // Temporary identifier containing the file and registration_id.
- FileTransmissionId file_transmission_id = 1;
- // Binary data of a chunk.
- bytes data = 5;
-}
-
-// Temporary identifier of a single file during a transmission.
-message FileTransmissionId {
- // The registration id which has been issued by the target of the trans
- string registration_id = 1;
- // A temporary identifier which identifies the file of this chunk. The
- // file_id is also used by transaction to associate entities (which are to be
- // inserted or updated) with a binary blob.
- string file_id = 2;
+ // The server's transmission settings for the upload.
}
// Request for a file upload which has been registered previously. Chunks may be
@@ -692,10 +737,7 @@ service FileTransmissionService {
returns (RegisterFileUploadResponse);
// The actual file upload. The upload has to be registered prior to this rpc.
rpc FileUpload(FileUploadRequest) returns (FileUploadResponse);
- // Register a file download. This needs to be done prior to the actual
- // download.
- rpc RegisterFileDownload(RegisterFileDownloadRequest)
- returns (RegisterFileDownloadResponse);
- // The actual download. The download has to be registered prior this rpc.
+ // A file download. The download has to be registered prior this rpc in the
+ // RetrieveRequest.
rpc FileDownload(FileDownloadRequest) returns (FileDownloadResponse);
}
--
GitLab