Merge branch 'f-files' into f-consolidation

Makefile

@@ -3,3 +3,6 @@ ALL:
 
 formatting:
 	clang-format -i $$(find proto/ -type f -iname "*.proto")
+
+linting:
+	buf lint

proto/caosdb/entity/v1alpha1/main.proto

@@ -431,21 +431,31 @@ message Entity {
   repeated Property properties = 9;
   // Parents of this entity.
   repeated Parent parents = 10;
-  // Errors for this entity which occurred during the current transaction. The
-  // presence of errors indicates that a transaction was not successful and the
-  // errors describe why this is the case.
-  repeated Message errors = 11;
-  // Warnings for this entity which occurred during the current transaction,
-  // but the transaction was nevertheless successful. The presence of warnings
-  // indicates that the entity is in an undesired or non-standard state itself
-  // or in an undesired or non-standard relation to other entities or the
-  // server's state or  configuration. This might cause errors in the future
-  // and should be remedied if possible.
-  repeated Message warnings = 12;
-  // Info messages for this entity which may inform the client about anything
-  // related to the transaction or the new state of the entity itself or in
-  // relation to other entities or the server's state or configuration.
-  repeated Message infos = 13;
+
+  // File meta data. The actual binary data has to uploaded or downloaded using
+  // the File message.
+  FileDescriptor file_descriptor = 14;
+}
+
+// The file descriptor contains the meta data of the file which describes the
+// file and where the binary data can be retrieved.
+message FileDescriptor {
+  // The corresponding file entity.
+  string entity_id = 1;
+  // The path of this file in the internal file system.
+  string path = 2;
+  // The size of the file
+  int64 size = 3;
+  // Hash sums for consistency checks.
+  repeated Hash hashes = 4;
+}
+
+// This represents a hash sum of a file.
+message Hash {
+  // The algorithm of the hash sum, e.g. SHA512
+  string algorithm = 1;
+  // The value of the hash sum
+  string value = 2;
 }
 
 // The property importance.
@@ -502,26 +512,81 @@ message Parent {
   repeated Message infos = 6;
 }
 
-// Response to a retrieve request
-message RetrieveResponse {
-  // Payload of the retrieval
-  oneof query_response {
-    // Result of a FIND query or a request by id.
-    Entity entity = 1;
-    // Result of a SELECT query
-    SelectQueryResult select_result = 3;
-    // Result of a COUNT query
-    int64 count_result = 4;
-  }
+// Wraps an entity and associates it with a previous file upload via a
+// transmission id.
+message EntityRequest {
+  // The entity.
+  Entity entity = 1;
+  // The transmission id (if any). Note: upload_ids are only meaningful for
+  // File entities.
+  FileTransmissionId upload_id = 2;
+}
+
+// Wraps an entity and associates it with a transimission id for a file
+// download
+message EntityResponse {
+  // The entity.
+  Entity entity = 1;
+  // The transmission id (if any). Note: download_ids are only meaningful for
+  // File entities.
+  FileTransmissionId download_id = 2;
+  // Errors for this entity which occurred during the current transaction. The
+  // presence of errors indicates that a transaction was not successful and the
+  // errors describe why this is the case.
+  repeated Message errors = 3;
+  // Warnings for this entity which occurred during the current transaction,
+  // but the transaction was nevertheless successful. The presence of warnings
+  // indicates that the entity is in an undesired or non-standard state itself
+  // or in an undesired or non-standard relation to other entities or the
+  // server's state or  configuration. This might cause errors in the future
+  // and should be remedied if possible.
+  repeated Message warnings = 4;
+  // Info messages for this entity which may inform the client about anything
+  // related to the transaction or the new state of the entity itself or in
+  // relation to other entities or the server's state or configuration.
+  repeated Message infos = 5;
+}
+
+// A response containing only the id (of a deleted, inserted, or updated
+// entity) and the usual trias of entity messages.
+message IdResponse {
+  // The entity id.
+  string id = 1;
+  // Errors for this entity which occurred during the current transaction. The
+  // presence of errors indicates that a transaction was not successful and the
+  // errors describe why this is the case.
+  repeated Message errors = 2;
+  // Warnings for this entity which occurred during the current transaction,
+  // but the transaction was nevertheless successful. The presence of warnings
+  // indicates that the entity is in an undesired or non-standard state itself
+  // or in an undesired or non-standard relation to other entities or the
+  // server's state or  configuration. This might cause errors in the future
+  // and should be remedied if possible.
+  repeated Message warnings = 3;
+  // Info messages for this entity which may inform the client about anything
+  // related to the transaction or the new state of the entity itself or in
+  // relation to other entities or the server's state or configuration.
+  repeated Message infos = 4;
 }
 
-// CaosDB Query
+///////////////////////////////////////////////////////////////////////////
+// Query stuff
+///////////////////////////////////////////////////////////////////////////
+
+// Query message. Currently this messge only wraps the query string. This might
+// be extended in the future.
 message Query {
   // The string representation of the query, e.g. "FIND Person WITH name =
   // Aisha"
   string query = 1;
 }
 
+// Result of a FIND query.
+message FindQueryResult {
+  // Entities in the result set.
+  repeated EntityResponse result_set = 1;
+}
+
 // A single result row of a SELECT query
 message SelectQueryRows {
   // The cells of the table row.
@@ -536,51 +601,135 @@ message SelectQueryResult {
   repeated SelectQueryRows data_rows = 2;
 }
 
-// Request which specifies one or multiple entities by query or by id.
-message QueryOrIdRequest {
-  // We wrap these two together because this makes it easier to add further
-  // requests (like "name", or "path") in the future.
+///////////////////////////////////////////////////////////////////////////
+// Request and Responses for single CRUD-Transaction
+///////////////////////////////////////////////////////////////////////////
+
+// Single retrieve request (by query or by id)
+message RetrieveRequest {
+  // specify be query or by id
   oneof wrapped_request {
     // A single id
     string id = 1;
     // A query
     Query query = 2;
   }
+  // Is the client requesting to download the file associated with the
+  // retrieved entity or query results?
+  bool register_file_download = 3;
+  //TODO Settings for the download. Ignored if register_file_download is false.
+  //FileTransmissionSettings download_settings = 4;
 }
 
-// Single response containing either an id, a COUNT query result, a SELECT-table
-// or complete entities.
-message TransactionResponse {
-  // We wrap these together in order to make this a repeatable field in the
-  // MultiTransactionResponse.
+// Response to a retrieve request
+message RetrieveResponse {
+  // Payload of the retrieval
+  oneof retrieve_response {
+    // Result of a request by id or any other unique identifier.
+    EntityResponse entity_response = 1;
+    // Result of a FIND query
+    FindQueryResult find_result = 2;
+    // Result of a SELECT query
+    SelectQueryResult select_result = 3;
+    // Result of a COUNT query
+    int64 count_result = 4;
+  }
+}
+
+// Single delete request (by id)
+message DeleteRequest {
+  // Wrapped to make it easier to add delete-by-query in the future.
+  oneof wrapped_request {
+    // A single id
+    string id = 1;
+  }
+}
+
+// Response to a single delete request.
+message DeleteResponse {
+  // Wrapped to make it easier to optionally return the full entity in the
+  // future.
   oneof wrapped_response {
-    // Single response to a delete transaction.
-    IdResponse delete_response = 1;
-    // Single response to an update transaction.
-    IdResponse update_response = 2;
-    // Single response to a retrieve transaction.
-    RetrieveResponse retrieve_response = 3;
-    // Single response to an insert transaction.
-    IdResponse insert_response = 4;
+    // Id and messages of the deleted entity.
+    IdResponse id_response = 1;
+  }
+}
+
+// Request to update a single Entity
+message UpdateRequest {
+  // The entity which is to be updated.
+  EntityRequest entity_request = 1;
+}
+
+// Response to a single update request.
+message UpdateResponse {
+  // Wraps an IdResponse. Might be extended in the future to optionally return
+  // the full entity.
+  oneof wrapped_response {
+    // Id and messages of the updated entity.
+    IdResponse id_response = 1;
+  }
+}
+
+// Request to insert a single entity.
+message InsertRequest {
+  // The new entity.
+  EntityRequest entity_request = 1;
+}
+
+// Response to a single insert request.
+message InsertResponse {
+  // Wraps an IdResponse. Might be extended in the future to optionally return
+  // also the full entity.
+  oneof wrapped_response {
+    // Id and messages of the inserted entity.
+    IdResponse id_response = 1;
   }
 }
 
+///////////////////////////////////////////////////////////////////////////
+// Highlevel Requests and Responses for multiple mixed CRUD transactions.
+///////////////////////////////////////////////////////////////////////////
+
 // Single request containing either a query, an id or a complete entity.
 message TransactionRequest {
   // We wrap these together in order to make this a repeatable field in the
   // MultiTransactionRequest.
   oneof wrapped_requests {
     // Single request for a retrieve transaction.
-    QueryOrIdRequest retrieve_request = 1;
+    RetrieveRequest retrieve_request = 1;
     // Single request for an update transaction.
-    Entity update_request = 2;
+    UpdateRequest update_request = 2;
     // Single request for an insert transaction.
-    Entity insert_request = 3;
+    InsertRequest insert_request = 3;
     // Single request for a delete transaction.
-    QueryOrIdRequest delete_request = 4;
+    DeleteRequest delete_request = 4;
   }
 }
 
+// Single response containing either an id, a COUNT query result, a SELECT-table
+// or complete entities.
+message TransactionResponse {
+  // We wrap these together in order to make this a repeatable field in the
+  // MultiTransactionResponse.
+  oneof transaction_response {
+    // Single response to a delete transaction.
+    DeleteResponse delete_response = 1;
+    // Single response to an update transaction.
+    UpdateResponse update_response = 2;
+    // Single response to a retrieve transaction.
+    RetrieveResponse retrieve_response = 3;
+    // Single response to an insert transaction.
+    InsertResponse insert_response = 4;
+  }
+}
+
+// Wraps multiple sub-transaction requests of mixed types.
+message MultiTransactionRequest {
+  // The actual payload.
+  repeated TransactionRequest requests = 1;
+}
+
 // Wraps the reponses to multiple sub-transactions of mixed types.
 message MultiTransactionResponse {
   // The actual payload.
@@ -620,3 +769,113 @@ service EntityTransactionService {
   rpc MultiTransaction(MultiTransactionRequest)
       returns (MultiTransactionResponse);
 }
+
+///////////////////////////////////////////////////////////////////////////
+// File transmission
+///////////////////////////////////////////////////////////////////////////
+
+// Stores a single chunk of a file
+message FileChunk {
+  // Temporary identifier containing the file and registration_id.
+  FileTransmissionId file_transmission_id = 1;
+  // Binary data of a chunk.
+  bytes data = 2;
+}
+
+// Temporary identifier of a single file during a transmission.
+message FileTransmissionId {
+  // The registration id which has been issued by the target of the trans
+  string registration_id = 1;
+  // A temporary identifier which identifies the file of this chunk. The
+  // file_id is also used by transaction to associate entities (which are to be
+  // inserted or updated) with a binary blob.
+  string file_id = 2;
+}
+
+// Settings for the the file transmission.
+message FileTransmissionSettings {
+  // The maximum chunk size.
+  int64 max_chunk_size = 1;
+  // The maximum file size.
+  int64 max_file_size = 2;
+}
+
+// Indicates whether a registration (for upload or download) has been accepted
+// or rejected.
+enum RegistrationStatus {
+  // The registration status is unspecified.
+  REGISTRATION_STATUS_UNSPECIFIED = 0;
+  // The registration has been accepted and the client may proceed with the
+  // actual transmissions.
+  REGISTRATION_STATUS_ACCEPTED = 1;
+  // The registration has been rejected and the client should not try to
+  // proceed with the transmission.
+  REGISTRATION_STATUS_REJECTED = 2;
+}
+
+// Indicates the state of an upload or a download (a stream of chunks).
+enum TransmissionStatus {
+  // The transmission status is unspecified.
+  TRANSMISSION_STATUS_UNSPECIFIED = 0;
+  // The transmission has been successful.
+  TRANSMISSION_STATUS_SUCCESS = 1;
+  // The transmission terminated with errors.
+  TRANSMISSION_STATUS_ERROR = 2;
+  // The transmission is incomplete and the client may send/request the next
+  // chunk.
+  TRANSMISSION_STATUS_GO_ON = 3;
+}
+
+// Register a file upload.
+message RegisterFileUploadRequest {}
+
+// Response of the file server upon an upload registration request.
+message RegisterFileUploadResponse {
+  // Whether the server accepted or rejected the registration.
+  RegistrationStatus status = 1;
+  // The registration id is used to identify chunks and files which belong to
+  // the same upload .
+  string registration_id = 2;
+  // The server's transmission settings for the upload.
+  FileTransmissionSettings upload_settings = 4;
+}
+
+// Request for a file upload which has been registered previously. Chunks may be
+// send in any order.
+message FileUploadRequest {
+  // A single file chunk
+  FileChunk chunk = 1;
+}
+
+// Response of the server upon a finished FileUpload.
+message FileUploadResponse {
+  // Status of the upload.
+  TransmissionStatus status = 1;
+}
+
+// Request for a file download which has been registered previously.
+message FileDownloadRequest {
+  // Request the next chunk for this file.
+  FileTransmissionId file_transmission_id = 1;
+}
+
+// Response containing a chunk of a file.
+message FileDownloadResponse {
+  // Status of the download
+  TransmissionStatus status = 1;
+  // A single file chunk
+  FileChunk chunk = 2;
+}
+
+// File Transaction Service
+service FileTransmissionService {
+  // Register a file upload. This needs to be done prior to the actual upload
+  // and prior to the transaction request which uses the uploaded files.
+  rpc RegisterFileUpload(RegisterFileUploadRequest)
+      returns (RegisterFileUploadResponse);
+  // The actual file upload. The upload has to be registered prior to this rpc.
+  rpc FileUpload(stream FileUploadRequest) returns (FileUploadResponse);
+  // A file download. The download has to be registered prior this rpc in the
+  // RetrieveRequest.
+  rpc FileDownload(FileDownloadRequest) returns (stream FileDownloadResponse);
+}