diff --git a/.docker/Dockerfile b/.docker/Dockerfile index 102bca2c1752accd58da615dc2050830c7b18cf9..e84f63ccfb2a79be6b1dcf58fdc15f3fedc8a3cc 100644 --- a/.docker/Dockerfile +++ b/.docker/Dockerfile @@ -12,7 +12,7 @@ RUN apt-get install -y protobuf-compiler # buf RUN BIN="/usr/local/bin" && \ - VERSION="0.43.2" && \ + VERSION="1.50.0" && \ BINARY_NAME="buf" && \ curl -sSL \ "https://github.com/bufbuild/buf/releases/download/v${VERSION}/${BINARY_NAME}-$(uname -s)-$(uname -m)" \ diff --git a/CHANGELOG.md b/CHANGELOG.md index eb572ac428a00b4965b5308e51c52a158e66ea16..d278c7c8dfb769d692dc7736df893632aef61767 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,8 +10,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added ### + +- The file transmission API was extended to support resumable file uploads via an LinkAhead + Upload Protocol. The currently planned protocol is the TUS protocol. The resumable upload + API is still in alpha status. + ### Changed ### +- The file transmission API was split from the entity API into its own proto file (v1alpha). + The entity API version is incremented to v2alpha1. is currently identical to v1 but + lacking the file transmission API). + ### Deprecated ### ### Removed ### diff --git a/Makefile b/Makefile index 5c2bedc6033b1341838f6f40532b83ff0bc02a9b..a4a2fcd845fc6567a9fe6b26312a4d1aef7aad26 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,17 @@ ALL: @echo "THIS IS ONLY FOR DEVELOPMENT. DON'T USE IT" +src: + mkdir -p src/java &&\ + mkdir -p src/python &&\ + mkdir -p src/cpp &&\ + mkdir -p src/js&&\ + protoc -Iproto --java_out=src/java --python_out=src/python --cpp_out=src/cpp --js_out=src/js $$(find ./proto -type f -iname "*.proto") + +clean: + rm -rf src +.PHONY: clean src + formatting: clang-format -i $$(find proto/ -type f -iname "*.proto") diff --git a/README_SETUP.md b/README_SETUP.md index 78222f06fbd69347ab3dbc54ac4943caa5eabb9c..746b15497267349e61fe2b23faf57115455d9d4e 100644 --- a/README_SETUP.md +++ b/README_SETUP.md @@ -17,6 +17,11 @@ contains all necessary configuration. We use [buf](https://docs.buf.build/) for linting. Run `buf lint`. The `buf.yaml` contains all necessary configuration. +## Source code generation + +`make src` generates the protobuf code for Java, C++, Python and JavaScript using +the locally installed `protoc`. + ## Docs We use [cmake](https://cmake.org) as build tool for the documenation and diff --git a/buf.yaml b/buf.yaml index 57d0c99f3996e730f83971c4b7b0d2626b848ea7..b2d4bd44ec2af3343e232d17757862f4d9697a13 100644 --- a/buf.yaml +++ b/buf.yaml @@ -1,15 +1,15 @@ -# configuration for buf. See https://docs.buf.build/ version: v1beta1 build: roots: - proto lint: use: - - DEFAULT - COMMENTS - - OTHER + - DEFAULT + - OTHER except: - ignore: + breaking: use: - FILE + diff --git a/proto/caosdb/CMakeLists.txt b/proto/caosdb/CMakeLists.txt index f7fa61531e3c135afc139c41337d8a59a3cca39f..62362586aed316bd1f056a38380170f06fcb7c88 100644 --- a/proto/caosdb/CMakeLists.txt +++ b/proto/caosdb/CMakeLists.txt @@ -21,7 +21,9 @@ set(CAOSDB_API_PACKAGES caosdb.info.v1 caosdb.entity.v1 + caosdb.entity.v2alpha1 caosdb.acm.v1alpha1 + caosdb.file_transmission.v1alpha1 ) # pass variable to parent scope diff --git a/proto/caosdb/entity/v2alpha1/entity.proto b/proto/caosdb/entity/v2alpha1/entity.proto new file mode 100644 index 0000000000000000000000000000000000000000..dedf01a1c890f47590cc44290fa9ae9ada919d85 --- /dev/null +++ b/proto/caosdb/entity/v2alpha1/entity.proto @@ -0,0 +1,846 @@ +// +// This file is a part of the LinkAhead Project. +// +// Copyright (C) 2021-2022 Timm Fitschen <t.fitschen@indiscale.com> +// Copyright (C) 2021-2025 IndiScale GmbH <info@indiscale.com> +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see <https://www.gnu.org/licenses/>. +// + +// This is the main file of the caosdb.entity.v1 package. +syntax = "proto3"; +package caosdb.entity.v2alpha1; + +import "caosdb/file_transmission/v1alpha1/file_transmission.proto"; // for FileTransmissionId + +option cc_enable_arenas = true; +option java_multiple_files = true; +option java_package = "org.caosdb.api.entity.v2alpha1"; + +// Data type for references to other entities. +message ReferenceDataType { + // The name of the record type which defines the scope of the reference. + string name = 1; +} + +// Data type for list value. +message ListDataType { + // A list can be a list of atomic data types or references. + oneof list_data_type { + // List of atomic data types (e.g. List of DOUBLE) + AtomicDataType atomic_data_type = 1; + // List of reference (e.g. List of Person(s)) + ReferenceDataType reference_data_type = 2; + } +} + +// Atomic data types. +enum AtomicDataType { + // The data type is unset/unknown. + ATOMIC_DATA_TYPE_UNSPECIFIED = 0; + // TEXT data type. + ATOMIC_DATA_TYPE_TEXT = 1; + // DOUBLE data type. + ATOMIC_DATA_TYPE_DOUBLE = 2; + // DATETIME data type. + ATOMIC_DATA_TYPE_DATETIME = 3; + // INTEGER data type. + ATOMIC_DATA_TYPE_INTEGER = 4; + // BOOLEAN data type. + ATOMIC_DATA_TYPE_BOOLEAN = 5; +} + +// The property value's data type. +message DataType { + // Wraps atomic types, LIST and REFERENCE + oneof data_type { + // The atomic data types. + AtomicDataType atomic_data_type = 1; + // A list data type. + ListDataType list_data_type = 2; + // A reference data type. + ReferenceDataType reference_data_type = 3; + } +} + +// Represents collection values (e.g. for lists). +message CollectionValues { + // List of scalar values. + repeated ScalarValue values = 1; +} + +// Represents special values which are otherwise hard to transfer via protobuf. +enum SpecialValue { + // Represent the NULL value. + SPECIAL_VALUE_UNSPECIFIED = 0; + // The empty string. + SPECIAL_VALUE_EMPTY_STRING = 1; +} + +// A scalar property value. +message ScalarValue { + // Wraps different optimized representations of a single value. + oneof scalar_value { + // The value as integer + int64 integer_value = 1; + // The value as double + double double_value = 2; + // The value as bool + bool boolean_value = 3; + // The value as string. For the empty string, you may use special_value. + string string_value = 4; + // The value is a special value (e.g. NULL or the empty string) + SpecialValue special_value = 5; + } +} + +// The value of a property. +message Value { + // Wraps scalar and multi-dimensional values. + oneof value { + // The scalar values. + ScalarValue scalar_value = 1; + // The list values. + CollectionValues list_values = 2; + } +} + +// The roles of entities. +enum EntityRole { + // Unspecified role. + ENTITY_ROLE_UNSPECIFIED = 0; + // RecordType + ENTITY_ROLE_RECORD_TYPE = 1; + // Record + ENTITY_ROLE_RECORD = 2; + // Property + ENTITY_ROLE_PROPERTY = 3; + // File + ENTITY_ROLE_FILE = 4; +} + +// Messages codes. Entities and transactions contain errors, warnings and info +// messages which use these codes. +enum MessageCode { + // Unspecific code. Use this for anything where a client is not expected to + // handle this message in a specific way. + MESSAGE_CODE_UNSPECIFIED = 0; + // Unknown code. This value is a fall-back which may be used when the message + // code was not in this list. An unknown error code is a sign of a wrong otr + // out-dated implementation on either side. + MESSAGE_CODE_UNKNOWN = 1; + + // The requested entity (retrieve, delete, or update) does not exist. + MESSAGE_CODE_ENTITY_DOES_NOT_EXIST = 2; + + // The entity has no properties. + MESSAGE_CODE_ENTITY_HAS_NO_PROPERTIES = 3; + + // This entity has been deleted successfully. + MESSAGE_CODE_ENTITY_HAS_BEEN_DELETED_SUCCESSFULLY = 4; + + // On insert or update: Entity has unqualified properties. + MESSAGE_CODE_ENTITY_HAS_UNQUALIFIED_PROPERTIES = 5; + + // On insert or update: Entity has unqualified parents. + MESSAGE_CODE_ENTITY_HAS_UNQUALIFIED_PARENTS = 6; + + // On update: Entity, Property or Parent has no ID and the server is + // configured to require a valid ID for all entities which are to be updated + // and for their properties and parents as well. + MESSAGE_CODE_ENTITY_HAS_NO_ID = 7; + + // On delete: Entity is required by other entities which are not to be + // deleted. + MESSAGE_CODE_REQUIRED_BY_PERSISTENT_ENTITY = 8; + + // Property has no datatype. + MESSAGE_CODE_PROPERTY_HAS_NO_DATA_TYPE = 9; + + // Entity has no description. + MESSAGE_CODE_ENTITY_HAS_NO_DESCRIPTION = 10; + + // Entity has no name. + MESSAGE_CODE_ENTITY_HAS_NO_NAME = 11; + + // On insert or update: An obligatory property is missing. + MESSAGE_CODE_OBLIGATORY_PROPERTY_MISSING = 12; + + // On insert or update: Entity has no parents. + MESSAGE_CODE_ENTITY_HAS_NO_PARENTS = 13; + + // No target path specified. + MESSAGE_CODE_FILE_HAS_NO_TARGET_PATH = 14; + + // This target path is not allowed. + MESSAGE_CODE_TARGET_PATH_NOT_ALLOWED = 15; + + // This target path does already exist. + MESSAGE_CODE_TARGET_PATH_EXISTS = 16; + + // Property has no unit. + MESSAGE_CODE_PROPERTY_HAS_NO_UNIT = 17; + + // Cannot parse value. + MESSAGE_CODE_CANNOT_PARSE_VALUE = 18; + + // Checksum test failed. File is corrupted. + MESSAGE_CODE_CHECKSUM_TEST_FAILED = 19; + + // Size test failed. File is corrupted. + MESSAGE_CODE_SIZE_TEST_FAILED = 20; + + // Cannot create parent folder in the file system. + MESSAGE_CODE_CANNOT_CREATE_PARENT_FOLDER = 21; + + // File has not been uploaded. + MESSAGE_CODE_FILE_HAS_NOT_BEEN_UPLOAED = 22; + + // Cannot move file to it's target folder. + MESSAGE_CODE_CANNOT_MOVE_FILE_TO_TARGET_PATH = 23; + + // Cannot parse value as datetime value. + MESSAGE_CODE_CANNOT_PARSE_DATETIME_VALUE = 24; + + // Cannot parse value as double value. + MESSAGE_CODE_CANNOT_PARSE_DOUBLE_VALUE = 25; + + // Cannot parse value as integer value. + MESSAGE_CODE_CANNOT_PARSE_INT_VALUE = 26; + + // Cannot parse value as boolean value (either 'true' or 'false', case + // insensitive). + MESSAGE_CODE_CANNOT_PARSE_BOOL_VALUE = 27; + + // File could not be found. + MESSAGE_CODE_FILE_NOT_FOUND = 28; + + // A warning occured while processing an entity with the strict flag. + MESSAGE_CODE_WARNING_OCCURED = 29; + + // On insert or update: Name is already in use. Choose a different name or + // reuse an existing entity. + MESSAGE_CODE_ENTITY_NAME_IS_NOT_UNIQUE = 30; + + // This query finished with errors. + MESSAGE_CODE_QUERY_EXCEPTION = 31; + + // An unknown error occured during the transaction and it was rolled back. + MESSAGE_CODE_TRANSACTION_ROLL_BACK = 32; + + // Unknown unit. Values with this unit cannot be converted to other units + // when used in search queries. + MESSAGE_CODE_UNKNOWN_UNIT = 34; + + // You are not allowed to do this. + MESSAGE_CODE_AUTHORIZATION_ERROR = 35; + + // Reference not qualified. The value of this Reference Property is to be a + // child of its data type. + MESSAGE_CODE_REFERENCE_IS_NOT_ALLOWED_BY_DATA_TYPE = 36; + + // This entity cannot be identified due to name duplicates. + MESSAGE_CODE_ENTITY_NAME_DUPLICATES = 37; + + // This data type cannot be identified due to name duplicates. + MESSAGE_CODE_DATA_TYPE_NAME_DUPLICATES = 38; + + // This entity cannot be identified as it didn't come with a name or id. + MESSAGE_CODE_ENTITY_HAS_NO_NAME_OR_ID = 39; + + // Affiliation is not defined for this child-parent constellation. + MESSAGE_CODE_AFFILIATION_ERROR = 40; + + // An error occured during the parsing of this query. Maybe you were using a + // wrong syntax? + MESSAGE_CODE_QUERY_PARSING_ERROR = 41; + + // A property which has 'name' as its parent must have a TEXT data type. + MESSAGE_CODE_NAME_PROPERTIES_MUST_BE_TEXT = 42; + + // This entity had parent duplicates. That is meaningless and only one parent + // had been inserted. + MESSAGE_CODE_PARENT_DUPLICATES_WARNING = 43; + + // This entity had parent duplicates. Parent duplicates are meaningless and + // would be ignored (and inserted only once). But these parents had diverging + // inheritance instructions which cannot be processed. + MESSAGE_CODE_PARENT_DUPLICATES_ERROR = 44; + + // One or more entities are not qualified. None of them have been + // inserted/updated/deleted. + MESSAGE_CODE_ATOMICITY_ERROR = 45; + + // There is no such role. + MESSAGE_CODE_NO_SUCH_ENTITY_ROLE = 46; + + // This entity cannot be deleted due to dependency problems + MESSAGE_CODE_REQUIRED_BY_UNQUALIFIED = 47; + + // This entity has an unqualified reference. I.e. the referenced entity is + // not qualified for the transaction. + MESSAGE_CODE_ENTITY_HAS_UNQUALIFIED_REFERENCE = 48; + + // Referenced entity does not exist. + MESSAGE_CODE_REFERENCED_ENTITY_DOES_NOT_EXIST = 49; + + // This reference cannot be identified due to name duplicates. + MESSAGE_CODE_REFERENCE_NAME_DUPLICATES = 50; + + // The datatype which is to be inherited could not be detected due to + // divergent datatypes of at least two parents. + MESSAGE_CODE_DATA_TYPE_INHERITANCE_AMBIGUOUS = 51; + + // This datatype does not accept collections of values (e.g. Lists). + MESSAGE_CODE_DATA_TYPE_DOES_NOT_ACCEPT_COLLECTION_VALUES = 52; + + // This unit cannot be parsed. + MESSAGE_CODE_CANNOT_PARSE_UNIT = 53; + + // This property is an additional property which has no corresponding + // property among the properties of the parents. + MESSAGE_CODE_ADDITIONAL_PROPERTY = 54; + + // This property overrides the datatype. + MESSAGE_CODE_PROPERTY_WITH_DATA_TYPE_OVERRIDE = 55; + + // This property overrides the description. + MESSAGE_CODE_PROPERTY_WITH_DESCRIPTION_OVERRIDE = 56; + + // This property overrides the name. + MESSAGE_CODE_PROPERTY_WITH_NAME_OVERRIDE = 57; + + // Indicates that this server cannot process this integer value because it is + // out of range. The integer range is an implementation detail and thus this + // can happen even though the API allows the value. + MESSAGE_CODE_INTEGER_VALUE_OUT_OF_RANGE = 58; + + // This entity caused an unexpected integrity violation. This is a strong + // indicator for a server bug. Please report. + MESSAGE_CODE_INTEGRITY_VIOLATION = 59; +} + +// Messages are used by server's or client's to transport *transient* +// information, e.g. general information about a transaction, warnings, or +// errors. +message Message { + // The code is a machine-readable identifier. + int32 code = 1; + // A short description of the message which is indented for human readers. + string description = 2; +} + +// Version of an entity. +message Version { + // Version id + string id = 1; +} + +// This is the main class for CaosDB's entities. +message Entity { + // Entity ID. The entity id is the primary key for all entities. + string id = 1; + // Entity name. The entity name is a possibly ambiguous identifier which is + // intended for human readers. In some cases, names can be restricted to be + // unique when this makes sense, but only as means to prevent the creation of + // duplicates. + string name = 2; + // Entity description. A short description of the purpose and meaning of an + // entity which is intended for human readers. + string description = 3; + // Entity version. This entity message represents only one version of a + // multitude of versions of the same entity (i.e. they have the same Entity + // ID). The different versions represent the change of an entity through + // transactions. + Version version = 4; + // Entity role (e.g. RecordType, Property, Record) + EntityRole role = 5; + // Default unit of this entity (only used by properties). + string unit = 6; + // Default data type of this entity (only used by properties). + DataType data_type = 7; + // Default value of this entity (only used by properties). + Value value = 8; + // Properties of this entity. + repeated Property properties = 9; + // Parents of this entity. + repeated Parent parents = 10; + + // File meta data. The actual binary data has to uploaded or downloaded using + // the File message. + FileDescriptor file_descriptor = 14; +} + +// The file descriptor contains the meta data of the file which describes the +// file and where the binary data can be retrieved. +message FileDescriptor { + // The corresponding file entity. + string entity_id = 1; + // The path of this file in the internal file system. + string path = 2; + // The size of the file + int64 size = 3; + // Hash sums for consistency checks. + repeated Hash hashes = 4; +} + +// This represents a hash sum of a file. +message Hash { + // The algorithm of the hash sum, e.g. SHA512 + string algorithm = 1; + // The value of the hash sum + string value = 2; +} + +// The property importance. +enum Importance { + // The importance is unset/unknown. + IMPORTANCE_UNSPECIFIED = 0; + // The property is obligatory for children of this entity. The server will + // flag an error when an obligatory property of a parent is missing during + // updates or insertions of a child entity. It says: "YOU MUST!" + IMPORTANCE_OBLIGATORY = 1; + // The property is recommended for children of this entity. The server will + // flag a warning when a recommended property of a parent is missing during + // updates or insertions of a child entity. It says: "YOU SHOULD!" + IMPORTANCE_RECOMMENDED = 2; + // The property is merely suggested for children of this entity. It says: + // "YOU MAY WANT TO?" + IMPORTANCE_SUGGESTED = 3; + // The property is fix and child entities should not add this property to + // themselves (This is a bit like static in object-oriented programming + // languages). It says: "KINDLY IGNORE THIS ONE FOR CHILDREN OF THIS ENTITY" + IMPORTANCE_FIX = 4; +} + +// Property of an entity. Don't confuse with an Entity with role="Property" +message Property { + // ID of the implemented Entity + string id = 1; + // Property name + string name = 2; + // Property description + string description = 3; + // Property value + Value value = 4; + // Property importance + Importance importance = 5; + // Property unit + string unit = 6; + // Property datatype + DataType data_type = 7; + // Errors of this property + repeated Message errors = 8; + // Warnings of this property + repeated Message warnings = 9; + // Info messages of this property + repeated Message infos = 10; +} + +// Parent of an entity. +message Parent { + // ID of the parernt entity. + string id = 1; + // Name of the parernt entity. + string name = 2; + // Description of the parent entity. + string description = 3; + // Errors of this parent + repeated Message errors = 4; + // Warnings of this parent + repeated Message warnings = 5; + // Info messages of this parent + repeated Message infos = 6; +} + +// Wraps an entity and associates it with a previous file upload via a +// transmission id. +message EntityRequest { + // The entity. + Entity entity = 1; + // The transmission id (if any). Note: upload_ids are only meaningful for + // File entities. + caosdb.file_transmission.v1alpha1.FileTransmissionId upload_id = 2; +} + +// Wraps an entity and associates it with a transimission id for a file +// download +message EntityResponse { + // The entity. + Entity entity = 1; + // The transmission id (if any). Note: download_ids are only meaningful for + // File entities. + caosdb.file_transmission.v1alpha1.FileTransmissionId download_id = 2; + // Errors for this entity which occurred during the current transaction. The + // presence of errors indicates that a transaction was not successful and the + // errors describe why this is the case. + repeated Message errors = 3; + // Warnings for this entity which occurred during the current transaction, + // but the transaction was nevertheless successful. The presence of warnings + // indicates that the entity is in an undesired or non-standard state itself + // or in an undesired or non-standard relation to other entities or the + // server's state or configuration. This might cause errors in the future + // and should be remedied if possible. + repeated Message warnings = 4; + // Info messages for this entity which may inform the client about anything + // related to the transaction or the new state of the entity itself or in + // relation to other entities or the server's state or configuration. + repeated Message infos = 5; +} + +// A response containing only the id (of a deleted, inserted, or updated +// entity) and the usual trias of entity messages. +message IdResponse { + // The entity id. + string id = 1; + // The entity version. + Version version = 5; + // Errors for this entity which occurred during the current transaction. The + // presence of errors indicates that a transaction was not successful and the + // errors describe why this is the case. + repeated Message errors = 2; + // Warnings for this entity which occurred during the current transaction, + // but the transaction was nevertheless successful. The presence of warnings + // indicates that the entity is in an undesired or non-standard state itself + // or in an undesired or non-standard relation to other entities or the + // server's state or configuration. This might cause errors in the future + // and should be remedied if possible. + repeated Message warnings = 3; + // Info messages for this entity which may inform the client about anything + // related to the transaction or the new state of the entity itself or in + // relation to other entities or the server's state or configuration. + repeated Message infos = 4; +} + +/////////////////////////////////////////////////////////////////////////// +// Query stuff +/////////////////////////////////////////////////////////////////////////// + +// Query message. Currently this messge only wraps the query string. This might +// be extended in the future. +message Query { + // The string representation of the query, e.g. "FIND Person WITH name = + // Aisha" + string query = 1; +} + +// Result of a FIND query. +message FindQueryResult { + // Entities in the result set. + repeated EntityResponse result_set = 1; +} + +// A single result row of a SELECT query +message SelectQueryRow { + // The cells of the table row contain the value of the SELECT'ed fields. The + // order of the cells follows the order of the columns of the + // SelectQueryHeader. + repeated Value cells = 2; +} + +// A single header column of the result of a SELECT query +message SelectQueryColumn { + // The column name. This is one of the SELECT'ed fields. + string name = 1; +} + +// The header line of the result of a SELECT query +message SelectQueryHeader { + // List of columns in this table. The order defines the order of the cells in + // each of the SelectQueryRows. + repeated SelectQueryColumn columns = 1; +} + +// A table which is a result of a SELECT query +message SelectQueryResult { + // The header contains the SELECT'ed fields + SelectQueryHeader header = 3; + // The actual results + repeated SelectQueryRow data_rows = 4; +} + +/////////////////////////////////////////////////////////////////////////// +// Request and Responses for single CRUD-Transaction +/////////////////////////////////////////////////////////////////////////// + +// Single retrieve request (by query or by id) +message RetrieveRequest { + // specify be query or by id + oneof wrapped_request { + // A single id + string id = 1; + // A query + Query query = 2; + } + // Is the client requesting to download the file associated with the + // retrieved entity or query results? + bool register_file_download = 3; +} + +// Response to a retrieve request +message RetrieveResponse { + // Payload of the retrieval + oneof retrieve_response { + // Result of a request by id or any other unique identifier. + EntityResponse entity_response = 1; + // Result of a FIND query + FindQueryResult find_result = 2; + // Result of a SELECT query + SelectQueryResult select_result = 3; + // Result of a COUNT query + int64 count_result = 4; + } +} + +// Single delete request (by id) +message DeleteRequest { + // Wrapped to make it easier to add delete-by-query in the future. + oneof wrapped_request { + // A single id + string id = 1; + } +} + +// Response to a single delete request. +message DeleteResponse { + // Wrapped to make it easier to optionally return the full entity in the + // future. + oneof wrapped_response { + // Id and messages of the deleted entity. + IdResponse id_response = 1; + } +} + +// Request to update a single Entity +message UpdateRequest { + // The entity which is to be updated. + EntityRequest entity_request = 1; +} + +// Response to a single update request. +message UpdateResponse { + // Wraps an IdResponse. Might be extended in the future to optionally return + // the full entity. + oneof wrapped_response { + // Id and messages of the updated entity. + IdResponse id_response = 1; + } +} + +// Request to insert a single entity. +message InsertRequest { + // The new entity. + EntityRequest entity_request = 1; +} + +// Response to a single insert request. +message InsertResponse { + // Wraps an IdResponse. Might be extended in the future to optionally return + // also the full entity. + oneof wrapped_response { + // Id and messages of the inserted entity. + IdResponse id_response = 1; + } +} + +/////////////////////////////////////////////////////////////////////////// +// Highlevel Requests and Responses for multiple mixed CRUD transactions. +/////////////////////////////////////////////////////////////////////////// + +// Single request containing either a query, an id or a complete entity. +message TransactionRequest { + // We wrap these together in order to make this a repeatable field in the + // MultiTransactionRequest. + oneof wrapped_requests { + // Single request for a retrieve transaction. + RetrieveRequest retrieve_request = 1; + // Single request for an update transaction. + UpdateRequest update_request = 2; + // Single request for an insert transaction. + InsertRequest insert_request = 3; + // Single request for a delete transaction. + DeleteRequest delete_request = 4; + } +} + +// Single response containing either an id, a COUNT query result, a SELECT-table +// or complete entities. +message TransactionResponse { + // We wrap these together in order to make this a repeatable field in the + // MultiTransactionResponse. + oneof transaction_response { + // Single response to a delete transaction. + DeleteResponse delete_response = 1; + // Single response to an update transaction. + UpdateResponse update_response = 2; + // Single response to a retrieve transaction. + RetrieveResponse retrieve_response = 3; + // Single response to an insert transaction. + InsertResponse insert_response = 4; + } +} + +// Wraps multiple sub-transaction requests of mixed types. +message MultiTransactionRequest { + // The actual payload. + repeated TransactionRequest requests = 1; +} + +// Wraps the reponses to multiple sub-transactions of mixed types. +message MultiTransactionResponse { + // The actual payload. + repeated TransactionResponse responses = 1; + // Transaction errors. + repeated Message transaction_errors = 2; + // Transaction warnings. + repeated Message transaction_warnings = 3; + // Transaction info messages. + repeated Message transaction_infos = 4; +} + +// Request of the MultiRetrieveEntityACL rpc +message MultiRetrieveEntityACLRequest { + // The requested entity ids. + repeated string id = 1; +} + +// Response of the MultiRetrieveEntityACL rpc +message MultiRetrieveEntityACLResponse { + // The entities ACLs + repeated EntityACL acls = 1; +} + +// Request of the MultiUpdateEntityACL rpc +message MultiUpdateEntityACLRequest { + // The requested entity ids. + repeated EntityACL acls = 1; +} + +// Response of the MultiUpdateEntityACL rpc +message MultiUpdateEntityACLResponse {} + +// Entity Access Controll List (ACL) +message EntityACL { + // The entity id + string id = 1; + // The rules which make up the ACL + repeated EntityPermissionRule rules = 2; + // relevant permissions of the current session, e.g. if the current user is + // allowed to update the ACL. This is read-only and will be ignored by the + // server. + EntityAclPermission permission = 3; +} + +// Entity ACL Permission indicates whether the current user is allowed to update +// the Entity's ACL. +enum EntityAclPermission { + // UNSPECIFIED, effectively means "NO" + ENTITY_ACL_PERMISSION_UNSPECIFIED = 0; + // Non-priority rules may be edited. + ENTITY_ACL_PERMISSION_EDIT_ACL = 1; + // Priority and non-priority rules may be edited. + ENTITY_ACL_PERMISSION_EDIT_PRIORITY_ACL = 2; +} + +// Permission rules for Entity ACL +message EntityPermissionRule { + // The role for which this rule applies + string role = 1; + // Is priority? + bool priority = 2; + // Is grant? + bool grant = 3; + // permissions + repeated EntityPermission permissions = 4; + // capabilities + repeated EntityPermissionRuleCapability capabilities = 5; +} + +// What (given enough permissions) can be done with an EntityPermissionRule. +// E.g. globel entity permission rules, which are to be specified in a +// configuration file cannot be deleted, thus they are missing the +// ENTITY_PERMISSION_RULE_CAPABILITY_DELETE capability. +enum EntityPermissionRuleCapability { + // Unspecified capability. + ENTITY_PERMISSION_RULE_CAPABILITY_UNSPECIFIED = 0; + // This permission rule can be deleted/removed from the ACL + ENTITY_PERMISSION_RULE_CAPABILITY_DELETE = 1; +} + +// Entity permissions +enum EntityPermission { + // Unspecified permission. + ENTITY_PERMISSION_UNSPECIFIED = 0; + // Permission to retrieve the full entity (name, description, data type, ...) + // with all parents and properties (unless prohibited by another rule on the + // property level). + ENTITY_PERMISSION_RETRIEVE_ENTITY = 1; + // Permission to retrieve the full and final ACL of this entity. + ENTITY_PERMISSION_RETRIEVE_ACL = 2; + // Permission to retrieve the history of this entity. + ENTITY_PERMISSION_RETRIEVE_HISTORY = 3; + // Permission to retrieve the owner(s) of this entity. + ENTITY_PERMISSION_RETRIEVE_OWNER = 4; + // Permission to download the file belonging to this entity. + ENTITY_PERMISSION_RETRIEVE_FILE = 5; + // Permission to delete an entity. + ENTITY_PERMISSION_DELETE = 6; + // Permission to change the user-specified part of this entity's ACL. Roles + // with this Permission are called 'Owners'. + ENTITY_PERMISSION_EDIT_ACL = 7; + // Permission to change the value of this entity. + ENTITY_PERMISSION_UPDATE_DESCRIPTION = 8; + // Permission to change the value of this entity. + ENTITY_PERMISSION_UPDATE_VALUE = 9; + // Permission to change the role of this entity. + ENTITY_PERMISSION_UPDATE_ROLE = 10; + // Permission to remove parents from this entity. + ENTITY_PERMISSION_UPDATE_REMOVE_PARENT = 11; + // Permission to add a parent to this entity. + ENTITY_PERMISSION_UPDATE_ADD_PARENT = 12; + // Permission to remove properties from this entity. + ENTITY_PERMISSION_UPDATE_REMOVE_PROPERTY = 13; + // Permission to add a property to this entity. + ENTITY_PERMISSION_UPDATE_ADD_PROPERTY = 14; + // Permission to change the name of this entity. + ENTITY_PERMISSION_UPDATE_NAME = 15; + // Permission to change the data type of this entity. + ENTITY_PERMISSION_UPDATE_DATA_TYPE = 16; + // Permission to delete the file of this entity. + ENTITY_PERMISSION_UPDATE_REMOVE_FILE = 17; + // Permission to set a file for this entity. + ENTITY_PERMISSION_UPDATE_ADD_FILE = 18; + // Permission to move an existing file to a new location. + ENTITY_PERMISSION_UPDATE_MOVE_FILE = 19; + // Permission to refer to this entity via a reference property. + ENTITY_PERMISSION_USE_AS_REFERENCE = 20; + // Permission to implement this entity as a property. + ENTITY_PERMISSION_USE_AS_PROPERTY = 21; + // Permission to use this entity as a super type for other entities. + ENTITY_PERMISSION_USE_AS_PARENT = 22; + // Permission to use this entity as a data type for reference properties. + ENTITY_PERMISSION_USE_AS_DATA_TYPE = 23; + // Permission to update the query template definition of this QueryTemplate + ENTITY_PERMISSION_UPDATE_QUERY_TEMPLATE_DEFINITION = 24; +} + +// Entity Transaction Service +service EntityTransactionService { + // Issue multiple mixed requests + rpc MultiTransaction(MultiTransactionRequest) returns (MultiTransactionResponse); + // Request the Entity ACLs + rpc MultiRetrieveEntityACL(MultiRetrieveEntityACLRequest) returns (MultiRetrieveEntityACLResponse); + // Set the Entity ACLs, + rpc MultiUpdateEntityACL(MultiUpdateEntityACLRequest) returns (MultiUpdateEntityACLResponse); +} diff --git a/proto/caosdb/file_transmission/v1alpha1/file_transmission.proto b/proto/caosdb/file_transmission/v1alpha1/file_transmission.proto new file mode 100644 index 0000000000000000000000000000000000000000..7dd80cc6b8ff3b8e9c690c50efbbe89b0e9f2c53 --- /dev/null +++ b/proto/caosdb/file_transmission/v1alpha1/file_transmission.proto @@ -0,0 +1,201 @@ +// +// This file is a part of the LinkAhead Project. +// +// Copyright (C) 2021-2022 Timm Fitschen <t.fitschen@indiscale.com> +// Copyright (C) 2025 Joscha Schmiedt <joscha@schmiedt.dev> +// Copyright (C) 2021-2025 IndiScale GmbH <info@indiscale.com> +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see <https://www.gnu.org/licenses/>. +// + +// This is the main file of the caosdb.entity.v1 package. +syntax = "proto3"; +package caosdb.file_transmission.v1alpha1; + +option cc_enable_arenas = true; +option java_multiple_files = true; +option java_package = "org.caosdb.api.file_transmission.v1"; + +/////////////////////////////////////////////////////////////////////////// +// File transmission +/////////////////////////////////////////////////////////////////////////// + +// Stores a single chunk of a file +message FileChunk { + // Temporary identifier containing the file and registration_id. + FileTransmissionId file_transmission_id = 1; + // Binary data of a chunk. + bytes data = 2; +} + +// Temporary identifier of a single file during a transmission. +message FileTransmissionId { + // The registration id which has been issued by the target of the trans + string registration_id = 1; + // A temporary identifier which identifies the file of this chunk. The + // file_id is also used by transaction to associate entities (which are to be + // inserted or updated) with a binary blob. + string file_id = 2; +} + +// Settings for the the file transmission. +message FileTransmissionSettings { + // The maximum chunk size. + int64 max_chunk_size = 1; + // The maximum file size. + int64 max_file_size = 2; +} + +// Indicates whether a registration (for upload or download) has been accepted +// or rejected. +enum RegistrationStatus { + // The registration status is unspecified. + REGISTRATION_STATUS_UNSPECIFIED = 0; + // The registration has been accepted and the client may proceed with the + // actual transmissions. + REGISTRATION_STATUS_ACCEPTED = 1; + // The registration has been rejected and the client should not try to + // proceed with the transmission. + REGISTRATION_STATUS_REJECTED = 2; +} + +// Indicates the state of an upload or a download (a stream of chunks). +enum TransmissionStatus { + // The transmission status is unspecified. + TRANSMISSION_STATUS_UNSPECIFIED = 0; + // The transmission has been successful. + TRANSMISSION_STATUS_SUCCESS = 1; + // The transmission terminated with errors. + TRANSMISSION_STATUS_ERROR = 2; + // The transmission is incomplete and the client may send/request the next + // chunk. + TRANSMISSION_STATUS_GO_ON = 3; +} + +// TODO: This is very similar to the Hash message in the entity API. We should discuss +// if and how they can be merged. +enum ChecksumAlgorithm { + // The checksum algorithm is unspecified. + CHECKSUM_ALGORITHM_UNSPECIFIED = 0; + // The MD5 checksum algorithm. + CHECKSUM_ALGORITHM_MD5 = 1; + // The SHA-1 checksum algorithm. + CHECKSUM_ALGORITHM_SHA1 = 2; + // The SHA-256 checksum algorithm. + CHECKSUM_ALGORITHM_SHA256 = 3; + // The SHA-512 checksum algorithm. + CHECKSUM_ALGORITHM_SHA512 = 4; +} + +// Checksum contains the checksum value and the algorithm used to compute it. +message Checksum { + // The checksum value. + string checksum = 1; + // The algorithm used to compute the checksum. + ChecksumAlgorithm checksum_algorithm = 2; +} + +// Information about a file which is to be uploaded. +// TODO: This is very similar to the FileDescriptor message in the entity API with the +// difference that it has no relation to an entity and is purely filesystem-oriented. +// Maybe we should re-use FileInformation s part of the entity API? +message FileInformation { + // The local filename is the filename, which is used on the client. + string local_filename = 1; + // The target filename is the filename which will be used on the server. + string target_filename = 2; + // Size of the file in bytes. + int64 size_bytes = 3; + // Checksum of the file according to the checksum algorithm. + Checksum checksum = 4; +} + +// Register a file upload for a list of files. The server will respond with a +// registration id and URLs to which the files can be uploaded. +message RegisterFileUploadRequest { + // Information about the files to be uploaded. + repeated FileInformation file_information = 1; +} + +// The URL to which a file should be uploaded using the given LinkAhead +// UploadProtocolVersion. +message UploadLocation { + // The URL to which the file should be uploaded. + string url = 1; +} + +// The version of the LinkAhead upload protocol. This is currently identical to +// the TUS protocol version. +message UploadProtocolVersion { + // The major version number of the upload protocol. + int32 major = 1; + // The minor version number of the upload protocol. + int32 minor = 2; + // The patch version number of the upload protocol. + int32 patch = 3; +} + +// Response of the file server upon an upload registration request. +message RegisterFileUploadResponse { + // Whether the server accepted or rejected the registration. + RegistrationStatus status = 1; + // The registration id is used to identify chunks and files which belong to + // the same upload . + string registration_id = 2; + // The server's transmission settings for the upload. + FileTransmissionSettings upload_settings = 4; + // The version of the LinkAhead upload protocol. + UploadProtocolVersion version = 5; + // The locations where the files should be uploaded. + repeated UploadLocation locations = 6; +} + +// Request for a file upload which has been registered previously. Chunks may be +// send in any order. +message FileUploadRequest { + // A single file chunk + FileChunk chunk = 1; +} + +// Response of the server upon a finished FileUpload. +message FileUploadResponse { + // Status of the upload. + TransmissionStatus status = 1; +} + +// Request for a file download which has been registered previously. +message FileDownloadRequest { + // Request the next chunk for this file. + FileTransmissionId file_transmission_id = 1; +} + +// Response containing a chunk of a file. +message FileDownloadResponse { + // Status of the download + TransmissionStatus status = 1; + // A single file chunk + FileChunk chunk = 2; +} + +// File Transaction Service +service FileTransmissionService { + // Register a file upload. This needs to be done prior to the actual upload + // and prior to the transaction request which uses the uploaded files. + rpc RegisterFileUpload(RegisterFileUploadRequest) returns (RegisterFileUploadResponse); + // The actual file upload. The upload has to be registered prior to this rpc. + rpc FileUpload(stream FileUploadRequest) returns (FileUploadResponse); + // A file download. The download has to be registered prior this rpc in the + // RetrieveRequest. + rpc FileDownload(FileDownloadRequest) returns (stream FileDownloadResponse); +}