diff --git a/.docker-base/Dockerfile b/.docker-base/Dockerfile index 923924e75e03c6ca8346b17cdf87eda78efd766f..02c7d5c3045f527c1f17c678633c0f42ee8ce3a5 100644 --- a/.docker-base/Dockerfile +++ b/.docker-base/Dockerfile @@ -1,12 +1,12 @@ # Use docker as parent image -FROM docker:19.03.0 +FROM docker:20.10 # http://bugs.python.org/issue19846 ENV LANG C.UTF-8 # install dependencies RUN apk add --no-cache py3-pip python3 python3-dev gcc make \ - git bash curl gettext py3-requests + git bash curl gettext py3-requests RUN apk add --no-cache libffi-dev openssl-dev libc-dev libxslt libxslt-dev \ libxml2 libxml2-dev diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8ebbefaa39650ddaff45b856a8a4d44a2ac495d1..73a9abb0b5a525ddb74ddbf33003b03e35c1cacf 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -58,8 +58,8 @@ test: - cd .docker - /bin/sh ./run.sh - cd .. - - docker logs docker_caosdb-server_1 &> caosdb_log.txt - - docker logs docker_sqldb_1 &> mariadb_log.txt + - docker logs docker-caosdb-server-1 &> caosdb_log.txt + - docker logs docker-sqldb-1 &> mariadb_log.txt - docker-compose -f .docker/docker-compose.yml down - rc=`cat .docker/result` - exit $rc diff --git a/CHANGELOG.md b/CHANGELOG.md index be44a47d1a0c79c8a4fa39f382d4d3a0e22439f6..99e9a1c0d06946c679f8fe1b32f573c3876b867c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,19 +4,19 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] ## +## [0.5.0] - 2022-09-05 ## +(Florian Spreckelsen) ### Added ### -### Changed ### +- You can now use `python -m caosadvancedtools.models.parser model_file` to + parse and potentially synchronize data models. ### Deprecated ### -### Removed ### - -### Fixed ### - -### Security ### +- [#36](https://gitlab.com/caosdb/caosdb-advanced-user-tools/-/issues/36) + `parent` keyword in yaml datamodel definition (replaced by + `inherit_from_{obligatory|recommended|suggested}` keywords). ## [0.4.1] - 2022-05-03 ## (Henrik tom Wörden) diff --git a/extra/emacs/readme.md b/extra/emacs/readme.md new file mode 100644 index 0000000000000000000000000000000000000000..2636eab68b9521acf75c190e3fcf34e6e86fc60b --- /dev/null +++ b/extra/emacs/readme.md @@ -0,0 +1,12 @@ +# Emacs extras # + +This directory contains extra utils for use with Emacs. + +## Snippets ## + +if you copy the contents of the `snippets` directory to your `~/.emacs.d/snippets/`, the following +*yasnippet* snippets will become available: + +- yaml-mode: + - `RT`: Insert a new RecordType, with inheritance and properties sections. + - `prop`: Insert a new Property into a RecordType, with datatype and description. diff --git a/extra/emacs/snippets/yaml-mode/Property inside RecordType b/extra/emacs/snippets/yaml-mode/Property inside RecordType new file mode 100644 index 0000000000000000000000000000000000000000..92769b78e5496ec4cb545556b3eff3fc924c872d --- /dev/null +++ b/extra/emacs/snippets/yaml-mode/Property inside RecordType @@ -0,0 +1,34 @@ +# -*- mode: snippet -*- +# This file is a part of the CaosDB Project. +# +# Copyright (C) 2022 IndiScale GmbH <info@indiscale.com> +# Copyright (C) 2022 Daniel Hornung <d.hornung@indiscale.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. + +# name: Property inside RecordType +# key: prop +# expand-env: ((yas-indent-line 'fixed)) +# -- +${1:property_name}: + datatype: ${2:$$(yas-choose-value '("BOOLEAN" + "DATETIME" + "DOUBLE" + "FILE" + "INTEGER" + "LIST" + "REFERENCE" + "TEXT"))} + description: ${3:description text} +$0 \ No newline at end of file diff --git a/extra/emacs/snippets/yaml-mode/RecordType b/extra/emacs/snippets/yaml-mode/RecordType new file mode 100644 index 0000000000000000000000000000000000000000..6b4a9c263806b6d57442470a11a2770d3d417741 --- /dev/null +++ b/extra/emacs/snippets/yaml-mode/RecordType @@ -0,0 +1,30 @@ +# -*- mode: snippet -*- +# This file is a part of the CaosDB Project. +# +# Copyright (C) 2022 IndiScale GmbH <info@indiscale.com> +# Copyright (C) 2022 Daniel Hornung <d.hornung@indiscale.com> +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see <https://www.gnu.org/licenses/>. + +# name: RecordType +# key: RT +# expand-env: ((yas-indent-line 'fixed)) +# -- +${1:RecordTypeName}: + inherit_from_obligatory:$0 + inherit_from_recommended: + inherit_from_suggested: + obligatory_properties: + recommended_properties: + suggested_properties: diff --git a/release.sh b/release.sh new file mode 100755 index 0000000000000000000000000000000000000000..1af097f014de6cd9eb3d3e8ba5da34aea0fe1671 --- /dev/null +++ b/release.sh @@ -0,0 +1,4 @@ +#!/bin/bash +rm -rf dist/ build/ .eggs/ +python setup.py sdist bdist_wheel +python -m twine upload -s dist/* diff --git a/setup.py b/setup.py index 9988a3afc6f2afda29942af4569c5f32b74f40e6..e6ebb6dd7ebe6a25659a4f7584e3a3ae649d4e29 100755 --- a/setup.py +++ b/setup.py @@ -46,7 +46,7 @@ from setuptools import find_packages, setup ######################################################################## MAJOR = 0 -MINOR = 4 +MINOR = 5 MICRO = 1 PRE = "" # e.g. rc0, alpha.1, 0.beta-23 ISRELEASED = True diff --git a/src/caosadvancedtools/cfoods/h5.py b/src/caosadvancedtools/cfoods/h5.py index cbf9d0baefa435b71eeaeefe63a9b018faabe7ea..4e6832f2e96e0950ed99146d4907f1ffb70d8494 100644 --- a/src/caosadvancedtools/cfoods/h5.py +++ b/src/caosadvancedtools/cfoods/h5.py @@ -97,7 +97,7 @@ def h5_attr_to_property(val): # TODO this can eventually be removed - if(hasattr(val, 'ndim')): + if hasattr(val, 'ndim'): if not isinstance(val, np.ndarray) and val.ndim != 0: print(val, val.ndim) raise Exception( diff --git a/src/caosadvancedtools/crawler.py b/src/caosadvancedtools/crawler.py index 0159688c7c7d59e779d576aed54b176e802fca85..099b8fd86656bd326c91e7754fa32a3d4ba76564 100644 --- a/src/caosadvancedtools/crawler.py +++ b/src/caosadvancedtools/crawler.py @@ -428,9 +428,9 @@ class Crawler(object): # only done in SSS mode if "SHARED_DIR" in os.environ: - filename = self.save_form([el[3] - for el in pending_changes], path) - self.send_mail([el[3] for el in pending_changes], filename) + filename = Crawler.save_form([el[3] + for el in pending_changes], path, self.run_id) + Crawler.send_mail([el[3] for el in pending_changes], filename) for i, el in enumerate(pending_changes): @@ -441,8 +441,7 @@ ____________________\n""".format(i+1, len(pending_changes)) + str(el[3])) logger.info("There where unauthorized changes (see above). An " "email was sent to the curator.\n" "You can authorize the updates by invoking the crawler" - " with the run id: {rid}\n".format(rid=self.run_id, - path=path)) + " with the run id: {rid}\n".format(rid=self.run_id)) if len(DataModelProblems.missing) > 0: err_msg = ("There were problems with one or more RecordType or " @@ -465,7 +464,8 @@ ____________________\n""".format(i+1, len(pending_changes)) + str(el[3])) else: logger.info("Crawler terminated successfully!") - def save_form(self, changes, path): + @staticmethod + def save_form(changes, path, run_id): """ Saves an html website to a file that contains a form with a button to authorize the given changes. @@ -547,13 +547,13 @@ ____________________\n""".format(i+1, len(pending_changes)) + str(el[3])) </body> </html> """.format(url=db.configuration.get_config()["Connection"]["url"], - rid=self.run_id, + rid=run_id, changes=escape("\n".join(changes)), path=path) if "SHARED_DIR" in os.environ: directory = os.environ["SHARED_DIR"] - filename = str(self.run_id)+".html" + filename = str(run_id)+".html" randname = os.path.basename(os.path.abspath(directory)) filepath = os.path.abspath(os.path.join(directory, filename)) filename = os.path.join(randname, filename) @@ -561,7 +561,8 @@ ____________________\n""".format(i+1, len(pending_changes)) + str(el[3])) f.write(form) return filename - def send_mail(self, changes, filename): + @staticmethod + def send_mail(changes, filename): """ calls sendmail in order to send a mail to the curator about pending changes diff --git a/src/caosadvancedtools/models/parser.py b/src/caosadvancedtools/models/parser.py index ad149222b5b90671a50943dc00bc9de8074a42f1..c9b890de570d29e4a013b14ebe4579e956277ed2 100644 --- a/src/caosadvancedtools/models/parser.py +++ b/src/caosadvancedtools/models/parser.py @@ -36,11 +36,13 @@ Parents can be provided under the 'inherit_from_xxxx' keywords. The value needs to be a list with the names. Here, NO NEW entities can be defined. """ import json +import argparse import re import sys import yaml from typing import List +from warnings import warn import jsonschema import caosdb as db @@ -518,6 +520,16 @@ class Parser(object): self._inherit(name, prop, db.RECOMMENDED) elif prop_name == "inherit_from_suggested": self._inherit(name, prop, db.SUGGESTED) + elif prop_name == "parent": + warn( + DeprecationWarning( + "The `parent` keyword is deprecated and will be " + "removed in a future version. Use " + "`inherit_from_{obligatory|recommended|suggested}` " + "instead." + ) + ) + self._inherit(name, prop, db.OBLIGATORY) else: raise ValueError("invalid keyword: {}".format(prop_name)) @@ -796,5 +808,25 @@ class JsonSchemaParser(Parser): if __name__ == "__main__": - model = parse_model_from_yaml('data_model.yml') - print(model) + parser = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument("data_model", + help="Path name of the data model file (yaml or json) to be used.") + parser.add_argument("--sync", action="store_true", + help="Whether or not to sync the data model with the server.") + parser.add_argument("--noquestion", action="store_true", + help="Whether or not to ask questions during synchronization.") + parser.add_argument("--print", action="store_true", + help="Whether or not to print the data model.") + + args = parser.parse_args() + if args.data_model.endswith(".json"): + model = parse_model_from_json_schema(args.data_model) + elif args.data_model.endswith(".yml") or args.data_model.endswith(".yaml"): + model = parse_model_from_yaml(args.data_model) + else: + RuntimeError("did not recognize file ending") + if args.print: + print(model) + if args.sync: + model.sync_data_model(noquestion=args.noquestion) diff --git a/src/doc/conf.py b/src/doc/conf.py index c7f82a99d3b287ca72ca57430b2d4b868539d39e..9eb8c08ae5a75cb0561d20d858350b7f6db3001b 100644 --- a/src/doc/conf.py +++ b/src/doc/conf.py @@ -27,9 +27,9 @@ copyright = '2021, IndiScale GmbH' author = 'Daniel Hornung' # The short X.Y version -version = '0.4.1' +version = '0.5.0' # The full version, including alpha/beta/rc tags -release = '0.4.1' +release = '0.5.0' # -- General configuration --------------------------------------------------- diff --git a/src/doc/index.rst b/src/doc/index.rst index 9aa045349ab05d3f5130a7f33b38c7eca0c4f32e..5fdb78da4eddfd0145d0357202246d4b5352dcf4 100644 --- a/src/doc/index.rst +++ b/src/doc/index.rst @@ -13,9 +13,8 @@ This documentation helps you to :doc:`get started<getting_started>`, explains th Getting started <README_SETUP> Concepts <concepts> - tutorials - Caosdb-Crawler <crawler> - YAML Interface <yaml_interface> + The Caosdb Crawler <crawler> + YAML data model specification <yaml_interface> _apidoc/modules diff --git a/src/doc/yaml_interface.rst b/src/doc/yaml_interface.rst index 476e92829238a0fc9dac851c61790c022e9fcde9..78ff4cdd6fee201c7ebe17977f497b84e9657aa2 100644 --- a/src/doc/yaml_interface.rst +++ b/src/doc/yaml_interface.rst @@ -1,10 +1,14 @@ -YAML-Interface --------------- -The yaml interface is a module in caosdb-pylib that can be used to create and update +=============================== + YAML data model specification +=============================== + +The ``caosadvancedtools`` library features the possibility to create and update CaosDB models using a simplified definition in YAML format. -Let's start with an example taken from https://gitlab.indiscale.com/caosdb/src/caosdb-advanced-user-tools/-/blob/dev/unittests/model.yml. +Let's start with an example taken from `model.yml +<https://gitlab.indiscale.com/caosdb/src/caosdb-advanced-user-tools/-/blob/dev/unittests/model.yml>`__ +in the library sources. .. code-block:: yaml @@ -44,7 +48,7 @@ Let's start with an example taken from https://gitlab.indiscale.com/caosdb/src/c -This example defines 3 ``RecordType``s: +This example defines 3 ``RecordTypes``: - A ``Project`` with one obligatory property ``datatype`` - A Person with a ``firstName`` and a ``lastName`` (as recommended properties) @@ -69,8 +73,10 @@ Note the difference between the three property declarations of ``LabbookEntry``: If the data model depends on record types or properties which already exist in CaosDB, those can be added using the ``extern`` keyword: ``extern`` takes a list of previously defined names. + + Datatypes ---------- +========= You can use any data type understood by CaosDB as datatype attribute in the yaml model. @@ -90,9 +96,8 @@ would declare a list of elements with datatype Project. Keywords --------- +======== -- **parent**: Parent of this entity. - **importance**: Importance of this entity. Possible values: "recommended", "obligatory", "suggested" - **datatype**: The datatype of this property, e.g. TEXT, INTEGER or Project. - **unit**: The unit of the property, e.g. "m/s". @@ -100,12 +105,14 @@ Keywords - **recommended_properties**: Add properties to this entity with importance "recommended". - **obligatory_properties**: Add properties to this entity with importance "obligatory". - **suggested_properties**: Add properties to this entity with importance "suggested". -- **inherit_from_recommended**: Inherit from another entity using the specified importance level including the higher importance level "obligatory". This would add a corresponding parent and add all obligatory and recommended properties from the parent. -- **inherit_from_suggested including higher importance levels**: Inherit from another entity using the specified importance level. This would add a corresponding parent and add all obligatory, recommended and suggested properties from the parent. -- **inherit_from_obligatory**: Inherit from another entity using the specified importance level. This would add a corresponding parent and add all obligatory properties from the parent. +- **inherit_from_XXX**: This keyword accepts a list of other RecordTypes. Those RecordTypes are + added as parents, and all Properties with at least the importance ``XXX`` are inherited. For + example, ``inherited_from_recommended`` will inherit all Properties of importance ``recommended`` + and ``obligatory``, but not ``suggested``. +- **parent**: Parent of this entity. Same as ``inherit_from_obligatory``. (*Deprecated*) Usage ------ +===== You can use the yaml parser directly in python as follows: @@ -124,3 +131,6 @@ the model with a CaosDB instance, e.g.: .. code-block:: python model.sync_data_model() + +.. LocalWords: yml projectId UID firstName lastName LabbookEntry entryId textElement labbook +.. LocalWords: associatedFile extern Textfile DataModel diff --git a/unittests/test_base_table_exporter.py b/unittests/test_base_table_exporter.py index 3b8276cdf947c5b22e829e050295dd47f3cfe9ea..8a65b71aa489f8fca457c0e700452a6dc5956eed 100644 --- a/unittests/test_base_table_exporter.py +++ b/unittests/test_base_table_exporter.py @@ -82,13 +82,13 @@ def test_simple_record(): assert my_exporter.prepare_csv_export( delimiter='\t', print_header=True) == "Test_Prop_1\tTest_Prop_2\nbla\tblabla" # remove optional entry from info - del(my_exporter.info["Test_Prop_2"]) + del my_exporter.info["Test_Prop_2"] assert my_exporter.prepare_csv_export(skip_empty_optionals=True) == "bla" assert my_exporter.prepare_csv_export( delimiter='\t', print_header=True) == "Test_Prop_1\tTest_Prop_2\nbla\t" # reload info, and delete mandatory entry my_exporter.collect_information() - del(my_exporter.info["Test_Prop_1"]) + del my_exporter.info["Test_Prop_1"] with raises(te.TableExportError) as exc: my_exporter.prepare_csv_export() assert "Test_Prop_1" in exc.value.msg @@ -184,7 +184,7 @@ def test_info_collection(): assert "optional_value" not in my_exporter.info # now error in mandatory value - del(export_dict["optional_value"]) + del export_dict["optional_value"] export_dict["mandatory_value"] = { "find_func": "find_function_with_error" } diff --git a/unittests/test_yaml_model_parser.py b/unittests/test_yaml_model_parser.py index a9f072b754618e38237cbf70e74c7944551f1045..6cdea7922a8503be082e8947edecd7e8c849730b 100644 --- a/unittests/test_yaml_model_parser.py +++ b/unittests/test_yaml_model_parser.py @@ -1,7 +1,7 @@ import unittest from datetime import date from tempfile import NamedTemporaryFile -from pytest import raises +from pytest import deprecated_call, raises import caosdb as db from caosadvancedtools.models.parser import (TwiceDefinedException, @@ -474,3 +474,40 @@ F: """ with raises(NotImplementedError): entities = parse_model_from_string(model) + + +def test_issue_36(): + """Test whether the `parent` keyword is deprecated. + + See https://gitlab.com/caosdb/caosdb-advanced-user-tools/-/issues/36. + + """ + model_string = """ +R1: + obligatory_properties: + prop1: + datatype: TEXT +R2: + obligatory_properties: + prop2: + datatype: TEXT + recommended_properties: + prop3: + datatype: TEXT +R3: + parent: + - R2 + inherit_from_obligatory: + - R1 +""" + with deprecated_call(): + # Check whether this is actually deprecated + model = parse_model_from_string(model_string) + + assert "R3" in model + r3 = model["R3"] + assert isinstance(r3, db.RecordType) + for par in ["R1", "R2"]: + # Until removal, both do the same + assert has_parent(r3, par) + assert r3.get_parent(par)._flags["inheritance"] == db.OBLIGATORY