Merge branch 'dev' into f-cache-version

.docker-base/Dockerfile

 # Use docker as parent image
-FROM docker:19.03.0
+FROM docker:20.10
 
 # http://bugs.python.org/issue19846
 ENV LANG C.UTF-8

.gitlab-ci.yml

@@ -58,8 +58,8 @@ test:
       - cd .docker 
       - /bin/sh ./run.sh
       - cd .. 
-      - docker logs docker_caosdb-server_1 &> caosdb_log.txt
-      - docker logs docker_sqldb_1 &> mariadb_log.txt
+      - docker logs docker-caosdb-server-1 &> caosdb_log.txt
+      - docker logs docker-sqldb-1 &> mariadb_log.txt
       - docker-compose -f .docker/docker-compose.yml down
       - rc=`cat .docker/result`  
       - exit $rc

CHANGELOG.md

@@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased] ##
 
 ### Added ###
+- You can now use `python -m caosadvancedtools.models.parser model_file` to 
+  parse and potentially synchronize data models.
 
 ### Changed ###
 

extra/emacs/readme.md

+# Emacs extras #
+
+This directory contains extra utils for use with Emacs.
+
+## Snippets ##
+
+if you copy the contents of the `snippets` directory to your `~/.emacs.d/snippets/`, the following
+*yasnippet* snippets will become available:
+
+- yaml-mode:
+  - `RT`: Insert a new RecordType, with inheritance and properties sections.
+  - `prop`: Insert a new Property into a RecordType, with datatype and description.

extra/emacs/snippets/yaml-mode/Property inside RecordType

+# -*- mode: snippet -*-
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2022 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2022 Daniel Hornung <d.hornung@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+# name: Property inside RecordType
+# key: prop
+# expand-env: ((yas-indent-line 'fixed))
+# --
+${1:property_name}:
+  datatype: ${2:$$(yas-choose-value '("BOOLEAN"
+                                      "DATETIME"
+                                      "DOUBLE"
+                                      "FILE"
+                                      "INTEGER"
+                                      "LIST"
+                                      "REFERENCE"
+                                      "TEXT"))}
+  description: ${3:description text}
+$0
\ No newline at end of file

extra/emacs/snippets/yaml-mode/RecordType

+# -*- mode: snippet -*-
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2022 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2022 Daniel Hornung <d.hornung@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+# name: RecordType
+# key: RT
+# expand-env: ((yas-indent-line 'fixed))
+# --
+${1:RecordTypeName}:
+  inherit_from_obligatory:$0
+  inherit_from_recommended:
+  inherit_from_suggested:
+  obligatory_properties:
+  recommended_properties:
+  suggested_properties:

src/caosadvancedtools/cfoods/h5.py

@@ -97,7 +97,7 @@ def h5_attr_to_property(val):
 
         # TODO this can eventually be removed
 
-        if(hasattr(val, 'ndim')):
+        if hasattr(val, 'ndim'):
             if not isinstance(val, np.ndarray) and val.ndim != 0:
                 print(val, val.ndim)
                 raise Exception(

src/caosadvancedtools/crawler.py

@@ -429,9 +429,9 @@ class Crawler(object):
             # only done in SSS mode
 
             if "SHARED_DIR" in os.environ:
-                filename = self.save_form([el[3]
-                                           for el in pending_changes], path)
-                self.send_mail([el[3] for el in pending_changes], filename)
+                filename = Crawler.save_form([el[3]
+                                              for el in pending_changes], path, self.run_id)
+                Crawler.send_mail([el[3] for el in pending_changes], filename)
 
             for i, el in enumerate(pending_changes):
 
@@ -442,8 +442,7 @@ ____________________\n""".format(i+1, len(pending_changes)) + str(el[3]))
             logger.info("There where unauthorized changes (see above). An "
                         "email was sent to the curator.\n"
                         "You can authorize the updates by invoking the crawler"
-                        " with the run id: {rid}\n".format(rid=self.run_id,
-                                                           path=path))
+                        " with the run id: {rid}\n".format(rid=self.run_id))
 
         if len(DataModelProblems.missing) > 0:
             err_msg = ("There were problems with one or more RecordType or "
@@ -466,7 +465,8 @@ ____________________\n""".format(i+1, len(pending_changes)) + str(el[3]))
         else:
             logger.info("Crawler terminated successfully!")
 
-    def save_form(self, changes, path):
+    @staticmethod
+    def save_form(changes, path, run_id):
         """
         Saves an html website to a file that contains a form with a button to
         authorize the given changes.
@@ -548,13 +548,13 @@ ____________________\n""".format(i+1, len(pending_changes)) + str(el[3]))
 </body>
 </html>
 """.format(url=db.configuration.get_config()["Connection"]["url"],
-           rid=self.run_id,
+           rid=run_id,
            changes=escape("\n".join(changes)),
            path=path)
 
         if "SHARED_DIR" in os.environ:
             directory = os.environ["SHARED_DIR"]
-        filename = str(self.run_id)+".html"
+        filename = str(run_id)+".html"
         randname = os.path.basename(os.path.abspath(directory))
         filepath = os.path.abspath(os.path.join(directory, filename))
         filename = os.path.join(randname, filename)
@@ -562,7 +562,8 @@ ____________________\n""".format(i+1, len(pending_changes)) + str(el[3]))
             f.write(form)
         return filename
 
-    def send_mail(self, changes, filename):
+    @staticmethod
+    def send_mail(changes, filename):
         """ calls sendmail in order to send a mail to the curator about pending
         changes
 

src/caosadvancedtools/models/parser.py

@@ -36,6 +36,7 @@ Parents can be provided under the 'inherit_from_xxxx' keywords. The value needs
 to be a list with the names. Here, NO NEW entities can be defined.
 """
 import json
+import argparse
 import re
 import sys
 import yaml
@@ -796,5 +797,25 @@ class JsonSchemaParser(Parser):
 
 
 if __name__ == "__main__":
-    model = parse_model_from_yaml('data_model.yml')
+    parser = argparse.ArgumentParser(description=__doc__,
+                                     formatter_class=argparse.RawTextHelpFormatter)
+    parser.add_argument("data_model",
+                        help="Path name of the data model file (yaml or json) to be used.")
+    parser.add_argument("--sync", action="store_true",
+                        help="Whether or not to sync the data model with the server.")
+    parser.add_argument("--noquestion", action="store_true",
+                        help="Whether or not to ask questions during synchronization.")
+    parser.add_argument("--print", action="store_true",
+                        help="Whether or not to print the data model.")
+
+    args = parser.parse_args()
+    if args.data_model.endswith(".json"):
+        model = parse_model_from_json_schema(args.data_model)
+    elif args.data_model.endswith(".yml") or args.data_model.endswith(".yaml"):
+        model = parse_model_from_yaml(args.data_model)
+    else:
+        RuntimeError("did not recognize file ending")
+    if args.print:
         print(model)
+    if args.sync:
+        model.sync_data_model(noquestion=args.noquestion)

src/doc/index.rst

@@ -13,9 +13,8 @@ This documentation helps you to :doc:`get started<getting_started>`, explains th
 
    Getting started <README_SETUP>
    Concepts <concepts>
-   tutorials
-   Caosdb-Crawler <crawler>
-   YAML Interface <yaml_interface>
+   The Caosdb Crawler <crawler>
+   YAML data model specification <yaml_interface>
    _apidoc/modules
 
 

src/doc/yaml_interface.rst

-YAML-Interface
---------------
 
-The yaml interface is a module in caosdb-pylib that can be used to create and update
+===============================
+ YAML data model specification
+===============================
+
+The ``caosadvancedtools`` library features the possibility to create and update
 CaosDB models using a simplified definition in YAML format.
 
-Let's start with an example taken from https://gitlab.indiscale.com/caosdb/src/caosdb-advanced-user-tools/-/blob/dev/unittests/model.yml.
+Let's start with an example taken from `model.yml
+<https://gitlab.indiscale.com/caosdb/src/caosdb-advanced-user-tools/-/blob/dev/unittests/model.yml>`__
+in the library sources.
 
 .. code-block:: yaml
 
@@ -44,7 +48,7 @@ Let's start with an example taken from https://gitlab.indiscale.com/caosdb/src/c
 
 
 
-This example defines 3 ``RecordType``s:
+This example defines 3 ``RecordTypes``:
 
 - A ``Project`` with one obligatory property ``datatype``
 - A Person with a ``firstName`` and a ``lastName`` (as recommended properties)
@@ -69,8 +73,10 @@ Note the difference between the three property declarations of ``LabbookEntry``:
 If the data model depends on record types or properties which already exist in CaosDB, those can be
 added using the ``extern`` keyword: ``extern`` takes a list of previously defined names.
 
+
+
 Datatypes
----------
+=========
 
 You can use any data type understood by CaosDB as datatype attribute in the yaml model.
 
@@ -90,9 +96,8 @@ would declare a list of elements with datatype Project.
 
 
 Keywords
---------
+========
 
-- **parent**: Parent of this entity.
 - **importance**: Importance of this entity. Possible values: "recommended", "obligatory", "suggested"
 - **datatype**: The datatype of this property, e.g. TEXT, INTEGER or Project.
 - **unit**: The unit of the property, e.g. "m/s".
@@ -100,12 +105,14 @@ Keywords
 - **recommended_properties**: Add properties to this entity with importance "recommended".
 - **obligatory_properties**: Add properties to this entity with importance "obligatory".
 - **suggested_properties**: Add properties to this entity with importance "suggested".
-- **inherit_from_recommended**: Inherit from another entity using the specified importance level including the higher importance level "obligatory". This would add a corresponding parent and add all obligatory and recommended properties from the parent.
-- **inherit_from_suggested including higher importance levels**: Inherit from another entity using the specified importance level. This would add a corresponding parent and add all obligatory, recommended and suggested properties from the parent.
-- **inherit_from_obligatory**: Inherit from another entity using the specified importance level. This would add a corresponding parent and add all obligatory properties from the parent.
+- **inherit_from_XXX**: This keyword accepts a list of other RecordTypes.  Those RecordTypes are
+  added as parents, and all Properties with at least the importance ``XXX`` are inherited.  For
+  example, ``inherited_from_recommended`` will inherit all Properties of importance ``recommended``
+  and ``obligatory``, but not ``suggested``.
+- **parent**: Parent of this entity. (*Deprecated*)
 
 Usage
------
+=====
 
 You can use the yaml parser directly in python as follows:
 
@@ -124,3 +131,6 @@ the model with a CaosDB instance, e.g.:
 .. code-block:: python
    
   model.sync_data_model()
+
+..  LocalWords:  yml projectId UID firstName lastName LabbookEntry entryId textElement labbook
+..  LocalWords:  associatedFile extern Textfile DataModel

unittests/test_base_table_exporter.py

@@ -82,13 +82,13 @@ def test_simple_record():
     assert my_exporter.prepare_csv_export(
         delimiter='\t', print_header=True) == "Test_Prop_1\tTest_Prop_2\nbla\tblabla"
     # remove optional entry from info
-    del(my_exporter.info["Test_Prop_2"])
+    del my_exporter.info["Test_Prop_2"]
     assert my_exporter.prepare_csv_export(skip_empty_optionals=True) == "bla"
     assert my_exporter.prepare_csv_export(
         delimiter='\t', print_header=True) == "Test_Prop_1\tTest_Prop_2\nbla\t"
     # reload info, and delete mandatory entry
     my_exporter.collect_information()
-    del(my_exporter.info["Test_Prop_1"])
+    del my_exporter.info["Test_Prop_1"]
     with raises(te.TableExportError) as exc:
         my_exporter.prepare_csv_export()
     assert "Test_Prop_1" in exc.value.msg
@@ -184,7 +184,7 @@ def test_info_collection():
     assert "optional_value" not in my_exporter.info
 
     # now error in mandatory value
-    del(export_dict["optional_value"])
+    del export_dict["optional_value"]
     export_dict["mandatory_value"] = {
         "find_func": "find_function_with_error"
     }