Merge branch 'dev' into 'f-fix-89'

# Conflicts:
#   unittests/test_yaml_model_parser.py

.coveragerc

+[html]
+show_contexts = True

.docker-base/Dockerfile

 # Use docker as parent image
-FROM docker:19.03.0
+FROM docker:20.10
 
 # http://bugs.python.org/issue19846
 ENV LANG C.UTF-8

.docker/Dockerfile

@@ -25,7 +25,7 @@ ADD https://gitlab.com/api/v4/projects/13656973/repository/branches/dev \
 RUN git clone https://gitlab.com/caosdb/caosdb-pylib.git && \
    cd caosdb-pylib && git checkout dev && pip3 install .
 # At least recommonmark 0.6 required.
-RUN pip3 install -U html2text pycodestyle pylint recommonmark sphinx-rtd-theme
+RUN pip3 install -U html2text pycodestyle pylint recommonmark sphinx-rtd-theme gitignore-parser
 COPY . /git
 RUN rm -r /git/.git \
     && mv /git/.docker/pycaosdb.ini /git/integrationtests

.gitignore

@@ -17,3 +17,5 @@ build/
 
 # documentation
 _apidoc
+/dist/
+*~

.gitlab-ci.yml

@@ -58,8 +58,8 @@ test:
       - cd .docker
       - /bin/sh ./run.sh
       - cd ..
-      - docker logs docker_caosdb-server_1 &> caosdb_log.txt
-      - docker logs docker_sqldb_1 &> mariadb_log.txt
+      - docker logs docker-caosdb-server-1 &> caosdb_log.txt
+      - docker logs docker-sqldb-1 &> mariadb_log.txt
       - docker-compose -f .docker/docker-compose.yml down
       - rc=`cat .docker/result`
       - exit $rc
@@ -123,14 +123,45 @@ linting:
       - make lint
   allow_failure: true
 
-unittest:
+unittest_py39:
   tags: [docker]
   stage: unittest
   image: $CI_REGISTRY_IMAGE
   needs: [build-testenv]
   script:
+    # First verify that system Python actually is 3.9
+    - python3 -c "import sys; assert sys.version.startswith('3.9')"
+    - python3 -c "import linkahead; print('LinkAhead Version:', linkahead.__version__)"
     - tox
 
+unittest_py37:
+  tags: [docker]
+  stage: unittest
+  image: python:3.7
+  script: &python_test_script
+    - pip install nose pandas pytest pytest-cov gitignore-parser openpyxl>=3.0.7 xlrd==1.2 h5py
+    - pip install git+https://gitlab.indiscale.com/caosdb/src/caosdb-pylib.git@dev
+    - pip install .
+    - pytest --cov=caosadvancedtools unittests
+
+unittest_py38:
+  tags: [docker]
+  stage: unittest
+  image: python:3.8
+  script: *python_test_script
+
+unittest_py310:
+  tags: [docker]
+  stage: unittest
+  image: python:3.10
+  script: *python_test_script
+
+unittest_py311:
+  tags: [docker]
+  stage: unittest
+  image: python:3.11
+  script: *python_test_script
+
 # Build the sphinx documentation and make it ready for deployment by Gitlab Pages
 # Special job for serving a static website. See https://docs.gitlab.com/ee/ci/yaml/README.html#pages
 pages_prepare: &pages_prepare

.gitlab/issue_templates/Default.md

+## Summary
+
+*Please give a short summary of what the issue is.*
+
+## Expected Behavior
+
+*What did you expect how the software should behave?*
+
+## Actual Behavior
+
+*What did the software actually do?*
+
+## Steps to Reproduce the Problem
+
+*Please describe, step by step, how others can reproduce the problem.  Please try these steps for yourself on a clean system.*
+
+1.
+2.
+3.
+
+## Specifications
+
+- Version: *Which version of this software?*
+- Platform: *Which operating system, which other relevant software versions?*
+
+## Possible fixes
+
+*Do you have ideas how the issue can be resolved?*

.gitlab/merge_request_templates/Default.md

+# Summary
+
+*Insert a meaningful description for this merge request here:  What is the new/changed behavior?
+Which bug has been fixed? Are there related issues?*
+
+
+# Focus
+
+*Point the reviewer to the core of the code change. Where should they start reading? What should
+they focus on (e.g. security, performance, maintainability, user-friendliness, compliance with the
+specs, finding more corner cases, concrete questions)?*
+
+
+# Test Environment
+
+*How to set up a test environment for manual testing?*
+
+
+# Check List for the Author
+
+Please, prepare your MR for a review. Be sure to write a summary and a focus and create gitlab
+comments for the reviewer. They should guide the reviewer through the changes, explain your changes
+and also point out open questions. For further good practices have a look at [our review
+guidelines](https://gitlab.com/caosdb/caosdb/-/blob/dev/REVIEW_GUIDELINES.md)
+
+- [ ] All automated tests pass
+- [ ] Reference related issues
+- [ ] Up-to-date CHANGELOG.md (or not necessary)
+- [ ] Up-to-date JSON schema (or not necessary)
+- [ ] Appropriate user and developer documentation (or not necessary)
+  - Update / write published documentation (`make doc`).
+  - How do I use the software?  Assume "stupid" users.
+  - How do I develop or debug the software?  Assume novice developers.
+- [ ] Annotations in code (Gitlab comments)
+  - Intent of new code
+  - Problems with old code
+  - Why this implementation?
+
+
+# Check List for the Reviewer
+
+- [ ] I understand the intent of this MR
+- [ ] All automated tests pass
+- [ ] Up-to-date CHANGELOG.md (or not necessary)
+- [ ] Appropriate user and developer documentation (or not necessary), also in published
+      documentation.
+- [ ] The test environment setup works and the intended behavior is reproducible in the test
+  environment
+- [ ] In-code documentation and comments are up-to-date.
+- [ ] Check: Are there specifications? Are they satisfied?
+
+For further good practices have a look at [our review guidelines](https://gitlab.com/caosdb/caosdb/-/blob/dev/REVIEW_GUIDELINES.md).
+
+
+/assign me
+/target_branch dev

CHANGELOG.md

@@ -18,6 +18,94 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Security ###
 
+### Documentation ###
+
+## [0.9.0] - 2023-11-27 ##
+
+### Added ###
+
+* Added support for passing callables as `find_func` to the `BaseTableExporter`.
+* Added member `BaseTableExporter.all_keys`
+* Parsing from YAML now allows to give an existing model to which the YAML data model shall be
+  added.
+* The `json_schema_exporter` module which introduces tools to create a json
+  schema from a RecordType, e.g., for the usage in web forms.
+* `DataModel.get_deep(name: str)` method which uses the DataModel as a kind of cache pool.
+
+### Changed ###
+
+* A bit better error handling in the yaml model parser.
+* `TableImporter.check_datatypes` allows numeric values in string columns if
+  `strict=False` (default).
+
+### Fixed ###
+
+* `TableImporter.check_missing` in case of array-valued fields in table
+* YAML model parser has better description handling.
+
+### Documentation ###
+
+* Test coverage reports are now generated in `.tox/cov_html/` by tox.
+
+## [0.8.0] - 2023-05-30 ##
+(Florian Spreckelsen)
+
+### Added ###
+
+- TableImporter now accepts a `existing_columns` argument which demands that certain columns exist
+- The `JsonSchemaParser` class supports `patternProperties`
+- The `JsonSchemaParser` calss supports json-schema references (`$ref`)
+
+### Changed ###
+
+- The converters and datatype arguments of TableImporter now may have keys for nonexisting columns
+- The `JsonSchemaParser` class does not require the top-level entry of a json
+  schema definition to specify a RecordType.
+
+### Fixed ###
+
+- refactored to work with the new default key word in FIND queries: RECORD
+
+## [0.7.0] - 2023-03-09 ##
+(Florian Spreckelsen)
+
+### Added ###
+
+- `create_entity_link` function to create html links to entities; useful for
+  logging
+
+## [0.6.1] - 2023-01-20##
+
+### Added ###
+
+* Re-introduced support for Python 3.7
+
+## [0.6.0] - 2022-10-11 ##
+(Florian Spreckelsen)
+
+### Added ###
+
+- Unauthorized inserts can now be cached. Note that the Crawler cannot postpone
+  inserts but the Cache has the functionality now.
+- caosdbignore; You can add one or more `.caosdbignore` files to the directory
+  structure that you want to make available in CaosDB and the run loadFiles.
+  The syntax is that of `.gitignore` files. For more information see `loadFiles`
+  section of the Crawler in the documentation.
+
+## [0.5.0] - 2022-09-05 ##
+(Florian Spreckelsen)
+
+### Added ###
+
+- You can now use `python -m caosadvancedtools.models.parser model_file` to
+  parse and potentially synchronize data models.
+
+### Deprecated ###
+
+- [#36](https://gitlab.com/caosdb/caosdb-advanced-user-tools/-/issues/36)
+  `parent` keyword in yaml datamodel definition (replaced by
+  `inherit_from_{obligatory|recommended|suggested}` keywords).
+
 ## [0.4.1] - 2022-05-03 ##
 (Henrik tom Wörden)
 

CITATION.cff

+cff-version: 1.2.0
+message: "If you use this software, please cite it as below."
+authors:
+  - family-names: Fitschen
+    given-names: Timm
+    orcid: https://orcid.org/0000-0002-4022-432X
+  - family-names: Schlemmer
+    given-names: Alexander
+    orcid: https://orcid.org/0000-0003-4124-9649
+  - family-names: Hornung
+    given-names: Daniel
+    orcid: https://orcid.org/0000-0002-7846-6375
+  - family-names: tom Wörden
+    given-names: Henrik
+    orcid: https://orcid.org/0000-0002-5549-578X
+  - family-names: Parlitz
+    given-names: Ulrich
+    orcid: https://orcid.org/0000-0003-3058-1435
+  - family-names: Luther
+    given-names: Stefan
+    orcid: https://orcid.org/0000-0001-7214-8125
+title: CaosDB - Advanced User Tools
+version: 0.9.0
+doi: 10.3390/data4020083
+date-released: 2023-11-27
\ No newline at end of file

README.md

@@ -4,6 +4,7 @@
 
 This is the **CaosDB Advanced User Tools** repository and a part of the
 CaosDB project.
+
 This project contains tools that are beyond the typical use of
 the CaosDB python client. Especially, this includes the crawler which will
 typically be used by a data curator.
@@ -43,7 +44,7 @@ Conduct](https://gitlab.com/caosdb/caosdb/-/blob/dev/CODE_OF_CONDUCT.md).
   [documentation](https://docs.indiscale.com/caosdb-advanced-user-tools/), the
   preferred way is also a merge request as describe above (the documentation
   resides in `src/doc`). However, you can also create an issue for it.
-- You can also contact us at **info (AT) caosdb.de** and join the CaosDB
+- You can also contact us at **info (AT) caosdb.org** and join the CaosDB
   community on
   [#caosdb:matrix.org](https://matrix.to/#/!unwwlTfOznjEnMMXxf:matrix.org).
 

README_SETUP.md

@@ -11,7 +11,7 @@ git clone 'https://gitlab.com/caosdb/caosdb-advanced-user-tools'
 Dependencies will be installed automatically if you use the below described
 procedure.
 - `caosdb>=0.6.0`
-- `openpyxl>=3.0.0`
+- `openpyxl>=3.0.7`
 - `xlrd>=1.2.0`
 - `pandas>=1.2.0`
 - `numpy>=1.17.3`
@@ -32,7 +32,10 @@ Optional h5-crawler:
 - `pip install .[h5-crawler] --user`
 
 ## Run Unit Tests
-`tox`
+
+- All tests: `tox`
+- One specific test with tox: `tox -- unittests/test_myusecase.py -k expression`
+- Or even using only pytest: `pytest unittests/test_myusecase.py -k expression`
 
 ## Run Integration Tests Locally
 
@@ -52,6 +55,8 @@ Optional h5-crawler:
 `make style`
 
 ## Documentation #
+We use sphinx to create the documentation. Docstrings in the code should comply
+with the Googly style (see link below).
 
 Build documentation in `build/` with `make doc`.
 
@@ -60,3 +65,9 @@ Build documentation in `build/` with `make doc`.
 - `sphinx`
 - `sphinx-autoapi`
 - `recommonmark >= 0.6.0`
+
+### How to contribute ###
+
+- [Google Style Python Docstrings](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html)
+- [Google Style Python Docstrings 2nd reference](https://github.com/google/styleguide/blob/gh-pages/pyguide.md#38-comments-and-docstrings)
+- [References to other documentation](https://www.sphinx-doc.org/en/master/usage/extensions/intersphinx.html#role-external)

RELEASE_GUIDELINES.md

@@ -24,6 +24,7 @@ guidelines of the CaosDB Project
    - `version` variables in `src/doc/conf.py`
    - Version on [setup.py](./setup.py): Check the `MAJOR`, `MINOR`, `MICRO`, `PRE` variables and set
      `ISRELEASED` to `True`. Use the possibility to issue pre-release versions for testing.
+   - `CITATION.cff` (update version and date)
 
 5. Merge the release branch into the main branch.
 
@@ -41,3 +42,7 @@ guidelines of the CaosDB Project
 11. After the merge of main to dev, start a new development version by
     setting `ISRELEASED` to `False` and by increasing at least the `MICRO`
     version in [setup.py](./setup.py) and preparing CHANGELOG.md.
+
+12. Create releases on gitlab.com and gitlab.indiscale.com that contain (at
+    least) the most recent section of the CHANGELOG as the description and link
+    to the PyPi package.

extra/emacs/readme.md

+# Emacs extras #
+
+This directory contains extra utils for use with Emacs.
+
+## Snippets ##
+
+if you copy the contents of the `snippets` directory to your `~/.emacs.d/snippets/`, the following
+*yasnippet* snippets will become available:
+
+- yaml-mode:
+  - `RT`: Insert a new RecordType, with inheritance and properties sections.
+  - `prop`: Insert a new Property into a RecordType, with datatype and description.

extra/emacs/snippets/yaml-mode/Property inside RecordType

+# -*- mode: snippet -*-
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2022 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2022 Daniel Hornung <d.hornung@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+# name: Property inside RecordType
+# key: prop
+# expand-env: ((yas-indent-line 'fixed))
+# --
+${1:property_name}:
+  datatype: ${2:$$(yas-choose-value '("BOOLEAN"
+                                      "DATETIME"
+                                      "DOUBLE"
+                                      "FILE"
+                                      "INTEGER"
+                                      "LIST"
+                                      "REFERENCE"
+                                      "TEXT"))}
+  description: ${3:description text}
+$0
\ No newline at end of file

extra/emacs/snippets/yaml-mode/RecordType

+# -*- mode: snippet -*-
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2022 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2022 Daniel Hornung <d.hornung@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+# name: RecordType
+# key: RT
+# expand-env: ((yas-indent-line 'fixed))
+# --
+${1:RecordTypeName}:
+  inherit_from_obligatory:$0
+  inherit_from_recommended:
+  inherit_from_suggested:
+  obligatory_properties:
+  recommended_properties:
+  suggested_properties:

integrationtests/caosdbignore

+lol
+~README.md

integrationtests/extroot/Software/2010_TestSoftware/2019-02-03_v0.1/~README.md

+stuff

integrationtests/extroot/Software/2010_TestSoftware/lol

+lol
+

integrationtests/filldb.sh

@@ -5,7 +5,7 @@ python3 -m caosadvancedtools.loadFiles /opt/caosdb/mnt/extroot/ExperimentalData
 python3 -m caosadvancedtools.loadFiles /opt/caosdb/mnt/extroot/DataAnalysis
 python3 -m caosadvancedtools.loadFiles /opt/caosdb/mnt/extroot/SimulationData
 python3 -m caosadvancedtools.loadFiles /opt/caosdb/mnt/extroot/Publications
-python3 -m caosadvancedtools.loadFiles /opt/caosdb/mnt/extroot/Software
+python3 -m caosadvancedtools.loadFiles -c caosdbignore -l $PWD/extroot/Software /opt/caosdb/mnt/extroot/Software
 python3 insert_model.py 
 python3 insert_some.py
 python3 crawl.py /

integrationtests/test.sh

@@ -16,7 +16,8 @@ fi
 OUT=/tmp/crawler.output
 ls
 cat pycaosdb.ini
-rm -rf cache.db
+python3 -c "import linkahead; print('LinkAhead Version:', linkahead.__version__)"
+rm -rf /tmp/caosdb_identifiable_cache.db
 set -e
 echo "Clearing database"
 python3 clear_database.py
@@ -42,21 +43,23 @@ mv DataAnalysis/2010_TestProject/2019-02-03_something/README.xlsx \
    DataAnalysis/2010_TestProject/2019-02-03_something/README.xlsx_back
 cd ..
 echo "run crawler"
-./crawl.py  / | tee $OUT
+./crawl.py  / | tee "$OUT"
 # rename the moved file
 mv extroot/DataAnalysis/2010_TestProject/2019-02-03_something/README.xlsx_back \
    extroot/DataAnalysis/2010_TestProject/2019-02-03_something/README.xlsx
 # check whether there was something UNAUTHORIZED
-grep "There where unauthorized changes" $OUT
+grep "There where unauthorized changes" "$OUT"
 # get the id of the run which is the last field of the output string
-RUN_ID=$(grep "run id:" $OUT | awk '{ print $NF }')
+RUN_ID=$(grep "run id:" "$OUT" | awk '{ print $NF }')
 echo $RUN_ID
 echo "run crawler again"
 echo "./crawl.py -a $RUN_ID /"
-./crawl.py -a $RUN_ID / | tee  $OUT
+./crawl.py -a $RUN_ID / | tee  "$OUT"
 set +e
-if grep "There where unauthorized changes" $OUT
+if grep "There where unauthorized changes" "$OUT"
 then
+    echo "There still were unauthorized changes, which should not have happend!"
+    echo "Test FAILED"
     exit 1
 fi
 set -e
@@ -93,5 +96,8 @@ python3 -m pytest test_json_schema_datamodel_parser.py
 echo "Testing yaml datamodel parser"
 python3 -m pytest test_yaml_parser.py
 
+echo "Testing json-schema exporter"
+python3 -m pytest test_json_schema_exporter.py
+
 # Obsolete due to teardown in the above test.
 # echo "/n/n/n YOU NEED TO RESTART THE SERVER TO REDO TESTS!!!"