Merge branch 'dev' into f-cache-version

.docker-base/Dockerfile

@@ -9,6 +9,34 @@ RUN apk add --no-cache py3-pip python3 python3-dev gcc make \
     git bash curl gettext  py3-requests 
 RUN apk add --no-cache libffi-dev openssl-dev libc-dev libxslt libxslt-dev \
     libxml2 libxml2-dev
+
+# install rust (needed for compiling a docker-compose dependency)
+# This is necessary until alpine comes with an up to date RUST
+# copied from https://github.com/rust-lang/docker-rust/blob/bbc7feb12033da3909dced4e88ddbb6964fbc328/1.50.0/alpine3.13/Dockerfile
+
+ENV RUSTUP_HOME=/usr/local/rustup \
+    CARGO_HOME=/usr/local/cargo \
+    PATH=/usr/local/cargo/bin:$PATH \
+    RUST_VERSION=1.50.0
+
+RUN set -eux; \
+    apkArch="$(apk --print-arch)"; \
+    case "$apkArch" in \
+        x86_64) rustArch='x86_64-unknown-linux-musl'; rustupSha256='05c5c05ec76671d73645aac3afbccf2187352fce7e46fc85be859f52a42797f6' ;; \
+        aarch64) rustArch='aarch64-unknown-linux-musl'; rustupSha256='6a8a480d8d9e7f8c6979d7f8b12bc59da13db67970f7b13161ff409f0a771213' ;; \
+        *) echo >&2 "unsupported architecture: $apkArch"; exit 1 ;; \
+    esac; \
+    url="https://static.rust-lang.org/rustup/archive/1.23.1/${rustArch}/rustup-init"; \
+    wget "$url"; \
+    echo "${rustupSha256} *rustup-init" | sha256sum -c -; \
+    chmod +x rustup-init; \
+    ./rustup-init -y --no-modify-path --profile minimal --default-toolchain $RUST_VERSION --default-host ${rustArch}; \
+    rm rustup-init; \
+    chmod -R a+w $RUSTUP_HOME $CARGO_HOME; \
+    rustup --version; \
+    cargo --version; \
+    rustc --version;
+
 RUN pip3 install docker-compose==1.25
 
 # Script for waiting on LA server

.docker/Dockerfile

-FROM debian:10
+FROM debian:11
 RUN apt-get update && \
     apt-get install \
     curl \
+    libhdf5-dev \
+    pkgconf \
     python3 \
     python3-pip \
     python3-requests \
@@ -11,29 +13,22 @@ RUN apt-get update && \
     tox \
     git \
     openjdk-11-jdk-headless \
-    python-autopep8 \
+    python3-autopep8 \
     python3-pytest \
     libxml2 \
     -y
+
+
 COPY .docker/wait-for-it.sh /wait-for-it.sh
 ADD https://gitlab.com/api/v4/projects/13656973/repository/branches/dev \
    pylib_version.json
 RUN git clone https://gitlab.com/caosdb/caosdb-pylib.git && \
    cd caosdb-pylib && git checkout dev && pip3 install .
-ADD https://gitlab.com/api/v4/projects/13656965/repository/branches/master \
-   model_version.json
-RUN git clone https://gitlab.com/caosdb/caosdb-models.git && \
-   cd caosdb-models && pip3 install .
-ADD https://gitlab.com/api/v4/projects/13601752/repository/branches/master \
-   scifolder_version.json
-RUN git clone \
-    https://gitlab.com/henrik_indiscale/scifolder.git && \
-    cd scifolder && pip3 install .
+# At least recommonmark 0.6 required.
+RUN pip3 install -U html2text pycodestyle pylint recommonmark sphinx-rtd-theme
 COPY . /git
 RUN rm -r /git/.git \
     && mv /git/.docker/pycaosdb.ini /git/integrationtests
-RUN cd /git && pip3 install .
+RUN cd /git && pip3 install .[h5-crawler]
 WORKDIR /git/integrationtests
-CMD /wait-for-it.sh caosdb-server:10443 -t 500 -- ./test.sh
-# At least recommonmark 0.6 required.
-RUN pip3 install recommonmark sphinx-rtd-theme
+CMD /wait-for-it.sh caosdb-server:10443 -t 500 -- ./test.sh --force

.docker/docker-compose.yml

@@ -7,7 +7,7 @@ services:
     networks:
       - caosnet
   caosdb-server:
-    image: "$CI_REGISTRY_INDISCALE/caosdb/src/caosdb-deploy:$CAOSDB_TAG"
+    image: "$CI_REGISTRY/caosdb/src/caosdb-deploy:$CAOSDB_TAG"
     user: 999:999
     depends_on:
       - sqldb

.gitignore

 # -*- mode:conf; -*-
+# generated
+src/caosadvancedtools/version.py
 
 # compiled python and dist stuff
 *.egg

.gitlab-ci.yml

@@ -21,13 +21,9 @@
 # along with this program. If not, see <https://www.gnu.org/licenses/>.
 
 variables:
-   CI_REGISTRY_IMAGE: $CI_REGISTRY/caosdb/caosdb-advanced-user-tools/testenv:latest
-   CI_REGISTRY_IMAGE_BASE: $CI_REGISTRY/caosdb/caosdb-advanced-user-tools/base:latest
-   # When using dind, it's wise to use the overlayfs driver for
-   # improved performance.
+   CI_REGISTRY_IMAGE: $CI_REGISTRY/caosdb/src/caosdb-advanced-user-tools/testenv:latest
+   CI_REGISTRY_IMAGE_BASE: $CI_REGISTRY/caosdb/src/caosdb-advanced-user-tools/base:latest
 
-services:
-  - docker:19.03.0-dind
 
 stages: 
   - setup
@@ -38,33 +34,37 @@ stages:
   - deploy
 
 test:
-  tags: [cached-dind]
+  tags: [docker]
+  services:
+    - docker:20.10.5-dind
+  variables:
+    # This is a workaround for the gitlab-runner health check mechanism when
+    # using docker-dind service.  The runner will otherwise guess the port
+    # wrong and the health check will timeout.
+    SERVICE_PORT_2376_TCP_PORT: 2375
   stage: integrationtest
   image: $CI_REGISTRY_IMAGE_BASE
   script:
       - if [[ "$CAOSDB_TAG" == "" ]]; then
-            CAOSDB_TAG=dev-latest;
+            CAOSDB_TAG=dev;
         fi
       - echo $CAOSDB_TAG
       - time docker load < /image-cache/caosdb-advanced-testenv.tar || true
       - time docker load < /image-cache/mariadb.tar || true
-      - time docker load < /image-cache/caosdb.tar || true
+      - time docker load < /image-cache/caosdb-dev.tar || true
       - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
-      - docker login -u gitlab+deploy-token-ci-pull -p $TOKEN_CI_PULL $CI_REGISTRY_INDISCALE
-      - time docker pull $CI_REGISTRY_IMAGE
-      - time docker pull mariadb:10.4
-      - time docker pull $CI_REGISTRY_INDISCALE/caosdb/src/caosdb-deploy:$CAOSDB_TAG
       - EXEPATH=`pwd` CAOSDB_TAG=$CAOSDB_TAG docker-compose 
         -f .docker/docker-compose.yml up -d
       - cd .docker 
       - /bin/sh ./run.sh
       - cd .. 
-      - docker logs docker_caosdb-server_1 &> ../caosdb_log.txt
-      - docker logs docker_sqldb_1 &> ../mariadb_log.txt
+      - docker logs docker_caosdb-server_1 &> caosdb_log.txt
+      - docker logs docker_sqldb_1 &> mariadb_log.txt
       - docker-compose -f .docker/docker-compose.yml down
       - rc=`cat .docker/result`  
       - exit $rc
   dependencies: [cert]
+  needs: [cert]
   artifacts:
     paths:
       - caosdb_log.txt
@@ -76,11 +76,10 @@ build-testenv:
   tags: [cached-dind]
   image: docker:18.09
   stage: setup
-  only:
-      - schedules
-      - web
+  # Hint: do not use only here; the image needs always to be build since it 
+  # contains the repo code
+  #only:
   script: 
-      - df -h
       - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
         # use here general latest or specific branch latest...
       - docker build 
@@ -97,6 +96,7 @@ cert:
   tags: [docker]
   stage: cert
   image: $CI_REGISTRY_IMAGE
+  needs: [build-testenv]
   artifacts:
     paths:
       - .docker/cert/
@@ -104,35 +104,43 @@ cert:
   script:
       - cd .docker
       - CAOSHOSTNAME=caosdb-server ./cert.sh
+
 style:
   tags: [docker]
   stage: style
   image: $CI_REGISTRY_IMAGE
+  needs: [build-testenv]
   script:
-      # For the moment, ignore type comparisons in datamodel_problems.py
-      - autopep8 -ar --diff --exit-code --exclude ./src/caosadvancedtools/datamodel_problems.py .
-      - autopep8 -ar --diff --exit-code --ignore E721 ./src/caosadvancedtools/datamodel_problems.py
+      - make style
+  allow_failure: true
+
+linting:
+  tags: [docker]
+  stage: style
+  image: $CI_REGISTRY_IMAGE
+  needs: [build-testenv]
+  script:
+      - make lint
   allow_failure: true
 
 unittest:
   tags: [docker]
   stage: unittest
   image: $CI_REGISTRY_IMAGE
+  needs: [build-testenv]
   script:
       - python3 -c "import caosdb; print('CaosDB Version:', caosdb.__version__)"
       - tox
 
 # Build the sphinx documentation and make it ready for deployment by Gitlab Pages
-# documentation:
-#   stage: deploy
-
 # Special job for serving a static website. See https://docs.gitlab.com/ee/ci/yaml/README.html#pages
-pages:
-  stage: deploy
-  image: $CI_REGISTRY_IMAGE
+pages_prepare: &pages_prepare
   tags: [docker]
+  image: $CI_REGISTRY_IMAGE
+  stage: deploy
   only:
-    - dev
+    refs:
+      - /^release-.*$/
   script:
     - echo "Deploying"
     - make doc
@@ -140,3 +148,9 @@ pages:
   artifacts:
     paths:
       - public
+pages:
+  <<: *pages_prepare
+  only:
+    refs:
+      # version tags: v0.1.1
+      - /^v(\d+\.\d+\.\d+)$/

.gitlab/merge_request_templates/Default.md

-# Summary
-
-    Insert a meaningful description for this merge request here.  What is the
-    new/changed behavior? Which bug has been fixed? Are there related Issues?
-
-# Focus
-
-    Point the reviewer to the core of the code change. Where should they start
-    reading? What should they focus on (e.g. security, performance,
-    maintainability, user-friendliness, compliance with the specs, finding more
-    corner cases, concrete questions)?
-
-# Test Environment
-
-    How to set up a test environment for manual testing?
-
-# Check List for the Author
-
-Please, prepare your MR for a review. Be sure to write a summary and a
-focus and create gitlab comments for the reviewer. They should guide the
-reviewer through the changes, explain your changes and also point out open
-questions. For further good practices have a look at [our review
-guidelines](https://gitlab.com/caosdb/caosdb/-/blob/dev/REVIEW_GUIDELINES.md)
-
-- [ ] All automated tests pass
-- [ ] Reference related Issues
-- [ ] Up-to-date CHANGELOG.md
-- [ ] Annotations in code (Gitlab comments)
-  - Intent of new code
-  - Problems with old code
-  - Why this implementation?
-
-
-# Check List for the Reviewer
-
-
-- [ ] I understand the intent of this MR
-- [ ] All automated tests pass
-- [ ] Up-to-date CHANGELOG.md
-- [ ] The test environment setup works and the intended behavior is
-  reproducible in the test environment
-- [ ] In-code documentation and comments are up-to-date.
-- [ ] Check: Are there spezifications? Are they satisfied?
-
-For further good practices have a look at [our review guidelines](https://gitlab.com/caosdb/caosdb/-/blob/dev/REVIEW_GUIDELINES.md).
-
-
-/assign me
-/target_branch dev

CHANGELOG.md

@@ -8,7 +8,75 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added ###
 
-* `send_mail` function in `caosadvancedtools.serverside.helper` module
+### Changed ###
+
+### Deprecated ###
+
+### Removed ###
+
+### Fixed ###
+
+### Security ###
+
+## [0.4.1] - 2022-05-03 ##
+(Henrik tom Wörden)
+
+### Changed ###
+
+- `JsonSchemaParser` now identifies `name` properties in the schema with the
+  CaosDB name property.
+
+### Fixed ###
+
+- [#40](https://gitlab.com/caosdb/caosdb-advanced-user-tools/-/issues/40)
+  `assure_object_is_in_list` now handles adding objects to an initially empty list correctly.
+
+## [0.4.0] - 2022-04-05  ##
+
+### Added ###
+
+- CFood that creates a Record for each line in a csv file
+- `generic_analysis.py` allows to easily call scripts to perform analyses in
+  server side scripting [EXPERIMENTAL]
+- **EXPERIMENTAL:** Models parser can import from Json Schema files now:
+  `models.parser.parse_model_from_json_schema(...)`. See the documentation of
+  `models.parser.JsonSchemaParser` for the limitations of the current
+  implementation.
+- New keyword "role" in yaml data model that allows creation of Records and Files.
+- It is now possible to set values of properties and default values of properties
+  directly in the yaml model.
+
+### Changed ###
+
+- `TableConverter` now converts int to float and vice versa to match the desired dtype.
+
+### Deprecated ###
+
+### Removed ###
+
+### Fixed ###
+
+- CaosDB internal properties `name`, `unit` and `description` can now be used via the `extern`
+  keyword in YAML datamodel specifications.
+
+### Security ###
+
+## [0.3.1] - 2021-12-06  ##
+
+### Added ###
+- `check_reference_field` function to check whether entities with provided ids
+  exits (for example when importing data from a table)
+- added the `datatypes` argument to `TableImporter` for columns that do not
+  need a special conversion function
+
+## [0.3.0] - 2021-11-02 ##
+
+### Added ###
+
+- Error handling for missing files when using the crawler
+- included the scifolder module
+- included the caosmodels module
+- `send_mail` function in `caosadvancedtools.serverside.helper` module
 - New class to collect possible problems with the data model
 - New class for checking and importing tables
 - Function to get a file path to a shared resource directory
@@ -23,12 +91,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   get_file_via_download
 - Automated documentation builds: `make doc`
 - Crawler documentation
+- Proof-of-concept integration with Bloxberg.
+- Introduce a cfood that can create a Record structure based on the contents of a hdf5 file
+  h5py is now an optional dependency
+- table importer implementations for csv and tsv
+- string-in-list check for table imports
+- AbtractCFood has new property, `update_flags`.
 
 ### Changed ###
 
-* `caosadvancedtools.serverside.helper.init_data_model` also checks the role
+- identifiables of single CFoods are now treated one after the other. This
+  allows them to have dependencies among each other if they are ordered
+  correctly
+- identifiables must have at least one property or a name
+- `caosadvancedtools.serverside.helper.init_data_model` also checks the role
   and data type of entities.
-* The `caosadvancedtools.table_importer.date_converter` now actually returns
+- The `caosadvancedtools.table_importer.date_converter` now actually returns
   `datetime.date` instance. A new
   `caosadvancedtools.table_importer.datetime_converter` replaces the old
   `date_converter` and returns a `datetime.datetime` instance.
@@ -37,18 +115,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - instead of `get_entity`, type-specific functions are used in
   `cfood.py` when the type of the entity in question is known.
 - Logger is used instead of `print` for errors in `crawler.py`.
-
-### Deprecated ###
-
-### Removed ###
+- complies with new exception handling, i.e., TransactionErros with
+  children being raised in all cases of EntityErrors (see
+  [#32](https://gitlab.com/caosdb/caosdb-pylib/-/issues/32) in
+  caosdb-pylib)
+- `caosadvancedtools.cfood.assure_object_is_in_list` conducts in-place
+  updates if no `to_be_updated` object is supplied.
 
 ### Fixed ###
 - An exception in collect_information does no longer lead to a break down.
-* Fixed an issue where `caosadvancedtools.cache.UpdateCache` would
+- Removed dependency on discontiued xlrd version
+- Fixed an issue where `caosadvancedtools.cache.UpdateCache` would
   cause an `sqlite3.IntegrityError` if more than one change was cached
   for the same entity.
-
-### Security ###
+- #40 Insertion of identifiables with missing obligatory properties
+- Before, a Property with the datatype "LIST(TEXT)" would lead to the creation
+  of a RecordType. This is fixed now.
+- #52 `XLSimporter.read_xls` throwed a wrong error when reading from a file with a wrong ending.
+  Now, a `DataInconsistencyError` is raised instead of a ValueError.
+- List properties are no longer updated unnecessarily by the crawler.
 
 ## [0.2.0] - 2020-05-28 ##
 

FEATURES.md

+# Features
+
+## Stable
+To be filled.
+
+## Experimental
+
+- `generic_analysis.py` allows to easily call scripts to perform analyses in
+  server side scripting
+-  Models parser can import from Json Schema files:
+  `models.parser.parse_model_from_json_schema(...)`. See the documentation of
+  `models.parser.JsonSchemaParser` for the limitations of the current
+  implementation.

Makefile

@@ -21,7 +21,7 @@
 
 # This Makefile is a wrapper for several other scripts.
 
-.PHONY: help doc install
+.PHONY: help doc install unittest
 
 help:
 	@echo 'Type `make doc` for documentation, or `make install` for (local) installation.'
@@ -30,4 +30,16 @@ doc:
 	$(MAKE) -C src/doc html
 
 install:
-	@echo "Not implemented yet, use pip for installation."
+	pip3 install .
+
+unittest:
+	pytest-3 unittests
+
+style:
+	pycodestyle --count src unittests --exclude=swagger_client
+	autopep8 -ar --diff --exit-code --exclude swagger_client .
+.PHONY: style
+
+lint:
+	pylint --unsafe-load-any-extension=y -d all -e E,F --ignore=swagger_client src/caosadvancedtools
+.PHONY: lint

README.md

-[![pipeline status](https://gitlab.com/caosdb/caosdb-advanced-user-tools/badges/master/pipeline.svg)](https://gitlab.com/caosdb/caosdb-advanced-user-tools/commits/master)
+# README
 
-Project migrated to https://gitlab.com/caosdb
-
-# Welcome
+## Welcome
 
 This is the **CaosDB Advanced User Tools** repository and a part of the
-CaosDB project. This project contains tools that are beyond the typical use of
+CaosDB project.
+This project contains tools that are beyond the typical use of
 the CaosDB python client. Especially, this includes the crawler which will
 typically be used by a data curator.
 
-# Setup
+## Setup
 
 Please read the [README_SETUP.md](README_SETUP.md) for instructions on how to
 setup this code.
 
+## Further Reading
+
+Please refer to the [official
+documentation](https://docs.indiscale.com/caosdb-advanced-user-tools/) for more
+information.
+
+## Contributing
+
+Thank you very much to all contributers—[past,
+present](https://gitlab.com/caosdb/caosdb/-/blob/dev/HUMANS.md), and prospective
+ones.
 
-# Further Reading
+### Code of Conduct
 
-Please refer to the [official gitlab repository of the CaosDB
-project](https://gitlab.com/caosdb/caosdb) for more information.
+By participating, you are expected to uphold our [Code of
+Conduct](https://gitlab.com/caosdb/caosdb/-/blob/dev/CODE_OF_CONDUCT.md).
 
-# License
+### How to Contribute
 
-Copyright (C) 2018 Research Group Biomedical Physics, Max Planck Institute for
-Dynamics and Self-Organization Göttingen.
+- You found a bug, have a question, or want to request a feature? Please [create
+  an issue](https://gitlab.com/caosdb/caosdb-advanced-user-tools/-/issues).
+- You want to contribute code? Please fork the repository and create a merge
+  request in GitLab and choose this repository as target. Make sure to select
+  "Allow commits from members who can merge the target branch" under
+  Contribution when creating the merge request. This allows our team to work
+  with you on your request.
+- If you have a suggestion for the
+  [documentation](https://docs.indiscale.com/caosdb-advanced-user-tools/), the
+  preferred way is also a merge request as describe above (the documentation
+  resides in `src/doc`). However, you can also create an issue for it.
+- You can also contact us at **info (AT) caosdb.de** and join the CaosDB
+  community on
+  [#caosdb:matrix.org](https://matrix.to/#/!unwwlTfOznjEnMMXxf:matrix.org).
+
+## License
+
+* Copyright (C) 2018 Research Group Biomedical Physics, Max Planck Institute
+  for Dynamics and Self-Organization Göttingen.
+* Copyright (C) 2020-2021 Indiscale GmbH <info@indiscale.com>
 
 All files in this repository are licensed under a [GNU Affero General Public
 License](LICENCE.md) (version 3 or later).
-

README_SETUP.md

@@ -8,23 +8,28 @@ git clone 'https://gitlab.com/caosdb/caosdb-advanced-user-tools'
 ```
 
 ## Dependencies
-Dependencies will be installed automatically if you use the below described procedure.
-- `caosdb>=0.4.0`                                      
+Dependencies will be installed automatically if you use the below described
+procedure.
+- `caosdb>=0.6.0`
 - `openpyxl>=3.0.0`
 - `xlrd>=1.2.0`
+- `pandas>=1.2.0`
+- `numpy>=1.17.3`
+
+If you want to use the optional h5-crawler the following dependencies will be
+installed additionally:
+- `h5py>=3.3.0`
 
 For testing:
 - `tox`
-- `scifolder`from https://gitlab.com/henrik_indiscale/scifolder
 
 
 ## Installation
 - `pip install . --user`
 - `pip install tox --user`
 
-In order to run the tests you need to install the [scifolder
-package](https://gitlab.com/henrik_indiscale/scifolder) by Henrik tom
-Wörden.
+Optional h5-crawler:
+- `pip install .[h5-crawler] --user`
 
 ## Run Unit Tests
 `tox`
@@ -32,17 +37,19 @@ Wörden.
 ## Run Integration Tests Locally
 
 1. Change directory to `integrationtests/`.
-2. Mount `extroot` to the folder that will be used as
-   extroot. E.g. `sudo mount -o bind extroot
-   ../../caosdb-deploy/profiles/empty/paths/extroot` (or whatever path
-   the extroot of the empty profile to be used is located at).
-3. Start an empty (!) CaosDB instance (with the mounted extroot). The
-   database will be cleared during testing, so it's important to use
-   an empty insctance.
-4. Run `test.sh`.
+2. Mount `extroot` to the folder that will be used as extroot. E.g. `sudo mount
+   -o bind extroot ../../caosdb-deploy/profiles/debug/paths/extroot` (or
+   whatever path the extroot of the empty profile to be used is located at).
+3. Start (or restart) an empty (!) CaosDB instance (with the mounted
+   extroot). The database will be cleared during testing, so it's important to
+   use an empty instance. Make sure your configuration for the python caosdb
+   module is correct and allows to connect to the server.
+4. Run `test.sh`. Note that this may modify content of the
+   `integrationtest/extroot/` directory.
 
 ## Code Formatting
-`autopep8 -i -r ./`
+
+`make style`
 
 ## Documentation #
 

RELEASE_GUIDELINES.md

+# Release Guidelines for the CaosDB Python Client Library
+
+This document specifies release guidelines in addition to the general release
+guidelines of the CaosDB Project
+([RELEASE_GUIDELINES.md](https://gitlab.com/caosdb/caosdb/blob/dev/RELEASE_GUIDELINES.md))
+
+## General Prerequisites
+
+* All tests are passing.
+* FEATURES.md is up-to-date and a public API is being declared in that document.
+* CHANGELOG.md is up-to-date.
+* dependencies in `setup.py` are up-to-date.
+
+## Steps
+
+1. Create a release branch from the dev branch. This prevents further changes
+   to the code base and a never ending release process. Naming: `release-<VERSION>`
+
+2. Update CHANGELOG.md
+
+3. Check all general prerequisites.
+
+4. Update the version:
+   - `version` variables in `src/doc/conf.py`
+   - Version on [setup.py](./setup.py): Check the `MAJOR`, `MINOR`, `MICRO`, `PRE` variables and set
+     `ISRELEASED` to `True`. Use the possibility to issue pre-release versions for testing.
+
+5. Merge the release branch into the main branch.
+
+6. Tag the latest commit of the main branch with `v<VERSION>`.
+
+7. Delete the release branch.
+
+8. Remove possibly existing `./dist` directory with old release.
+
+9. Publish the release by executing `./release.sh` with uploads the caosdb
+   module to the Python Package Index [pypi.org](https://pypi.org).
+
+10. Merge the main branch back into the dev branch.
+
+11. After the merge of main to dev, start a new development version by
+    setting `ISRELEASED` to `False` and by increasing at least the `MICRO`
+    version in [setup.py](./setup.py) and preparing CHANGELOG.md.

integrationtests/crawl.py

@@ -32,8 +32,11 @@ import caosdb as db
 from caosadvancedtools.cfood import fileguide
 from caosadvancedtools.crawler import FileCrawler
 from caosadvancedtools.guard import INSERT, UPDATE
-from scifolder import (AnalysisCFood, ExperimentCFood, PublicationCFood,
-                       SimulationCFood, SoftwareCFood)
+from caosadvancedtools.scifolder import (AnalysisCFood, ExperimentCFood,
+                                         PublicationCFood, SimulationCFood,
+                                         SoftwareCFood, ResultTableCFood)
+
+from example_hdf5cfood import ExampleH5CFood
 
 try:
     from sss_helper import get_argument_parser, print_success
@@ -88,6 +91,8 @@ if __name__ == "__main__":
                     interactive=False, hideKnown=False,
                     cfood_types=[ExperimentCFood, AnalysisCFood, SoftwareCFood,
                                  PublicationCFood, SimulationCFood,
+                                 ResultTableCFood,
+                                 ExampleH5CFood
                                  ])
 
     if args.authorize_run:

integrationtests/create_analysis.py

+#!/usr/bin/env python3
+# encoding: utf-8
+#
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2021 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2021 Henrik tom Wörden <h.tomwoerden@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+
+"""
+A small script that creates an Analysis Record that can be used for testing the
+automated analysis pipeline.
+"""
+
+import sys
+from datetime import datetime
+
+import caosdb as db
+
+
+def main():
+    script = db.File(
+        file="../src/caosadvancedtools/serverside/examples/example_script.py",
+        path=("AutomatedAnalysis/scripts/"
+              + str(datetime.now())+"example_script.py"),
+    )
+    script.insert()
+
+    da = db.Record()
+    da.add_parent("Analysis")
+    da.add_property("scripts", value=[script], datatype=db.LIST(db.FILE))
+    da.add_property("sources",
+                    value=db.execute_query(
+                        "FIND FILE which is stored at '**/timeseries.npy'",
+                        unique=True),
+                    )
+    da.add_property("date", "2020-01-01")
+    da.add_property("identifier", "TEST")
+    only = db.execute_query(
+                        "FIND RECORD Person WITH firstname=Only",
+                        unique=True)
+    only.add_property(db.Property("Email").retrieve().id, "only@example.com")
+    only.update()
+    da.add_property("responsible", only)
+    da.insert()
+
+
+if __name__ == "__main__":
+    sys.exit(main())

integrationtests/example_hdf5cfood.py

+#!/usr/bin/env python3
+# encoding: utf-8
+#
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2021 IndiScale GmbH <www.indiscale.com>
+# Copyright (C) 2021 Henrik tom Wörden <h.tomwoerden@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+
+"""
+An exemplary definition of a HDF5 CFood for integration testing
+"""
+
+import caosdb as db
+from caosadvancedtools.cfoods.h5 import H5CFood
+from caosadvancedtools.scifolder import ExperimentCFood
+from caosadvancedtools.scifolder.generic_pattern import readme_pattern
+
+
+class ExampleH5CFood(H5CFood):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.root_name = "ExampleH5"
+
+    @staticmethod
+    def get_re():
+        return ExperimentCFood.get_re()[:-len(readme_pattern)] + r".*\.hdf5"
+
+    def create_identifiables(self):
+        super().create_identifiables()
+        self.identifiable_root = db.Record()
+        self.identifiable_root.add_property("hdf5File", self.crawled_file)
+        self.identifiable_root.add_parent("ExampleH5")
+        self.identifiables.append(self.identifiable_root)
+
+    def special_treatment(self, key, value, dtype):
+        if key == "attr_data_root":
+            return "single_attribute", value, dtype
+
+        return key, value, dtype

integrationtests/example_script.py

+../src/caosadvancedtools/serverside/examples/example_script.py
\ No newline at end of file

integrationtests/extroot/DataAnalysis/2010_TestProject/2019-02-03/plot.py

+import plot

integrationtests/extroot/ExperimentalData/2010_TestProject/2019-02-03/datafile.dat

+datadatadata