Merge branch 'f-ext-applicable' into f-entity-state

.gitignore

+# -*- mode:conf; -*-
+
 # dot files
 .*
 !/.git*
@@ -8,13 +10,19 @@
 
 # the build dir
 /public
+/sss_bin
+/node_modules/
+/build
+__pycache__
+
+# auto-generated sources
+/src/doc/api
 
 # screen logs
 screenlog.*
 xerr.log
 
 # extensions
-
 conf/ext
 test/ext
 src/ext

.gitlab-ci.yml

@@ -6,6 +6,7 @@
 # Copyright (C) 2019 Henrik tom Wörden
 # Copyright (C) 2020 Timm Fitschen (t.fitschen@indiscale.com)
 # Copyright (C) 2020 IndiScale GmbH (info@indiscale.com)
+# Copyright (C) 2020 Daniel Hornung <d.hornung@indiscale.com>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as
@@ -58,6 +59,14 @@ test:
   script:
     - make run-qunit
 
+test-server-side-scripting:
+  timeout: 10 minutes
+  tags: [ docker ]
+  stage: test
+  script:
+    - whereis pytest pytest3 py.test pytest-3 py.test-3
+    - make test-sss
+
 # Trigger building of server image and integration tests
 trigger_build:
   timeout: 15 minutes
@@ -78,13 +87,33 @@ build-testenv:
   tags: [ cached-dind ]
   image: docker:19.03
   stage: setup
+  timeout: 3 h
   script: 
     - cd test/docker
     - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
       # use here general latest or specific branch latest...
-    - docker pull $CI_REGISTRY_IMAGE:latest || true
     - docker build 
       --pull
-      --cache-from $CI_REGISTRY_IMAGE:latest
       -t $CI_REGISTRY_IMAGE:latest .
     - docker push $CI_REGISTRY_IMAGE:latest
+
+# Build the sphinx documentation and make it ready for deployment by Gitlab Pages
+# documentation:
+#   stage: deploy
+
+# Special job for serving a static website. See https://docs.gitlab.com/ee/ci/yaml/README.html#pages
+pages:
+  tags: [ docker ]
+  stage: deploy
+  only:
+    - dev
+  script:
+      # TODO is this a good location here?
+    - npm install jsdoc
+    - npm install jsdoc-sphinx
+    - echo "Deploying"
+    - make doc
+    - rm -r public || true ; cp -r build/doc/html public
+  artifacts:
+    paths:
+      - public

CHANGELOG.md

@@ -15,7 +15,51 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Removed (for now removed features)
 
 ### Fixed
+
+### Security (in case of vulnerabilities)
+
+## [0.3.0] - 2021-02-10
+
+### Added (for new features, dependecies etc.)
+
+- The versioning model has a new styling and can show and tsv-export the full
+  version history now.
+- Module `ext_bookmarks` which allows users to bookmark entities, store them in
+  a link or export them to TSV.
+- table previews in the bottom line module
+- added preview for tif images
+* new function `form_elements.make_alert` which generates a proceed/cancel
+  dialog which intercepts a function call and asks the user for confirmation.
+* Deleting entities prompts for user confirmation after hitting the "Delete"
+  button (edit_mode).
+* Plotly preview has an additional parameter for a config object,
+  e.g., for disabling the plotly logo
+- The map can now show entities that have no geo location but are related to
+  entities that have one. This also effects the search from the map.
+* `getPropertyValues` function which generates a table of property values from
+  a xml representation of entities.
+* After a SELECT statement now also all referenced files can be downloaded.
+* Automated documentation builds: `make doc`
+- documentation on queries
+
+### Changed (for changes in existing functionality)
+
+* ext_map version bumped to 0.4
+- enabled and enhanced autocompletion
+* Login form is hidden behind another button.
+
+### Deprecated (for soon-to-be removed features) 
+
+### Removed (for now removed features)
+
+### Fixed
+
+- #144 (Select with ANY VERSION OF).
+- #136 (adding reference properties to entities in edit mode)
 - exclude configuration files when reading files from build.properties.d
+- summaries when opening preview
+- #125 special characters like "\t", \"n", "#" are replaced in table
+  download
 
 ### Security (in case of vulnerabilities)
 

DEPENDENCIES.md

+* CaosDB Server == 0.3
+* Make 4.2.0

makefile → Makefile

@@ -33,14 +33,17 @@ SQ=\'
 ROOT_DIR = $(abspath .)
 MISC_DIR = $(abspath misc)
 PUBLIC_DIR = $(abspath public)
+SSS_BIN_DIR = $(abspath sss_bin)
 CONF_CORE_DIR = $(abspath conf/core)
 CONF_EXT_DIR = $(abspath conf/ext)
 SRC_CORE_DIR = $(abspath src/core)
 SRC_EXT_DIR = $(abspath src/ext)
+SRC_SSS_DIR = $(abspath src/server_side_scripting)
 LIBS_DIR = $(abspath libs)
 TEST_CORE_DIR = $(abspath test/core/)
 TEST_EXT_DIR = $(abspath test/ext)
-LIBS = fonts css/bootstrap.css js/bootstrap.js js/state-machine.js js/jquery.js js/showdown.js js/dropzone.js css/dropzone.css js/loglevel.js js/leaflet.js css/leaflet.css css/images js/leaflet-latlng-graticule.js js/proj4.js js/proj4leaflet.js js/leaflet-coordinates.js css/leaflet-coordinates.css js/leaflet-graticule.js js/bootstrap-select.js css/bootstrap-select.css js/bootstrap-autocomplete.min.js js/plotly.js
+TEST_SSS_DIR =$(abspath test/server_side_scripting)
+LIBS = fonts css/bootstrap.css js/bootstrap.js js/state-machine.js js/jquery.js js/showdown.js js/dropzone.js css/dropzone.css js/loglevel.js js/leaflet.js css/leaflet.css css/images js/leaflet-latlng-graticule.js js/proj4.js js/proj4leaflet.js js/leaflet-coordinates.js css/leaflet-coordinates.css js/leaflet-graticule.js js/bootstrap-select.js css/bootstrap-select.css js/bootstrap-autocomplete.min.js js/plotly.js js/pako.js js/utif.js
 
 TEST_LIBS = $(LIBS) js/qunit.js css/qunit.css $(subst $(TEST_CORE_DIR)/,,$(shell find $(TEST_CORE_DIR)/))
 
@@ -49,9 +52,9 @@ LIBS_SUBDIRS = $(addprefix $(LIBS_DIR)/, js css fonts)
 
 ALL: install
 
-install: clean cp-src cp-ext cp-conf $(addprefix $(PUBLIC_DIR)/, $(LIBS)) build_properties merge_xsl
+install: clean install-sss cp-src cp-ext cp-conf $(addprefix $(PUBLIC_DIR)/, $(LIBS)) build_properties merge_xsl
 
-test: clean cp-src cp-ext cp-ext-test cp-conf $(addprefix $(PUBLIC_DIR)/, $(TEST_LIBS)) build_properties merge_xsl
+test: clean install-sss cp-src cp-ext cp-ext-test cp-conf $(addprefix $(PUBLIC_DIR)/, $(TEST_LIBS)) build_properties merge_xsl
 	@for f in $(shell find $(TEST_EXT_DIR) -type f -iname *.js) ; do \
 		sed -i "/EXTENSIONS/a \<script src=\"$${f#$(TEST_EXT_DIR)/}\" ></script>" $(PUBLIC_DIR)/index.html ; \
 		echo include $$f; \
@@ -129,18 +132,33 @@ run-qunit: test
 		exit 1; \
 	fi
 
+install-sss:
+	@set -a -e ; \
+	pushd build.properties.files ; \
+	for f in ../build.properties.d/* ; do source "$$f" ; done ; \
+	popd ; \
+	./install-sss.sh $(SRC_SSS_DIR) $(SSS_BIN_DIR)
+
+PYTEST ?= pytest
+PIP ?= pip3
+test-sss: install-sss
+	$(PIP) freeze
+	$(PYTEST) -vv $(TEST_SSS_DIR)
+
+
+CMD_COPY_EXT_FILES = cp -i -r -L
 cp-ext:
 	# TODO FIXME Base path for not-XSL-expanded files
 	mkdir -p $(PUBLIC_DIR)/html
 	for f in $(wildcard $(SRC_EXT_DIR)/html/*) ; do \
-		echo "y" | cp -i -r "$$(realpath "$$f")" $(PUBLIC_DIR)/html/ ; \
+		echo "y" | $(CMD_COPY_EXT_FILES) "$$(realpath "$$f")" $(PUBLIC_DIR)/html/ ; \
 	done
 	for f in $(wildcard $(SRC_EXT_DIR)/js/*) ; do \
-		echo "y" | cp -i -r "$$f" $(PUBLIC_DIR)/js/ ; \
+		echo "y" | $(CMD_COPY_EXT_FILES) "$$f" $(PUBLIC_DIR)/js/ ; \
 		sed -i "/JS_EXTENSIONS/a \<xsl:element name=\"script\"><xsl:attribute name=\"src\"><xsl:value-of select=\"concat\(\$$basepath, 'webinterface/${BUILD_NUMBER}$${f#$(SRC_EXT_DIR)}'\)\" /></xsl:attribute></xsl:element>" $(PUBLIC_DIR)/xsl/main.xsl ; \
 	done
 	for f in $(wildcard $(SRC_EXT_DIR)/css/*) ; do \
-		echo "y" | cp -i -r "$$f" $(PUBLIC_DIR)/css/ ; \
+		echo "y" | $(CMD_COPY_EXT_FILES) "$$f" $(PUBLIC_DIR)/css/ ; \
 		sed -i "/CSS_EXTENSIONS/a \<xsl:element name=\"link\"><xsl:attribute name=\"rel\">stylesheet</xsl:attribute><xsl:attribute name=\"href\"><xsl:value-of select=\"concat\(\$$basepath, 'webinterface/${BUILD_NUMBER}$${f#$(SRC_EXT_DIR)}'\)\" /></xsl:attribute></xsl:element>" $(PUBLIC_DIR)/xsl/main.xsl ; \
 		for html in $(PUBLIC_DIR)/html/* ; do \
 			echo "$$html"; \
@@ -148,26 +166,26 @@ cp-ext:
 		done \
 	done
 	for f in $(wildcard $(SRC_EXT_DIR)/pics/*) ; do \
-		echo "y" | cp -i -r "$$(realpath "$$f")" $(PUBLIC_DIR)/pics/ ; \
+		echo "y" | $(CMD_COPY_EXT_FILES) "$$(realpath "$$f")" $(PUBLIC_DIR)/pics/ ; \
 	done
 	for f in $(wildcard $(SRC_EXT_DIR)/xsl/*) ; do \
-		echo "y" | cp -i -r "$$(realpath "$$f")" $(PUBLIC_DIR)/xsl/ ; \
+		echo "y" | $(CMD_COPY_EXT_FILES) "$$(realpath "$$f")" $(PUBLIC_DIR)/xsl/ ; \
 	done
 
 cp-ext-test:
 	for f in $(wildcard $(TEST_EXT_DIR)/js/*) ; do \
-		echo "y" | cp -i -r "$$f" $(PUBLIC_DIR)/js/ ; \
+		echo "y" | $(CMD_COPY_EXT_FILES) "$$f" $(PUBLIC_DIR)/js/ ; \
 		sed -i "/JS_EXTENSIONS/a \<xsl:element name=\"script\"><xsl:attribute name=\"src\"><xsl:value-of select=\"concat\(\$$basepath, 'webinterface/${BUILD_NUMBER}$${f#$(SRC_EXT_DIR)}'\)\" /></xsl:attribute></xsl:element>" $(PUBLIC_DIR)/xsl/main.xsl ; \
 	done
 	mkdir -p $(PUBLIC_DIR)/html
 	for f in $(wildcard $(TEST_EXT_DIR)/html/*) ; do \
-		echo "y" | cp -i -r "$$(realpath "$$f")" $(PUBLIC_DIR)/html/ ; \
+		echo "y" | $(CMD_COPY_EXT_FILES) "$$(realpath "$$f")" $(PUBLIC_DIR)/html/ ; \
 	done
 	for f in $(wildcard $(TEST_EXT_DIR)/pics/*) ; do \
-		echo "y" | cp -i -r "$$(realpath "$$f")" $(PUBLIC_DIR)/pics/ ; \
+		echo "y" | $(CMD_COPY_EXT_FILES) "$$(realpath "$$f")" $(PUBLIC_DIR)/pics/ ; \
 	done
 	for f in $(wildcard $(TEST_EXT_DIR)/xsl/*) ; do \
-		echo "y" | cp -i -r "$$(realpath "$$f")" $(PUBLIC_DIR)/xsl/ ; \
+		echo "y" | $(CMD_COPY_EXT_FILES) "$$(realpath "$$f")" $(PUBLIC_DIR)/xsl/ ; \
 	done
 
 cp-conf: convert-yaml
@@ -193,13 +211,13 @@ $(PUBLIC_DIR)/%: $(TEST_EXT_DIR)/%
 	cp -r $< $@
 
 $(LIBS_DIR)/fonts: unzip
-	ln -s $(LIBS_DIR)/bootstrap-3.3.7-dist/fonts $@
+	ln -s $(LIBS_DIR)/bootstrap-3.4.1-dist/fonts $@
 
 $(LIBS_DIR)/js/bootstrap.js: unzip $(LIBS_DIR)/js
-	ln -s $(LIBS_DIR)/bootstrap-3.3.7-dist/js/bootstrap.min.js $@
+	ln -s $(LIBS_DIR)/bootstrap-3.4.1-dist/js/bootstrap.min.js $@
 
 $(LIBS_DIR)/css/bootstrap.css: unzip $(LIBS_DIR)/css
-	ln -s $(LIBS_DIR)/bootstrap-3.3.7-dist/css/bootstrap.min.css $@
+	ln -s $(LIBS_DIR)/bootstrap-3.4.1-dist/css/bootstrap.min.css $@
 
 $(LIBS_DIR)/js/bootstrap-select.js: unzip $(LIBS_DIR)/js
 	ln -s $(LIBS_DIR)/bootstrap-select-1.13.9/dist/js/bootstrap-select.min.js $@
@@ -208,7 +226,7 @@ $(LIBS_DIR)/css/bootstrap-select.css: unzip $(LIBS_DIR)/css
 	ln -s $(LIBS_DIR)/bootstrap-select-1.13.9/dist/css/bootstrap-select.min.css $@
 
 $(LIBS_DIR)/js/jquery.js: unzip $(LIBS_DIR)/js
-	ln -s $(LIBS_DIR)/jquery-3.3.1/jquery-3.3.1.min.js $@
+	ln -s $(LIBS_DIR)/jquery-3.5.1/jquery-3.5.1.min.js $@
 
 $(LIBS_DIR)/js/showdown.js: unzip $(LIBS_DIR)/js
 	ln -s $(LIBS_DIR)/showdown-1.8.6/dist/showdown.min.js $@
@@ -259,17 +277,24 @@ $(LIBS_DIR)/css/leaflet-coordinates.css: unzip $(LIBS_DIR)/css
 	ln -s $(LIBS_DIR)/Leaflet.Coordinates-0.1.5/dist/Leaflet.Coordinates-0.1.5.css $@
 
 $(LIBS_DIR)/js/bootstrap-autocomplete.min.js: unzip $(LIBS_DIR)/js
-	ln -s $(LIBS_DIR)/bootstrap-autocomplete-2.3.0/dist/latest/bootstrap-autocomplete.min.js $@
+	ln -s $(LIBS_DIR)/bootstrap-autocomplete-2.3.5/dist/latest/bootstrap-autocomplete.min.js $@
 
 $(LIBS_DIR)/js/plotly.js: unzip $(LIBS_DIR)/js
 	ln -s $(LIBS_DIR)/plotly.js-1.52.2/dist/plotly.min.js $@
 
+$(LIBS_DIR)/js/pako.js: unzip $(LIBS_DIR)/js
+	ln -s $(LIBS_DIR)/pako-dummy/pako.js $@
+
+$(LIBS_DIR)/js/utif.js: unzip $(LIBS_DIR)/js
+	ln -s $(LIBS_DIR)/UTIF-8205c1f/UTIF.js $@
+
 
 $(addprefix $(LIBS_DIR)/, js css):
 	mkdir $@ || true
 
 .PHONY: clean
 clean:
+	$(RM) -r $(SSS_BIN_DIR)
 	$(RM) -r $(PUBLIC_DIR)
 	for f in $(LIBS_SUBDIRS); do unlink $$f || $(RM) -r $$f || true; done
 	for f in $(patsubst %.zip,%/,$(LIBS_ZIP)); do $(RM) -r $$f; done
@@ -280,11 +305,13 @@ unzip:
 	for f in $(LIBS_ZIP); do unzip -q -o -d libs $$f; done
 
 
-PYLINT = pylint3 -d all -e E,F
+PYLINT ?= pylint
 PYTHON_FILES = $(subst $(ROOT_DIR)/,,$(shell find $(ROOT_DIR)/ -iname "*.py"))
 pylint: $(PYTHON_FILES)
-	for f in $(PYTHON_FILES); do $(PYLINT) $$f || exit 1; done
+	for f in $(PYTHON_FILES); do $(PYLINT) -d all -e E,F $$f || exit 1; done
+
 
-PYLINT_LOCAL = /usr/bin/pylint3 -d all -e E,F
-pylint-local: $(PYTHON_FILES)
-	for f in $(PYTHON_FILES); do $(PYLINT_LOCAL) $$f || exit 1; done
+# Compile the standalone documentation
+.PHONY: doc
+doc:
+	$(MAKE) -C src/doc html

README_SETUP.md

@@ -21,12 +21,16 @@
  * ** end header
 -->
 
-# Folder Structure
+# Getting Started with the Web Interface 
+Here, we document how to install and build the CaosDB Web Interface. If you are
+only interested in how to use it, please continue [here](tutorials/first_steps.html)
 
-* The `src` folder contains all source code for the webinterface.
+## Folder Structure
+
+* The `src` folder contains all source code for the web interface.
 * The `libs` folder contains all necessary third-party libraries as zip files.
-* The `test` folder contains the unittests for the webinterface.
-* The `ext` folder contains extension for the webinterface. The make file will
+* The `test` folder contains the unittests for the web interface.
+* The `ext` folder contains extension for the web interface. The make file will
   copy all javascript files from `ext/js/` into the public folder and links the
   javascript in the `public/xsl/main.xsl`.
 * The `misc` folder contains a simple http server which is used for running the
@@ -34,7 +38,7 @@
 * The `build.properties.d/` folder contains configuration files for the build.
 
 
-# Build Configuration
+## Build Configuration
 
 The default configuration is defined in
 `build.properties.d/00_default.properties`.
@@ -46,21 +50,37 @@ All files in that directory will be sourced during `make install` and `make test
 Thus any customized configuration can also be added to that folder by just placing
 files in there which override the default values from `00_default.properties`.
 
-See `build.properties.d/00_default.properties` for more
-information.
+See `build.properties.d/00_default.properties` for more information.
 
-# Setup
+## Setup
 
-* Run `make install` to compile/copy the webinterface to a newly created
+* Run `make install` to compile/copy the web interface to a newly created
   `public` folder.
+* Also, `make install` will copy the scripts from `src/server_side_scripting/`
+  to `sss_bin/`. If you want to make the server-side scripts callable for the
+  server as server-side scripts you need to include the `sss_bin/` directory
+  into the server property `SERVER_SIDE_SCRIPTING_BIN_DIRS`.
 
-# Test
+## Test
 
-* Run `make test` to compile/copy the webinterface and the tests to a newly
+* Run `make test` to compile/copy the web interface and the tests to a newly
   created `public` folder.
 * Run `make run-test-server` to start a python http server.
 * The test suite can be started with `firefox http://localhost:8000/`.
 
-# Clean
+## Clean
 
 * Run `make clean` to clean up everything.
+
+## Documentation #
+
+Build documentation in `build/` with `make doc`.
+
+### Requirements ##
+
+- sphinx
+- sphinx-autoapi
+- jsdoc (`npm install jsdoc`)
+- jsdoc-sphinx  (`npm install jsdoc-sphinx`)
+- sphinx-js
+- recommonmark

build.properties.d/00_default.properties

@@ -46,6 +46,11 @@ BUILD_MODULE_EXT_RESOLVE_REFERENCES=ENABLED
 BUILD_MODULE_EXT_SSS_MARKDOWN=DISABLED
 BUILD_MODULE_EXT_TRIGGER_CRAWLER_FORM=DISABLED
 BUILD_MODULE_EXT_ENTITY_STATE=ENABLED
+BUILD_MODULE_EXT_AUTOCOMPLETE=ENABLED
+BUILD_MODULE_EXT_BOTTOM_LINE=ENABLED
+BUILD_MODULE_EXT_BOTTOM_LINE_TABLE_PREVIEW=DISABLED
+BUILD_MODULE_EXT_BOTTOM_LINE_TIFF_PREVIEW=DISABLED
+BUILD_MODULE_EXT_BOOKMARKS=ENABLED
 
 ##############################################################################
 # Navbar properties

install-sss.sh

+SRC_DIR=$1
+INSTALL_DIR=$2
+
+mkdir -p $INSTALL_DIR
+
+# from here on do your module-wise installing
+
+# ext_table_preview
+if [ "${BUILD_MODULE_EXT_BOTTOM_LINE_TABLE_PREVIEW}" == "ENABLED" ]; then
+    mkdir -p $INSTALL_DIR/ext_table_preview
+    cp $SRC_DIR/ext_table_preview/*.py $INSTALL_DIR/ext_table_preview/
+    echo "installed all server-side scripts for ext_table_preview"
+fi
+# ext_file_download; should always be installed - No build variable
+mkdir -p $INSTALL_DIR/ext_file_download
+cp $SRC_DIR/ext_file_download/*.py $INSTALL_DIR/ext_file_download/
+echo "installed all server-side scripts for ext_file_download"

misc/versioning_test_data.py

@@ -91,3 +91,8 @@ else:
                              str(rec1.id),
                              str(rec1.id)])
 rec4.insert()
+
+for i in range(4,11):
+    rec1.name = f"TestRecord1-{i}thVersion"
+    rec1.description = f"This is the {i}th version."
+    rec1.update()

src/core/css/webcaosdb.css

@@ -27,6 +27,55 @@ body {
     flex-direction: column;
 }
 
+
+div.export-data {
+    display: none;
+}
+
+tr:not(:hover) .caosdb-v-entity-version-hint-cur {
+    color: #DDD;
+}
+
+tr:hover .caosdb-v-entity-version-hint {
+    color: unset;
+}
+
+.caosdb-v-entity-version-hint {
+    color: #DDD;
+}
+
+tbody:not(:hover) tr .caosdb-v-entity-version-hint-cur {
+    color: unset;
+}
+
+.caosdb-v-entity-version-no-related {
+    color: #DDD;
+}
+
+.caosdb-v-entity-version-no-related:hover {
+    color: unset;
+}
+
+#top-navbar>ul>li>a {
+    margin: 8px 0px;
+    padding: 6px 12px;
+    display: inline-block;
+}
+
+.caosdb-v-bookmark-button,
+.caosdb-v-bookmark-button:focus,
+.caosdb-v-bookmark-button:hover {
+    color: #333;
+    position: relative;
+    top: 3px;
+}
+
+.caosdb-v-bookmark-button:active,
+.caosdb-v-bookmark-button.active {
+    color: red;
+}
+
+
 .caosdb-v-navbar-toolbox li a:hover,
 .caosdb-v-navbar-toolbox li input:hover,
 .caosdb-v-navbar-toolbox li button:hover {