diff --git a/.gitignore b/.gitignore
index 39be17dc807dc3e2f5c505dc9341027457403421..06c8c148f1e9f45493f574a11c6789398246defd 100644
--- a/.gitignore
+++ b/.gitignore
@@ -29,3 +29,6 @@ log/
OUTBOX
ConsistencyTest.xml
testlog/
+
+# python
+__pycache__
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 7161569a7e43017869d9bb0025c35388db6c3623..34e93fd70f3252e3dd04b2df14e06a63299d940c 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -26,10 +26,6 @@ variables:
CI_REGISTRY_IMAGE: $CI_REGISTRY/caosdb-server-testenv:latest
# When using dind, it's wise to use the overlayfs driver for
# improved performance.
- DOCKER_DRIVER: overlay2
-
-services:
- - docker:19.03-dind
image: $CI_REGISTRY_IMAGE
stages:
@@ -42,6 +38,7 @@ test:
tags: [ docker ]
stage: test
script:
+ - make test_misc
- make easy-units
- mvn dependency:purge-local-repository
- mvn antlr4:antlr4
@@ -55,20 +52,22 @@ trigger_build:
stage: deploy
script:
- /usr/bin/curl -X POST
- -F token=8f29e5eeb7db2123d9c2bb84634da2
+ -F token=$DEPLOY_TRIGGER_TOKEN
-F "variables[SERVER]=$CI_COMMIT_REF_NAME"
-F "variables[TriggerdBy]=SERVER"
-F "variables[TriggerdByHash]=$CI_COMMIT_SHORT_SHA"
- -F ref=master https://gitlab.indiscale.com/api/v4/projects/14/trigger/pipeline
+ -F ref=dev https://gitlab.indiscale.com/api/v4/projects/14/trigger/pipeline
# Build a docker image in which tests for this repository can run
build-testenv:
- tags: [ docker ]
+ tags: [ cached-dind ]
image: docker:19.03
stage: setup
+ only:
+ - schedules
script:
- cd src/test/docker
- - docker login -u testuser -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+ - docker login -u indiscale -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
# use here general latest or specific branch latest...
- docker pull $CI_REGISTRY_IMAGE || true
- docker build
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4d8a87996586347cb32457bfbb06e524f380098b..8b65bafc9f260dabecd592fea4f08829f08db7c4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,7 +2,7 @@
All notable changes to this project will be documented in this file.
-The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [Unreleased]
@@ -11,16 +11,27 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Scripting is simplified by adding a `home` directory, of which a copy is
created for each called script and set as the `HOME` environment variable.
+- [bend_symlinks.sh](misc/bend_symlinks/bend_symlinks.sh) (version 0.1, experimental)
+ fix broken symlinks in the internal file system. See
+ [README.md](misc/bend_symlinks/README.md)
+- [move_files.py](misc/move_files/move_files.py) (version 0.1, experimental)
+ Script for moving files (change their path) in the internal file system based
+ on a two-column tsv file (with columns "from" and "to"). See
+ [README.md](misc/move_files/README.md).
### Changed
--
-
+- The sever by default now only serves TLS 1.2 and 1.3, all previous versions
+ have been disabled in the default settings. Make sure that your clients
+ (especially the Python client) are up to date.
### Deprecated
-
+### Removed
+
+-
### Fixed
@@ -29,10 +40,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- NaN Double Values (see #41)
-
+- #14 - Handle files on file system without File entity: Those entries are
+ returned without ID but with a notice now.
### Security (in case of vulnerabilities)
+- TLS is by default restricted to v1.2 and v1.3 now.
+
+
## [0.1.0] - 2018-10-09
Tag `v0.1` - Commit 3b17b49
diff --git a/caosdb-webui b/caosdb-webui
index 88566d42a698e939832fbff535a04fb5f12eec6b..b6677e848ed30cadeda5d1782bd4376ec6546031 160000
--- a/caosdb-webui
+++ b/caosdb-webui
@@ -1 +1 @@
-Subproject commit 88566d42a698e939832fbff535a04fb5f12eec6b
+Subproject commit b6677e848ed30cadeda5d1782bd4376ec6546031
diff --git a/conf/core/cache.ccf b/conf/core/cache.ccf
index 87d1338a5cf70da9b551744bc154190965d4bb46..821e5d7862efb21e0aa13f8410886c6c14b10a7c 100644
--- a/conf/core/cache.ccf
+++ b/conf/core/cache.ccf
@@ -1,3 +1,51 @@
+# default caching options
jcs.default.cacheattributes=org.apache.commons.jcs.engine.CompositeCacheAttributes
jcs.default.cacheattributes.MaxObjects=1000
-jcs.default.cacheattributes.MemoryCacheName=org.apache.commons.jcs.engine.memory.lru.LRUMemoryCache
\ No newline at end of file
+jcs.default.elementattributes=org.apache.commons.jcs.engine.ElementAttributes
+
+# Caching of the backend transactions
+jcs.region.BACKEND_UserRoles
+jcs.region.BACKEND_UserRoles.cacheattributes.MaxObjects=101
+
+jcs.region.BACKEND_Users
+jcs.region.BACKEND_Users.cacheattributes.MaxObjects=102
+
+jcs.region.BACKEND_PermissionRules
+jcs.region.BACKEND_PermissionRules.cacheattributes.MaxObjects=1001
+
+jcs.region.BACKEND_EntityProperties
+jcs.region.BACKEND_EntityProperties.cacheattributes.MaxObjects=1003
+
+jcs.region.BACKEND_EntityParents
+jcs.region.BACKEND_EntityParents.cacheattributes.MaxObjects=1004
+
+jcs.region.BACKEND_SparseFileRecordsByPath
+jcs.region.BACKEND_SparseFileRecordsByPath.cacheattributes.MaxObjects=1005
+
+jcs.region.BACKEND_JobRules
+jcs.region.BACKEND_JobRules.cacheattributes.MaxObjects=103
+
+jcs.region.BACKEND_SparseEntities
+jcs.region.BACKEND_SparseEntities.cacheattributes.MaxObjects=1002
+
+
+# PAM UserSource Caching: Cached Items expire after 60 seconds if they are not requested (idle) and after 600 seconds max.
+# PAM_UnixUserGroups
+jcs.region.PAM_UnixUserGroups
+jcs.region.PAM_UnixUserGroups.elementattributes.IsEternal=false
+jcs.region.PAM_UnixUserGroups.elementattributes.MaxLife=601
+jcs.region.PAM_UnixUserGroups.elementattributes.IdleTime=61
+
+# PAM_UnixUserExists
+jcs.region.PAM_UnixUserExists
+jcs.region.PAM_UnixUserExists.elementattributes.IsEternal=false
+jcs.region.PAM_UnixUserExists.elementattributes.MaxLife=599
+jcs.region.PAM_UnixUserExists.elementattributes.IdleTime=59
+
+# PAM_Authentication
+jcs.region.PAM_Authentication
+jcs.region.PAM_Authentication.elementattributes.IsEternal=false
+jcs.region.PAM_Authentication.elementattributes.MaxLife=600
+jcs.region.PAM_Authentication.elementattributes.IdleTime=60
+
+
diff --git a/conf/core/log4j2-default.properties b/conf/core/log4j2-default.properties
index ab7aed268e1e1e1a4088796d68b9e065086b5958..b1697a73fe6703850865406e1441947614d00f47 100644
--- a/conf/core/log4j2-default.properties
+++ b/conf/core/log4j2-default.properties
@@ -12,6 +12,11 @@ appender.stderr.name = stderr
appender.stderr.layout.type = PatternLayout
appender.stderr.layout.pattern = [%d{yy-MMM-dd HH:mm:ss:SSS}] [%p] [%c{1}:%L] - %m%n
appender.stderr.target = SYSTEM_ERR
+appender.stderr.filter.threshold.type = ThresholdFilter
+appender.stderr.filter.threshold.level = INFO
+appender.stderr.filter.threshold.onMatch = ACCEPT
+appender.stderr.filter.threshold.onMismatch = DENY
+
# ${LOG_DIR}/request_errors/...log
appender.request_errors.type = RollingRandomAccessFile
diff --git a/conf/core/server.conf b/conf/core/server.conf
index 00b3b1a33dbab19f9931424cc91b5b11f9d137e3..c3cf62bafe6951f1b1412e648ea2eb10f469a2a4 100644
--- a/conf/core/server.conf
+++ b/conf/core/server.conf
@@ -26,10 +26,10 @@ CONTEXT_ROOT=
SERVER_PORT_HTTPS=443
SERVER_PORT_HTTP=80
-HTTPS_ENABLED_PROTOCOLS=TLSv1.2 TLSv1.1 TLSv1
-HTTPS_DISABLED_PROTOCOLS=SSLv3 SSLv2Hello
-HTTPS_ENABLED_CIPHER_SUITES=TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA TLS_RSA_WITH_AES_256_CBC_SHA TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA TLS_ECDH_RSA_WITH_AES_256_CBC_SHA TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA TLS_RSA_WITH_AES_128_CBC_SHA TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA TLS_ECDH_RSA_WITH_AES_128_CBC_SHA TLS_ECDHE_ECDSA_WITH_RC4_128_SHA TLS_ECDHE_RSA_WITH_RC4_128_SHA TLS_ECDH_ECDSA_WITH_RC4_128_SHA TLS_ECDH_RSA_WITH_RC4_128_SHA TLS_ECDHE_ECDSA_WITH_3DES_EDE_CBC_SHA TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA TLS_ECDH_ECDSA_WITH_3DES_EDE_CBC_SHA TLS_ECDH_RSA_WITH_3DES_EDE_CBC_SHA
-HTTPS_DISABLED_CIPHER_SUITES=SSL_RSA_WITH_RC4_128_MD5 SSL_RSA_WITH_3DES_EDE_CBC_SHA SSL_RSA_WITH_RC4_128_SHA TLS_DHE_RSA_WITH_AES_256_CBC_SHA TLS_DHE_DSS_WITH_AES_256_CBC_SHA TLS_DHE_RSA_WITH_AES_128_CBC_SHA TLS_DHE_DSS_WITH_AES_128_CBC_SHA SSL_DHE_RSA_WITH_3DES_EDE_CBC_SHA SSL_DHE_DSS_WITH_3DES_EDE_CBC_SHA
+HTTPS_ENABLED_PROTOCOLS=TLSv1.3 TLSv1.2
+HTTPS_DISABLED_PROTOCOLS=SSLv3 SSLv2Hello TLSv1.1 TLSv1.0
+HTTPS_ENABLED_CIPHER_SUITES=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 TLS_AES_128_GCM_SHA256 TLS_AES_256_GCM_SHA384 TLS_CHACHA20_POLY1305_SHA256 TLS_AES_128_CCM_SHA256 TLS_AES_128_CCM_8_SHA256 TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 TLS_DHE_RSA_WITH_AES_256_GCM_SHA384 TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384
+HTTPS_DISABLED_CIPHER_SUITES=TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA TLS_RSA_WITH_AES_256_CBC_SHA TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA TLS_ECDH_RSA_WITH_AES_256_CBC_SHA TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA TLS_RSA_WITH_AES_128_CBC_SHA TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA TLS_ECDH_RSA_WITH_AES_128_CBC_SHA TLS_ECDHE_ECDSA_WITH_RC4_128_SHA TLS_ECDHE_RSA_WITH_RC4_128_SHA TLS_ECDH_ECDSA_WITH_RC4_128_SHA TLS_ECDH_RSA_WITH_RC4_128_SHA TLS_ECDHE_ECDSA_WITH_3DES_EDE_CBC_SHA TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA TLS_ECDH_ECDSA_WITH_3DES_EDE_CBC_SHA TLS_ECDH_RSA_WITH_3DES_EDE_CBC_SHASSL_RSA_WITH_RC4_128_MD5 SSL_RSA_WITH_3DES_EDE_CBC_SHA SSL_RSA_WITH_RC4_128_SHA TLS_DHE_RSA_WITH_AES_256_CBC_SHA TLS_DHE_DSS_WITH_AES_256_CBC_SHA TLS_DHE_RSA_WITH_AES_128_CBC_SHA TLS_DHE_DSS_WITH_AES_128_CBC_SHA SSL_DHE_RSA_WITH_3DES_EDE_CBC_SHA SSL_DHE_DSS_WITH_3DES_EDE_CBC_SHA
INITIAL_CONNECTIONS=1
@@ -50,13 +50,6 @@ BUGTRACKER_URI=
TRANSACTION_BENCHMARK_ENABLED=true
CACHE_CONF_LOC=./conf/core/cache.ccf
-RULES_CACHE_CAPACITY=100
-SPARSE_ENTITY_CACHE_CAPACITY=1000
-PROPERTIES_CACHE_CAPACITY=1000
-PARENTS_CACHE_CAPACITY=1000
-USER_ACCOUNT_CACHE_CAPACITY=100
-GROUP_CACHE_CAPACITY=100
-
INSERT_FILES_IN_DIR_ALLOWED_DIRS=
SUDO_PASSWORD=
@@ -70,4 +63,4 @@ CERTIFICATES_KEY_PASSWORD=
CERTIFICATES_KEY_STORE_PATH=
CERTIFICATES_KEY_STORE_PASSWORD=
-WEBUI_HTTP_HEADER_CACHE_MAX_AGE=28800
\ No newline at end of file
+WEBUI_HTTP_HEADER_CACHE_MAX_AGE=28800
diff --git a/doc/devel/Benchmarking.md b/doc/devel/Benchmarking.md
new file mode 100644
index 0000000000000000000000000000000000000000..a244a3a64b771060fbf025fc0ee47054b9b95b48
--- /dev/null
+++ b/doc/devel/Benchmarking.md
@@ -0,0 +1,13 @@
+# Manual Java-Side Benchmarking #
+
+Benchmarking can be done using the `TransactionBenchmark` class (in package
+`caosdb.server.database.misc`).
+
+- Single timings can be added to instances of that class via the
+ `addBenchmark(object, time)` method. Multiple benchmarks for the same object
+ (typically just strings) can be averaged.
+- Benchmarks can be serialized into XML, `Container` and `Query` objects already
+ use this with their included benchmarks to output benchmarking results.
+- To work with the benchmarks of often used objects, use these methods:
+ - `Container.getTransactionBenchmark().addBenchmark()`
+ - `Query.addBenchmark()`
diff --git a/makefile b/makefile
index 688675fe7a1f73e113f9668b3772f470f365e4f1..ea03f993e5de1dbc19606e3dae4804b0e4c6bf7a 100644
--- a/makefile
+++ b/makefile
@@ -4,6 +4,8 @@
#
# Copyright (C) 2018 Research Group Biomedical Physics,
# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+# Copyright (C) 2019 IndiScale GmbH
+# Copyright (C) 2019 Timm Fitschen (t.fitschen@indiscale.com)
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as
@@ -22,6 +24,7 @@
#
SHELL:=/bin/bash
+JPDA_PORT ?= 9000
compile: easy-units
mvn compile
@@ -33,7 +36,7 @@ run: compile
mvn exec:java@run
run-debug: jar
- java -Dcaosdb.debug=true -jar target/caosdb-server-0.1-SNAPSHOT-jar-with-dependencies.jar
+ java -Xrunjdwp:transport=dt_socket,address=0.0.0.0:$(JPDA_PORT),server=y,suspend=n -Dcaosdb.debug=true -jar target/caosdb-server-0.1-SNAPSHOT-jar-with-dependencies.jar
run-single:
java -jar target/caosdb-server-0.1-SNAPSHOT-jar-with-dependencies.jar
@@ -42,15 +45,18 @@ formatting:
mvn fmt:format
# Compile into a standalone jar file
-jar: compile
- mvn clean compile assembly:single
-# mvn assembly:single
+jar: easy-units
+ mvn package -DskipTests
antlr:
mvn antlr4:antlr4
test: easy-units
- mvn test
+ MAVEN_DEBUG_OPTS="-Xdebug -Xnoagent -Djava.compiler=NONE -Dcaosdb.debug=true -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=0.0.0.0:9000"
+ mvn test -X
+
+test_misc:
+ cd misc/bend_symlinks/ && /bin/bash -c /usr/bin/shunit2 test/test_suite.sh
clean: clean-antlr
mvn clean
@@ -105,4 +111,5 @@ stop-debug-screen:
mkdir .m2-local
easy-units: .m2-local
+ mvn clean
mvn deploy:deploy-file -DgroupId=de.timmfitschen -DartifactId=easy-units -Dversion=0.0.1-SNAPSHOT -Durl=file:./.m2-local/ -DrepositoryId=local-maven-repo -DupdateReleaseInfo=true -Dfile=./lib/easy-units-0.0.1-SNAPSHOT-jar-with-dependencies.jar
diff --git a/misc/bend_symlinks/README.md b/misc/bend_symlinks/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..f72357f8636a2cf961434108a52e98ef05181e62
--- /dev/null
+++ b/misc/bend_symlinks/README.md
@@ -0,0 +1,106 @@
+# About
+
+./bend_symlinks.sh - fix broken symlinks in the internal file system
+
+# Copyright and License Disclaimer:
+
+ Copyright (C) 2019 Timm Fitschen (t.fitschen@indiscale.com)
+ Copyright (C) 2019 IndiScale (info@indiscale.com)
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as
+ published by the Free Software Foundation, either version 3 of the
+ License, or (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+
+# Version
+ 0.1
+
+# Usage
+
+ ./bend_symlinks.sh [-D] FILE_SYSTEM_DIR TARGET_PATTERN TARGET_REPLACEMENT
+
+ Find all broken symlinks below FILE_SYSTEM_DIR which targets match
+ TARGET_PATTERN and replace the targets (i.e. overwrite the symlinks)
+ with TARGET_REPLACEMENT according to the rules of sed's replace
+ function.
+
+ Print OLD_TARGET<tab>NEW_TARGET to stdout for each replaced symlink.
+
+ ./bend_symlinks.sh -d [-D] FILE_SYSTEM_DIR OLD_TARGET_PREFIX NEW_TARGET_PREFIX
+
+ Find all broken symlinks below FILE_SYSTEM_DIR which targets a
+ path prefixed by OLD_TARGET_PREFIX and replace the targets (i.e.
+ overwrite the symlinks) by changing only the directory prefix to
+ NEW_TARGET_PREFIX. This is the preferred way to fix symlinks which
+ targets have just been moved to another directory while the structure
+ under this directory stayed the same.
+
+ Print OLD_TARGET<tab>NEW_TARGET to stdout for each replaced symlink.
+
+ ./bend_symlinks.sh (-h|-v)
+
+# Parameters
+
+ FILE_SYSTEM_DIR A directory of the internal file system's back-end
+ storage.
+ All symlinks below this directory are being process by
+ this script.
+ E.g. '/mnt/caosdb_fs/ExperimentalData/'
+ TARGET_PATTERN A (extended sed-style) regular expression for matching
+ broken symlink targets.
+ TARGET_REPLACEMENT A (sed-style) replacement string for the new (fixed)
+ symlink targets.
+ OLD_TARGET_PREFIX The directory of old and broken symlink targets.
+ E.g. '/mnt/data/current/experiments/'
+ NEW_TARGET_PREFIX The directory of the new symlink targets.
+ E.g. '/mnt/data/archive/experiments/2019/'
+ -d Bend all symlinks under FILE_SYSTEM_DIR which target a
+ file prefixed by OLD_TARGET_PREFIX to point to
+ NEW_TARGET_PREFIX and keep the substructure the same.
+ This is the most useful special case for the scenario
+ where orginal data files have just been moved from one
+ folder into another and the symlinks need to updated
+ accordingly.
+ -D Dry-run: Only print what would happen.
+ -h Print this help message and exit.
+ -v Print the version of this script and ext.
+
+# Examples
+
+ 1. Files have been moved from '/mnt/data/current/experiments/' to
+ /mnt/data/archive/experiments/2019/'. Execute the script in the root
+ directory of the caosdb server's internal file system:
+
+ $ ./bend_symlinks.sh -d ./ /mnt/data/current/experiments /mnt/data/archive/experiments/2019
+
+ 2. A File was renamed from '/mnt/data/procotol.pdf' to
+ '/mnt/data/protocol.pdf'. The symlink is located at
+ '/mnt/caosdb_fs/procotol.pdf'. Execute the script in the root directory
+ of the caosdb server's internal file system:
+
+ $ ./bend_symlinks.sh ./ procotol\.pdf$ protocol.pdf
+
+ 3. In order to print a table which contains the corrected name from example 2
+ and which can be understood by the the
+ [move_files.py](../move_files/move_files.py) script pipe the standard
+ output like this.
+
+ $ ./bend_symlinks.sh ./ procotol\.pdf$ protocol.pdf | sed -e 's/\/mnt\/data// > changes.tsv
+
+ Then the changes.tsv file contains 'procotol.pdf<tab>protocol.pdf<EOF>'.
+
+
+# Tests
+
+Run test suite with
+
+ $ shunit2 test/test_suite.sh
diff --git a/misc/bend_symlinks/bend_symlinks.sh b/misc/bend_symlinks/bend_symlinks.sh
new file mode 100755
index 0000000000000000000000000000000000000000..5a296fcdf7c447619751b958b6dc411273e9c951
--- /dev/null
+++ b/misc/bend_symlinks/bend_symlinks.sh
@@ -0,0 +1,132 @@
+#!/bin/bash
+#
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2019 Timm Fitschen (t.fitschen@indiscale.com)
+# Copyright (C) 2019 IndiScale (info@indiscale.com)
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+
+set -o errexit -o noclobber -o nounset -o pipefail
+
+VERSION=0.1
+
+LICENCE="# Copyright and License Disclaimer:
+
+ Copyright (C) 2019 Timm Fitschen (t.fitschen@indiscale.com)
+ Copyright (C) 2019 IndiScale (info@indiscale.com)
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as
+ published by the Free Software Foundation, either version 3 of the
+ License, or (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with this program. If not, see <https://www.gnu.org/licenses/>.
+"
+
+USAGE="# Usage
+
+ $0 [-D] FILE_SYSTEM_DIR TARGET_PATTERN TARGET_REPLACEMENT
+
+ Find all broken symlinks below FILE_SYSTEM_DIR which targets match
+ TARGET_PATTERN and replace the targets (i.e. overwrite the symlinks)
+ with TARGET_REPLACEMENT according to the rules of sed's replace
+ function.
+
+ Print OLD_TARGET<tab>NEW_TARGET to stdout for each replaced symlink.
+
+ $0 -d [-D] FILE_SYSTEM_DIR OLD_TARGET_PREFIX NEW_TARGET_PREFIX
+
+ Find all broken symlinks below FILE_SYSTEM_DIR which targets a
+ path prefixed by OLD_TARGET_PREFIX and replace the targets (i.e.
+ overwrite the symlinks) by changing only the directory prefix to
+ NEW_TARGET_PREFIX. This is the preferred way to fix symlinks which
+ targets have just been moved to another directory while the structure
+ under this directory stayed the same.
+
+ Print OLD_TARGET<tab>NEW_TARGET to stdout for each replaced symlink.
+
+ $0 (-h|-v)
+
+# Parameters
+
+ FILE_SYSTEM_DIR A directory of the internal file system's back-end
+ storage.
+ All symlinks below this directory are being process by
+ this script.
+ E.g. '/mnt/caosdb_fs/ExperimentalData/'
+ TARGET_PATTERN A (extended sed-style) regular expression for matching
+ broken symlink targets.
+ TARGET_REPLACEMENT A (sed-style) replacement string for the new (fixed)
+ symlink targets.
+ OLD_TARGET_PREFIX The directory of old and broken symlink targets.
+ E.g. '/mnt/data/current/experiments/'
+ NEW_TARGET_PREFIX The directory of the new symlink targets.
+ E.g. '/mnt/data/archive/experiments/2019/'
+ -d Bend all symlinks under FILE_SYSTEM_DIR which target a
+ file prefixed by OLD_TARGET_PREFIX to point to
+ NEW_TARGET_PREFIX and keep the substructure the same.
+ This is the most useful special case for the scenario
+ where orginal data files have just been moved from one
+ folder into another and the symlinks need to updated
+ accordingly.
+ -D Dry-run: Only print what would happen.
+ -h Print this help message and exit.
+ -v Print the version of this script and ext.
+
+# Examples
+
+ 1. Files have been moved from '/mnt/data/current/experiments/' to
+ /mnt/data/archive/experiments/2019/'. Execute the script in the root
+ directory of the caosdb server's internal file system:
+
+ $ $0 -d ./ /mnt/data/current/experiments /mnt/data/archive/experiments/2019
+
+ 2. A File was renamed from '/mnt/data/procotol.pdf' to
+ '/mnt/data/protocol.pdf'. The symlink is located at
+ '/mnt/caosdb_fs/procotol.pdf'. Execute the script in the root directory
+ of the caosdb server's internal file system:
+
+ $ $0 ./ procotol\\.pdf$ protocol.pdf
+
+ 3. In order to print a table which contains the corrected name from example 2
+ and which can be understood by the the
+ [move_files.py](../move_files/move_files.py) script pipe the standard
+ output like this.
+
+ $ $0 ./ procotol\\.pdf$ protocol.pdf | sed -e 's/\/mnt\/data// > changes.tsv
+
+ Then the changes.tsv file contains 'procotol.pdf<tab>protocol.pdf<EOF>'.
+"
+
+HELP="$0 - fix broken symlinks in the internal file system
+
+$LICENCE
+
+# Version
+ $VERSION
+
+$USAGE
+"
+
+source src/main.sh
diff --git a/misc/bend_symlinks/src/main.sh b/misc/bend_symlinks/src/main.sh
new file mode 100644
index 0000000000000000000000000000000000000000..c2a6a94766d437e41619c2602d1c62417e09ee42
--- /dev/null
+++ b/misc/bend_symlinks/src/main.sh
@@ -0,0 +1,98 @@
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2019 Timm Fitschen (t.fitschen@indiscale.com)
+# Copyright (C) 2019 IndiScale (info@indiscale.com)
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+
+source "src/utils.sh"
+
+### PARSE COMMAND LINE ARGUMENTS ###
+
+# OPTIONS -v -h
+IS_DRY_RUN=0
+IS_MOVE=0
+while getopts ":hvdD" opt; do
+ case ${opt} in
+ h )
+ echo "$HELP"
+ exit 0
+ ;;
+ v )
+ echo "$VERSION"
+ exit 0
+ ;;
+ d )
+ IS_MOVE=1
+ ;;
+ D )
+ IS_DRY_RUN=1
+ ;;
+ \? )
+ echo "Invalid option. See '$0 -h' for more information" 1>&2
+ exit 1
+ ;;
+ esac
+done
+shift $((OPTIND -1))
+
+
+# POSTIONAL ARGUMENTS
+if [ $# -ne 3 ] ; then
+ echo "Illegal number of positional parameters. See '$0 -h' for more information" 1>&2
+ exit 1
+fi
+FILE_SYSTEM_ROOT=$1
+REGEX_OLD=$2
+REPLACEMENT=$3
+
+
+if [ $IS_MOVE -eq 1 ] ; then
+ REGEX_OLD=$(old_dir "$REGEX_OLD")
+ REPLACEMENT=$(new_dir "$REPLACEMENT")
+fi
+
+set -o noglob
+for syml in $(find -P $(realpath $FILE_SYSTEM_ROOT) -type l) ; do
+ OLD_TARGET=$(realpath -m "$syml" | sed -n -r "/$REGEX_OLD/p")
+ if [ -z "$OLD_TARGET" ] ; then
+ # filter non matching
+ continue
+ fi
+
+ if [ -e "$syml" ] ; then
+ echo "#IGNORING (not broken): $syml" 1>&2
+ continue
+ fi
+
+ NEW_TARGET=$(echo "$OLD_TARGET" | sed -r "s/$REGEX_OLD/$REPLACEMENT/g")
+ if [ ! -e "$NEW_TARGET" ] ; then
+ echo "#IGNORING (broken new): $NEW_TARGET" 1>&2
+ continue
+ fi
+
+ echo -e "$OLD_TARGET\t$NEW_TARGET"
+ if [ $IS_DRY_RUN -eq 1 ] ; then
+ continue
+ fi
+
+ # -f means force overwriting
+ ln -fs "$NEW_TARGET" "$syml" ;
+
+done
+set +o noglob
+
diff --git a/misc/bend_symlinks/src/utils.sh b/misc/bend_symlinks/src/utils.sh
new file mode 100644
index 0000000000000000000000000000000000000000..0e10fe9acc0c1c27fa3f2f58add8e7daf845e24c
--- /dev/null
+++ b/misc/bend_symlinks/src/utils.sh
@@ -0,0 +1,58 @@
+#!/bin/bash
+#
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2019-2020 Timm Fitschen (t.fitschen@indiscale.com)
+# Copyright (C) 2019-2020 IndiScale GmbH (info@indiscale.com)
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+set -o errexit -o noclobber -o nounset -o pipefail
+
+function escape_simple_path () {
+ SPATH=$(escape_slash "$1")
+ # ?
+ SPATH=$(echo "$SPATH" | sed "s/\?/\\?/g")
+ # .
+ SPATH=$(echo "$SPATH" | sed "s/\./\\\\./g")
+ # $
+ SPATH=$(echo "$SPATH" | sed "s/\\$/\\\\$/g")
+ # [
+ SPATH=$(echo "$SPATH" | sed -r "s/\[/\\\\[/g")
+ # (
+ SPATH=$(echo "$SPATH" | sed -r "s/\(/\\\\(/g")
+ # {
+ SPATH=$(echo "$SPATH" | sed -r "s/\{/\\\\{/g")
+ echo "$SPATH"
+}
+
+function escape_slash () {
+ echo "${1//\//\\\/}"
+}
+
+
+function old_dir () {
+ OLD_DIR=$(realpath -m "$1")
+ OLD_DIR=$(escape_simple_path "$OLD_DIR")
+ echo "^$OLD_DIR\/(.*)$"
+}
+
+function new_dir () {
+ NEW_DIR=$(realpath -m "$1")
+ NEW_DIR=$(escape_slash "$NEW_DIR")
+ echo "$NEW_DIR\/\1"
+}
diff --git a/misc/bend_symlinks/test/test_suite.sh b/misc/bend_symlinks/test/test_suite.sh
new file mode 100644
index 0000000000000000000000000000000000000000..781e477e1ffacda36b12687ec05ebf5c9d021e15
--- /dev/null
+++ b/misc/bend_symlinks/test/test_suite.sh
@@ -0,0 +1,229 @@
+
+
+source "./src/utils.sh"
+set +o errexit
+
+BEND=./bend_symlinks.sh
+FILE_SYSTEM_ROOT=test_dir/links
+DATA_DIR=test_dir/original
+
+oneTimeSetUp () {
+ mkdir -p $FILE_SYSTEM_ROOT $DATA_DIR
+}
+
+oneTimeTearDown () {
+ rm -rf test_dir
+}
+
+tearDown () {
+ rm -rf $FILE_SYSTEM_ROOT/*
+ rm -rf $DATA_DIR/*
+}
+
+_make_test_file () {
+ touch "$DATA_DIR/$1"
+ ln -s $(realpath "$DATA_DIR/$1") "$FILE_SYSTEM_ROOT/$1"
+ assertEquals "initial target $1" $(realpath "$FILE_SYSTEM_ROOT/$1") $(realpath "$DATA_DIR/$1")
+}
+
+_break_link_move_file () {
+ set -o noglob
+ OLD_PATH="$DATA_DIR/$1"
+ OLD_PATH_REAL=$(realpath "$OLD_PATH")
+ NEW_PATH="$DATA_DIR/$2"
+ NEW_PATH_REAL=$(realpath "$NEW_PATH")
+ LINK="$FILE_SYSTEM_ROOT/$1"
+ mv "$OLD_PATH_REAL" "$NEW_PATH_REAL"
+ assertEquals "still target $OLD_PATH_REAL" $(realpath "$LINK") "$OLD_PATH_REAL"
+ assertFalse "$LINK link is broken" "[ -f '$LINK' ]"
+ assertFalse "$OLD_PATH_REAL was moved" "[ -f '$OLD_PATH_REAL' ]"
+ assertTrue "$NEW_PATH_REAL is there" "[ -f '$NEW_PATH_REAL' ]"
+ set +o noglob
+}
+
+testVersion () {
+ assertEquals "version 0.1" "0.1" "$($BEND -v)"
+}
+
+assertLinkOk () {
+ set -o noglob
+ LINK=$(realpath "$FILE_SYSTEM_ROOT/$1")
+ TARGET=$(realpath "$DATA_DIR/$2")
+ assertTrue "target exists $LINK" "[ -f '$LINK' ]"
+ assertEquals "target matches $TARGET" $TARGET "$LINK"
+ set +o noglob
+}
+
+testIgnoreUnbroken () {
+ _make_test_file "fileA"
+ RESULTS=$($BEND $FILE_SYSTEM_ROOT "fileA" "fileA.new" 2>&1) # attempt to rename
+
+ assertEquals "ignoring not broken" "#IGNORING (not broken): $(realpath "$PWD")/test_dir/links/fileA" "$RESULTS"
+
+ assertLinkOk "fileA" "fileA"
+}
+
+testIgnoreMissingNew () {
+ _make_test_file "fileA"
+
+ _break_link_move_file "fileA" "fileA.new"
+
+ RESULTS=$($BEND $FILE_SYSTEM_ROOT "fileA" "fileA.non" 2>&1)
+
+ assertEquals "ignoring broken new" "#IGNORING (broken new): $(realpath $DATA_DIR)/fileA.non" "$RESULTS"
+
+ TARGET=$(realpath -m "$DATA_DIR/fileA")
+ LINK=$(realpath -m "$FILE_SYSTEM_ROOT/fileA")
+ assertFalse "symlink still broken" "[ -e '$LINK' ]"
+ assertEquals "target still old" "$TARGET" "$LINK"
+}
+
+
+testFileName () {
+ _make_test_file "fileA"
+ _make_test_file "fileB"
+
+ _break_link_move_file "fileA" "fileA.new"
+
+ RESULTS=$($BEND $FILE_SYSTEM_ROOT "fileA" "fileA.new" 2>&1) # rename all fileA to fileA.new
+
+ assertLinkOk "fileA" "fileA.new"
+ assertLinkOk "fileB" "fileB"
+}
+
+testFullPath () {
+ _make_test_file "fileA"
+ _make_test_file "fileB"
+
+ _break_link_move_file "fileA" "fileA.new"
+
+ REGEX_OLD=$(escape_simple_path "$DATA_DIR/fileA")
+ REPLACEMENT=$(escape_slash "$DATA_DIR/fileA.new")
+ RESULTS=$($BEND $FILE_SYSTEM_ROOT "$REGEX_OLD" "$REPLACEMENT" 2>&1)
+
+ assertLinkOk "fileA" "fileA.new"
+ assertLinkOk "fileB" "fileB"
+}
+
+_testFullPathWithStrageChars () {
+ file_name="fileA$1bla"
+ _make_test_file "$file_name"
+
+ _break_link_move_file "$file_name" "$file_name.new"
+
+ REGEX_OLD=$(escape_simple_path "$DATA_DIR/$file_name")
+ REPLACEMENT=$(escape_slash "$DATA_DIR/$file_name.new")
+ RESULTS=$($BEND "$FILE_SYSTEM_ROOT" "$REGEX_OLD" "$REPLACEMENT" 2>&1)
+
+ assertLinkOk "$file_name" "$file_name.new"
+ assertLinkOk "fileB" "fileB"
+}
+
+testFullPathWithStrangeChars () {
+ _make_test_file "fileB"
+ _testFullPathWithStrageChars "A"
+ _testFullPathWithStrageChars "#"
+ _testFullPathWithStrageChars "0"
+ _testFullPathWithStrageChars "!"
+ _testFullPathWithStrageChars "."
+ _testFullPathWithStrageChars ";"
+ _testFullPathWithStrageChars ","
+ _testFullPathWithStrageChars "$"
+ _testFullPathWithStrageChars "["
+ _testFullPathWithStrageChars "("
+ _testFullPathWithStrageChars "{"
+ _testFullPathWithStrageChars "]"
+ _testFullPathWithStrageChars "[.]"
+}
+
+testRegex () {
+ _make_test_file "fileA.0"
+ _make_test_file "dataA.1"
+ _make_test_file "fileA.0ok"
+ _make_test_file "fileB"
+
+ _break_link_move_file "fileA.0" "file0-A"
+ _break_link_move_file "dataA.1" "data1-A"
+
+ RESULTS=$($BEND "$FILE_SYSTEM_ROOT" "([a-z]+)([A-Z])+\.([01])$" "\1\3-\2" 2>&1)
+
+ assertLinkOk "fileA.0" "file0-A"
+ assertLinkOk "dataA.1" "data1-A"
+ assertLinkOk "fileB" "fileB"
+ assertLinkOk "fileA.0ok" "fileA.0ok"
+}
+
+
+testDryRun () {
+ _make_test_file "fileA"
+
+ OLD_TARGET=$(realpath "$DATA_DIR/fileA")
+ NEW_TARGET=$(realpath "$DATA_DIR/fileA.new")
+
+ SYMLINK=$(realpath "$FILE_SYSTEM_ROOT/fileA")
+ assertTrue "1 target exists $SYMLINK" "[ -f '$SYMLINK' ]"
+ assertEquals "1 target matches $OLD_TARGET" "$OLD_TARGET" "$SYMLINK"
+
+ _break_link_move_file "fileA" "fileA.new"
+
+ SYMLINK=$(realpath "$FILE_SYSTEM_ROOT/fileA")
+ assertFalse "2 target does not exist $SYMLINK" "[ -f '$SYMLINK' ]"
+ assertEquals "2 target matches $OLD_TARGET" "$OLD_TARGET" "$SYMLINK"
+
+ RESULTS=$($BEND -D "$FILE_SYSTEM_ROOT" "fileA" "fileA.new" 2>&1)
+
+ SYMLINK=$(realpath "$FILE_SYSTEM_ROOT/fileA")
+ assertFalse "3 target does not exist $SYMLINK" "[ -f '$SYMLINK' ]"
+ assertEquals "3 target matches $OLD_TARGET" $OLD_TARGET "$SYMLINK"
+
+ RESULTS=$($BEND "$FILE_SYSTEM_ROOT" "fileA" "fileA.new")
+
+ SYMLINK=$(realpath "$FILE_SYSTEM_ROOT/fileA")
+ assertTrue "4 target exists $SYMLINK" "[ -f '$SYMLINK' ]"
+ assertEquals "4 target matches $NEW_TARGET" $NEW_TARGET "$SYMLINK"
+
+}
+
+testUtilsOldAndNewDir () {
+ OLD_DIR="/root/to/old/"
+ assertTrue "old root does not exist" "[ ! -e '$OLD_DIR' ]"
+ NEW_DIR=$(realpath -m "$FILE_SYSTEM_ROOT/root/to/new/")
+ # new_dir must exist
+ mkdir -p "$NEW_DIR"
+
+ OLD_DIR=$(old_dir "$OLD_DIR")
+ NEW_DIR=$(new_dir "$NEW_DIR")
+ PWD_ESC=$(escape_simple_path $(realpath "$FILE_SYSTEM_ROOT"))
+
+ assertEquals "OLD_DIR correct" "^\/root\/to\/old\/(.*)$" "$OLD_DIR"
+ assertEquals "NEW_DIR correct" "$PWD_ESC\/root\/to\/new\/\1" "$NEW_DIR"
+
+ RESULT=$(echo "/root/to/old/subdir/fileA" | sed -r "s/$OLD_DIR/$NEW_DIR/g")
+ assertEquals "result" $(realpath -m "$FILE_SYSTEM_ROOT/root/to/new/subdir/fileA") "$RESULT"
+}
+
+
+testSymlinkToSymlink () {
+ touch "$DATA_DIR/fileA"
+ ln -s $(realpath "$DATA_DIR/fileA") "$DATA_DIR/symlinkA"
+ ln -s $(realpath -s "$DATA_DIR/symlinkA") "$FILE_SYSTEM_ROOT/symlinkA"
+ assertLinkOk "symlinkA" "symlinkA"
+ assertLinkOk "symlinkA" "fileA"
+
+
+ # move only the symlink in data_dir
+ mv $DATA_DIR/symlinkA $DATA_DIR/symlinkA.new
+ assertFalse "symlink in fs broken" "[ -e '$FILE_SYSTEM_ROOT/symlinkA' ]"
+ assertTrue "symlink in data ok ($TARGET)" "[ -e '$DATA_DIR/symlinkA.new' ]"
+
+ RESULT=$($BEND "$FILE_SYSTEM_ROOT" "symlinkA" "symlinkA.new" 2>&1)
+
+ assertTrue "simlink is fixed" "[ -e '$FILE_SYSTEM_ROOT/symlinkA' ]"
+
+
+ assertLinkOk "symlinkA" "symlinkA.new"
+ assertLinkOk "symlinkA" "fileA"
+
+}
+
+
diff --git a/misc/check_symlinks/check_symlinks b/misc/check_symlinks/check_symlinks
index 5545613982e7772189697eb596d9786af0d29b1b..0f6eb0d1d3626ed32fa99aa6a31891c6ddf90635 100755
--- a/misc/check_symlinks/check_symlinks
+++ b/misc/check_symlinks/check_symlinks
@@ -77,7 +77,6 @@ function check_recursively() {
else
echo "$subdir is empty"
fi
-
done
}
diff --git a/misc/move_files/README.md b/misc/move_files/README.md
new file mode 100644
index 0000000000000000000000000000000000000000..df67f2025b8627a09e6886038646db4ec4436941
--- /dev/null
+++ b/misc/move_files/README.md
@@ -0,0 +1,24 @@
+# About
+
+Version: 0.1
+
+Usage: `./move_files.py [-h] changes`
+
+This script moves files in the internal file system of the CaosDB server. It
+reads file paths form a tsv file with columns from and to. For each line it
+creates an update of a caosdb file object where the path that equals "from" is
+changed to "to".
+
+positional arguments:
+ changes The file that defines the renames
+
+optional arguments:
+ -h, --help show this help message and exit
+
+# Tests
+
+The tests a integration tests which require a running test database and a sufficiently configured caosdb-pylib.
+
+Run `pytest test_move_files.py` to insert a bunch of test files, rename and subsequently delete them.
+
+
diff --git a/misc/move_files/move_files.py b/misc/move_files/move_files.py
new file mode 100755
index 0000000000000000000000000000000000000000..44cc3f4f7e61027b833a7774705577c59e756e72
--- /dev/null
+++ b/misc/move_files/move_files.py
@@ -0,0 +1,101 @@
+#!/usr/bin/env python3
+# encoding: utf-8
+#
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2019 Henrik tom Wörden
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+
+"""
+This script moves files in the internal file system of the CaosDB server. It
+reads file paths form a tsv file with columns from and to. For each line it
+creates an update of a caosdb file object where the path that equals "from" is
+changed to "to".
+
+Version: 0.1
+"""
+
+import argparse
+import sys
+import time
+from argparse import ArgumentParser
+
+import pandas as pd
+from tqdm import tqdm
+
+import caosdb as db
+
+
+def rename(changes, chunksize=10):
+ """change the path of files based on a two-column table (from, to).
+
+ Parameters
+ ----------
+
+ changes : pd.DataFrame
+ A table with two columns, the old path and the new path.
+ chunksize : int, optional
+ How many files are being moved in one go (default is 10).
+ """
+ i = 0
+
+ for i in tqdm(range(changes.shape[0]//chunksize+1)):
+ chunk = changes.iloc[i*chunksize:(i+1)*chunksize]
+
+ if chunk.shape[0] == 0:
+ continue
+ cont = db.Container()
+
+ for _, (old, new) in chunk.iterrows():
+ cont.append(db.File(path=old))
+
+ cont.retrieve()
+
+ for fi, (_, (old, new)) in zip(cont, chunk.iterrows()):
+ assert fi.path == old
+ fi.path = new
+ cont.update()
+ i += 1
+
+
+def main(argv=None):
+ '''Command line options.'''
+
+ if argv is None:
+ argv = sys.argv
+ else:
+ sys.argv.extend(argv)
+
+ # Setup argument parser
+ parser = ArgumentParser(description=__doc__)
+ parser.add_argument("changes", help="The file that defines the renames")
+ args = parser.parse_args()
+
+ changes = pd.read_csv(args.changes, sep="\t")
+
+ if ("to" not in changes.columns or "from" not in changes.columns):
+ raise ValueError("The file supplied under changes shall have a 'to'"
+ " and a 'from' column.")
+
+ assert 0 == pd.isnull(changes).sum().sum()
+ rename(changes)
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/misc/move_files/test_move_files.py b/misc/move_files/test_move_files.py
new file mode 100644
index 0000000000000000000000000000000000000000..2edb76509dc6a350057997cc6f894abd4a4bb163
--- /dev/null
+++ b/misc/move_files/test_move_files.py
@@ -0,0 +1,78 @@
+# encoding: utf-8
+#
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2019 Henrik tom Wörden
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+
+import os
+import random
+import unittest
+
+import pandas as pd
+
+import caosdb as db
+from move_files import rename
+
+
+def create_filename():
+ name = os.path.normpath("".join([
+ random.choice("qwertyuiopkkhgfdsxcvbnm/") for el in range(30)]))
+
+ if not name.startswith("/"):
+ name = "/" + name
+
+ return name
+
+
+class TestMoveFiles(unittest.TestCase):
+ """
+ Files are being created, changed and then it is checked whether the changes
+ were correct.
+ """
+
+ def setUp(self):
+ self.files_to_be_changed = [create_filename() for i in range(40)]
+ self.files_not_to_be_changed = [create_filename() for i in range(20)]
+ self.new_names = [f+"new" for f in self.files_to_be_changed]
+
+ table = pd.DataFrame([self.files_to_be_changed, self.new_names])
+ table = table.T
+ table.columns = ["from", "to"]
+ self.table = table
+ self.cont = db.Container()
+ self.cont.extend([db.File(path=f, file=__file__)
+ for f in self.files_to_be_changed
+ + self.files_not_to_be_changed])
+ self.cont.insert()
+ print("inserted")
+
+ def test_move(self):
+ rename(self.table)
+ self.cont.retrieve()
+
+ for i, (fi, name) in enumerate(zip(
+ self.cont,
+ self.files_to_be_changed + self.files_not_to_be_changed)):
+
+ if i < len(self.files_to_be_changed):
+ self.assertEqual(fi.path, self.new_names[i])
+ else:
+ self.assertEqual(fi.path, name)
+
+ def tearDown(self):
+ self.cont.delete()
+ print("deleted")
diff --git a/misc/mv_unknown_files_script/mv_unknown_files b/misc/mv_unknown_files_script/mv_unknown_files
deleted file mode 100755
index 8b26b464d8dd40a15f3418895dd440903a6da8a6..0000000000000000000000000000000000000000
--- a/misc/mv_unknown_files_script/mv_unknown_files
+++ /dev/null
@@ -1,27 +0,0 @@
-#!/bin/bash
-#
-# ** header v3.0
-# This file is a part of the CaosDB Project.
-#
-# Copyright (C) 2018 Research Group Biomedical Physics,
-# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <https://www.gnu.org/licenses/>.
-#
-# ** end header
-#
-
-grep ": Unknown file." $1 | sed 's/<Warning.*tion="//' | sed 's/:\sUnknown.*//' | awk '{ print "move ", $1 }'
-
-
diff --git a/pom.xml b/pom.xml
index 42e44ca49b82963a16ac655755ccf29b8fcde6e0..ada3403287a657f7684ba8984c57cc80a51820a9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -49,16 +49,6 @@
</repository>
</repositories>
<dependencies>
- <dependency>
- <groupId>org.apache.logging.log4j</groupId>
- <artifactId>log4j-slf4j18-impl</artifactId>
- <version>2.11.2</version>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- <version>1.8.0-beta4</version>
- </dependency>
<dependency>
<groupId>de.timmfitschen</groupId>
<artifactId>easy-units</artifactId>
@@ -67,7 +57,7 @@
<dependency>
<groupId>org.apache.shiro</groupId>
<artifactId>shiro-core</artifactId>
- <version>1.3.2</version>
+ <version>1.4.1</version>
</dependency>
<dependency>
<groupId>junit</groupId>
@@ -162,15 +152,25 @@
<artifactId>jetty-util-ajax</artifactId>
<version>9.2.14.v20151106</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j-impl</artifactId>
+ <version>2.11.1</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>1.7.21</version>
+ </dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
- <version>2.11.2</version>
+ <version>2.11.1</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-core</artifactId>
- <version>2.11.2</version>
+ <version>2.11.1</version>
</dependency>
</dependencies>
<build>
@@ -180,6 +180,35 @@
<outputDirectory>${basedir}/target/classes</outputDirectory>
<testOutputDirectory>${basedir}/target/test-classes</testOutputDirectory>
<plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <version>2.4.3</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <finalName>${project.artifactId}-${project.version}-jar-with-dependencies</finalName>
+ <transformers>
+ <transformer implementation="com.github.edwgiz.mavenShadePlugin.log4j2CacheTransformer.PluginsCacheFileTransformer"></transformer>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <mainClass>caosdb.server.CaosDBServer</mainClass>
+ </transformer>
+ </transformers>
+ </configuration>
+ </execution>
+ </executions>
+ <dependencies>
+ <dependency>
+ <groupId>com.github.edwgiz</groupId>
+ <artifactId>maven-shade-plugin.log4j2-cachefile-transformer</artifactId>
+ <version>2.8.1</version>
+ </dependency>
+ </dependencies>
+ </plugin>
<plugin>
<groupId>org.antlr</groupId>
<artifactId>antlr4-maven-plugin</artifactId>
@@ -204,27 +233,6 @@
<target>1.8</target>
</configuration>
</plugin>
- <plugin>
- <artifactId>maven-assembly-plugin</artifactId>
- <configuration>
- <descriptorRefs>
- <descriptorRef>jar-with-dependencies</descriptorRef>
- </descriptorRefs>
- <archive>
- <manifest>
- <mainClass>caosdb.server.CaosDBServer</mainClass>
- </manifest>
- </archive>
- </configuration>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>single</goal> <!-- Run with: mvn assembly:single -->
- </goals>
- </execution>
- </executions>
- </plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
@@ -238,8 +246,8 @@
<log4j2.debug>true</log4j2.debug>
</systemPropertyVariables>
<reuseForks>false</reuseForks>
- <!-- Start 0.5 JVMs per CPU core -->
- <!-- Higher numbers *seem* to lead to higher failure rates... :-/ -->
+ <!-- Start 0.5 JVMs per CPU core
+ Higher numbers *seem* to lead to higher failure rates... :-/ -->
<forkCount>0.5C</forkCount>
</configuration>
</plugin>
@@ -298,13 +306,11 @@
</execution>
</executions>
</plugin>
- <!-- Remove easy-units from classpath generation because of
- https://github.com/jdee-emacs/jdee/issues/125 (no sources inside
- easy-units) -->
+ <!-- Remove easy-units from classpath generation because of https://github.com/jdee-emacs/jdee/issues/125
+ (no sources inside easy-units) -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
- <version>2.8</version>
<configuration>
<excludeArtifactIds>easy-units</excludeArtifactIds>
</configuration>
diff --git a/scripting/bin/xls_from_csv.py b/scripting/bin/xls_from_csv.py
index b64173a34be8d00bcd40e225122d865768b77ebb..38dfbc1c9392ce60d338b102f9b9fd05309919d3 100755
--- a/scripting/bin/xls_from_csv.py
+++ b/scripting/bin/xls_from_csv.py
@@ -31,27 +31,18 @@ import sys
import pandas as pd
-def _parse_to_dataframe(csv_string):
- """Attempts to create a valid dataframe from a CSV string.
+def _parse_to_dataframe(tsv_file):
+ """Attempts to create a valid dataframe from a TSV file.
-The CSV string typically starts with a header like this:
-
-```
-data:text/csv;charset=utf-8,colname1 colname2
-value1 value2
-...
-```
Parameters
----------
-csv_string : The URL encoded CSV content, starts with `data:text/csv`.
+tsv_file : path to a tsv file.
Returns
-------
out : The created dataframe.
"""
- csv_string = csv_string.split(",")[1]
- sio = io.StringIO(csv_string)
- dataframe = pd.read_csv(sio, sep="\t")
+ dataframe = pd.read_csv(tsv_file, sep="\t")
return dataframe
@@ -91,6 +82,7 @@ out : str
return filename
+
def _parse_arguments():
"""Parses the command line arguments.
@@ -100,17 +92,17 @@ def _parse_arguments():
tempdir = os.environ["SHARED_DIR"]
parser.add_argument('-t', '--tempdir', required=False, default=tempdir,
help="Temporary dir for saving the result.")
- parser.add_argument('-u', '--urlencoded', required=True,
- help="The URL encoded csv data.")
parser.add_argument('-a', '--auth-token', required=False,
help="An authentication token (not needed, only for compatibility).")
+ parser.add_argument('tsv', help="The tsv file.")
return parser.parse_args()
+
def main():
- args = _parse_arguments()
- dataframe = _parse_to_dataframe(args.urlencoded)
- filename = _write_xls(dataframe, directory=args.tempdir)
- print(filename)
+ args = _parse_arguments()
+ dataframe = _parse_to_dataframe(args.tsv)
+ filename = _write_xls(dataframe, directory=args.tempdir)
+ print(filename)
if __name__ == "__main__":
- main()
+ main()
diff --git a/src/main/java/caosdb/server/CaosDBServer.java b/src/main/java/caosdb/server/CaosDBServer.java
index 3245902d5721e00d1ae0418a33e7a5f468794ea6..19560aacd50ede38471c0fb02e31fd4560e9d8bb 100644
--- a/src/main/java/caosdb/server/CaosDBServer.java
+++ b/src/main/java/caosdb/server/CaosDBServer.java
@@ -27,7 +27,7 @@ import caosdb.server.accessControl.OneTimeTokenRealm;
import caosdb.server.accessControl.Principal;
import caosdb.server.accessControl.SessionToken;
import caosdb.server.accessControl.SessionTokenRealm;
-import caosdb.server.database.Database;
+import caosdb.server.database.BackendTransaction;
import caosdb.server.database.access.Access;
import caosdb.server.database.backend.transaction.RetrieveDatatypes;
import caosdb.server.database.misc.TransactionBenchmark;
@@ -49,17 +49,16 @@ import caosdb.server.resource.ScriptingResource;
import caosdb.server.resource.ServerLogsResource;
import caosdb.server.resource.ServerPropertiesResource;
import caosdb.server.resource.SharedFileResource;
-import caosdb.server.resource.TestCaseFileSystemResource;
-import caosdb.server.resource.TestCaseResource;
import caosdb.server.resource.ThumbnailsResource;
import caosdb.server.resource.UserResource;
import caosdb.server.resource.UserRolesResource;
import caosdb.server.resource.Webinterface;
+import caosdb.server.resource.WebinterfaceBuildNumber;
import caosdb.server.resource.transaction.EntityResource;
import caosdb.server.terminal.CaosDBTerminal;
import caosdb.server.terminal.StatsPanel;
import caosdb.server.terminal.SystemErrPanel;
-import caosdb.server.utils.ChecksumUpdater;
+import caosdb.server.transaction.ChecksumUpdater;
import caosdb.server.utils.FileUtils;
import caosdb.server.utils.Initialization;
import caosdb.server.utils.NullPrintStream;
@@ -111,7 +110,6 @@ import org.slf4j.LoggerFactory;
public class CaosDBServer extends Application {
private static Logger logger = LoggerFactory.getLogger(CaosDBServer.class);
- private static boolean DEBUG_MODE = false;
private static boolean START_GUI = true;
private static Properties SERVER_PROPERTIES = null;
private static Component component = null;
@@ -222,7 +220,26 @@ public class CaosDBServer extends Application {
INSECURE = true;
}
}
- DEBUG_MODE = Boolean.getBoolean("caosdb.debug");
+ }
+
+ public static Ini getShiroConfig() {
+ final Ini config = new Ini();
+ final Section mainSec = config.addSection("main");
+ mainSec.put("CaosDB", CaosDBDefaultRealm.class.getCanonicalName());
+ mainSec.put("SessionTokenValidator", SessionTokenRealm.class.getCanonicalName());
+ mainSec.put("OneTimeTokenValidator", OneTimeTokenRealm.class.getCanonicalName());
+ mainSec.put("CaosDBAuthorizingRealm", CaosDBAuthorizingRealm.class.getCanonicalName());
+ mainSec.put("AnonymousRealm", AnonymousRealm.class.getCanonicalName());
+ mainSec.put(
+ "securityManager.realms",
+ "$CaosDB, $SessionTokenValidator, $OneTimeTokenValidator, $CaosDBAuthorizingRealm, $AnonymousRealm");
+
+ // disable shiro's default session management. We have quasi-stateless
+ // sessions
+ // using our SessionToken class.
+ mainSec.put(
+ "securityManager.subjectDAO.sessionStorageEvaluator.sessionStorageEnabled", "false");
+ return config;
}
/**
@@ -248,28 +265,12 @@ public class CaosDBServer extends Application {
System.exit(1);
}
- INSECURE = INSECURE && DEBUG_MODE; // only allow insecure in debug mode
- START_BACKEND = START_BACKEND || !DEBUG_MODE; // always start backend if
+ INSECURE = INSECURE && isDebugMode(); // only allow insecure in debug mode
+ START_BACKEND = START_BACKEND || !isDebugMode(); // always start backend if
// not in debug mode
// init Shiro (user authentication/authorization and session management)
- final Ini config = new Ini();
- final Section mainSec = config.addSection("main");
- mainSec.put("CaosDB", CaosDBDefaultRealm.class.getCanonicalName());
- mainSec.put("SessionTokenValidator", SessionTokenRealm.class.getCanonicalName());
- mainSec.put("OneTimeTokenValidator", OneTimeTokenRealm.class.getCanonicalName());
- mainSec.put("CaosDBAuthorizingRealm", CaosDBAuthorizingRealm.class.getCanonicalName());
- mainSec.put("AnonymousRealm", AnonymousRealm.class.getCanonicalName());
- mainSec.put(
- "securityManager.realms",
- "$CaosDB, $SessionTokenValidator, $OneTimeTokenValidator, $CaosDBAuthorizingRealm, $AnonymousRealm");
-
- // disable shiro's default session management. We have quasi-stateless
- // sessions
- // using our SessionToken class.
- mainSec.put(
- "securityManager.subjectDAO.sessionStorageEvaluator.sessionStorageEnabled", "false");
-
+ final Ini config = getShiroConfig();
final Factory<SecurityManager> factory = new IniSecurityManagerFactory(config);
final SecurityManager securityManager = factory.getInstance();
SecurityUtils.setSecurityManager(securityManager);
@@ -278,8 +279,10 @@ public class CaosDBServer extends Application {
try {
// init backend
if (START_BACKEND) {
+ BackendTransaction.init();
+
// init benchmark
- TransactionBenchmark.getInstance();
+ TransactionBenchmark.getRootInstance();
// Role
Role.init(init.getAccess());
@@ -315,7 +318,7 @@ public class CaosDBServer extends Application {
Thread.sleep(1000);
// add Benchmark
- StatsPanel.addStat("TransactionBenchmark", TransactionBenchmark.getInstance());
+ StatsPanel.addStat("TransactionBenchmark", TransactionBenchmark.getRootInstance());
} else {
logger.info("NO GUI");
System.setOut(new NullPrintStream());
@@ -356,7 +359,9 @@ public class CaosDBServer extends Application {
private static void initDatatypes(final Access access) throws Exception {
final RetrieveDatatypes t = new RetrieveDatatypes();
- final Container<? extends EntityInterface> dts = Database.execute(t, access).getDatatypes();
+ t.setAccess(access);
+ t.executeTransaction();
+ final Container<? extends EntityInterface> dts = t.getDatatypes();
AbstractDatatype.initializeDatatypes(dts);
}
@@ -579,39 +584,8 @@ public class CaosDBServer extends Application {
}
};
- // -- Section only for debug mode --
- if (isDebugMode()) {
- baseRouter.attach("/TestCase/", DefaultResource.class);
-
- final Variable pathVariable =
- baseRouter
- .attach(
- "/TestCase/FileSystem/{path}",
- baseRouter.createFinder(TestCaseFileSystemResource.class))
- .getTemplate()
- .getDefaultVariable();
- pathVariable.setRequired(false);
- pathVariable.setType(Variable.TYPE_URI_PATH);
- pathVariable.setDefaultValue("");
-
- baseRouter
- .attach("/TestCase/Thumbnails/{path}", ThumbnailsResource.class)
- .getTemplate()
- .getDefaultVariable()
- .setType(Variable.TYPE_URI_PATH);
-
- baseRouter
- .attach("/TestCase/webinterface/{path}", Webinterface.class)
- .getTemplate()
- .getDefaultVariable()
- .setType(Variable.TYPE_URI_PATH);
- baseRouter.attach("/TestCase/Entity", TestCaseResource.class);
- baseRouter.attach("/TestCase/Entity/", TestCaseResource.class);
- baseRouter.attach("/TestCase/Entity/{specifier}", TestCaseResource.class);
- }
- // -- End of debug section --
-
// These routes can be used without logging in:
+ baseRouter.attach("/webinterface/version/build", WebinterfaceBuildNumber.class);
baseRouter
.attach("/webinterface/{path}", Webinterface.class)
.getTemplate()
@@ -829,7 +803,7 @@ public class CaosDBServer extends Application {
}
public static boolean isDebugMode() {
- return DEBUG_MODE;
+ return Boolean.getBoolean("caosdb.debug");
}
/**
diff --git a/src/main/java/caosdb/server/FileSystem.java b/src/main/java/caosdb/server/FileSystem.java
index 44b663e958841ac92a24f44244096034eb9154c3..79e764b96d3b477a57b90b83b1a3f437a1d21266 100644
--- a/src/main/java/caosdb/server/FileSystem.java
+++ b/src/main/java/caosdb/server/FileSystem.java
@@ -24,9 +24,10 @@
package caosdb.server;
-import caosdb.server.database.Database;
import caosdb.server.database.access.Access;
import caosdb.server.database.backend.transaction.GetFileRecordByPath;
+import caosdb.server.database.exceptions.EntityDoesNotExistException;
+import caosdb.server.database.misc.TransactionBenchmark;
import caosdb.server.entity.EntityInterface;
import caosdb.server.entity.FileProperties;
import caosdb.server.entity.Message;
@@ -300,8 +301,8 @@ public class FileSystem {
* @return
* @throws Message
*/
- public static boolean checkTarget(final EntityInterface entity, final Access access)
- throws Message {
+ public static boolean checkTarget(
+ final EntityInterface entity, final Access access, TransactionBenchmark b) throws Message {
final FileProperties file = entity.getFileProperties();
// target file name = where the file is to be stored.
@@ -321,16 +322,25 @@ public class FileSystem {
if (file.getFile() != null && file.getFile().equals(target)) {
return true;
} else {
- final GetFileRecordByPath t =
- Database.execute(new GetFileRecordByPath(file.getPath()), access);
+ final GetFileRecordByPath t = new GetFileRecordByPath(file.getPath());
+ t.setAccess(access);
+ t.setTransactionBenchmark(b);
+ try {
+ t.executeTransaction();
+ } catch (EntityDoesNotExistException e) {
+ // could not determine which entity owns this path
+ // this is usually the case when target is a directory
+ }
if (t.getEntity() != null) {
final Integer foreign = t.getId();
if (foreign != null && foreign.equals(entity.getId())) {
+ // entity already owns this path
return true;
}
- throw ServerMessages.TARGET_PATH_EXISTS;
}
}
+ // another entity owns this path
+ throw ServerMessages.TARGET_PATH_EXISTS;
}
return true;
diff --git a/src/main/java/caosdb/server/ServerProperties.java b/src/main/java/caosdb/server/ServerProperties.java
index 929f845d8023dc9d81c62294a5318442fcfb52ea..e68016894d65a3b77e38aa2f84f48ece7db05683 100644
--- a/src/main/java/caosdb/server/ServerProperties.java
+++ b/src/main/java/caosdb/server/ServerProperties.java
@@ -4,6 +4,8 @@
*
* Copyright (C) 2018 Research Group Biomedical Physics,
* Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+ * Copyright (C) 2019 IndiScale GmbH
+ * Copyright (C) 2019 Timm Fitschen (t.fitschen@indiscale.com)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
@@ -90,13 +92,6 @@ public class ServerProperties extends Properties {
public static final String KEY_CACHE_CONF_LOC = "CACHE_CONF_LOC";
- public static final String KEY_RULES_CACHE_CAPACITY = "RULES_CACHE_CAPACITY";
- public static final String KEY_SPARSE_ENTITY_CACHE_CAPACITY = "SPARSE_ENTITY_CACHE_CAPACITY";
- public static final String KEY_PROPERTIES_CACHE_CAPACITY = "PROPERTIES_CACHE_CAPACITY";
- public static final String KEY_PARENTS_CACHE_CAPACITY = "PARENTS_CACHE_CAPACITY";
- public static final String KEY_USER_ACCOUNT_CACHE_CAPACITY = "USER_ACCOUNT_CACHE_CAPACITY";
- public static final String KEY_GROUP_CACHE_CAPACITY = "GROUP_CACHE_CAPACITY";
-
public static final String KEY_TRANSACTION_BENCHMARK_ENABLED = "TRANSACTION_BENCHMARK_ENABLED";
public static final String KEY_INSERT_FILES_IN_DIR_ALLOWED_DIRS =
diff --git a/src/main/java/caosdb/server/accessControl/Pam.java b/src/main/java/caosdb/server/accessControl/Pam.java
index c222fb83dabe0e4ad4e99d096d33192bf65086a9..d640a66f4fffdb6254c7d11d696a1f2a1bf5ebf6 100644
--- a/src/main/java/caosdb/server/accessControl/Pam.java
+++ b/src/main/java/caosdb/server/accessControl/Pam.java
@@ -4,6 +4,8 @@
*
* Copyright (C) 2018 Research Group Biomedical Physics,
* Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+ * Copyright (C) 2019 IndiScale GmbH
+ * Copyright (C) 2019 Timm Fitschen (t.fitschen@indiscale.com)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
@@ -22,23 +24,38 @@
*/
package caosdb.server.accessControl;
+import caosdb.server.caching.Cache;
import java.io.File;
import java.io.IOException;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
+import org.apache.commons.jcs.access.behavior.ICacheAccess;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
import org.apache.shiro.authz.AuthorizationException;
import org.jvnet.libpam.PAMException;
import org.jvnet.libpam.UnixUser;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+/**
+ * PAM UserSource for authenticating users via the Host's pam module.
+ *
+ * <p>A User's existence check and the retrieval of a user's groups is done by the org.jvnet.libpam
+ * library.
+ *
+ * <p>The authentication of a user via the password need root-access and is therefore done by a
+ * special shell script running with root privileges on the host.
+ *
+ * @author Timm Fitschen (t.fitschen@indiscale.com)
+ */
public class Pam implements UserSource {
+ private Logger logger = LogManager.getLogger(Pam.class);
+
public static class DefaultPamScriptCaller implements PamScriptCaller {
- private Logger logger = LoggerFactory.getLogger(getClass());
+ private Logger logger = LogManager.getLogger(Pam.class);
private final String pam_script;
@@ -64,7 +81,7 @@ public class Pam implements UserSource {
try {
pam_authentication = getProcess(username, password);
- logger.info("call pam script");
+ logger.trace("call pam script");
return pam_authentication.waitFor() == 0;
} catch (final IOException e) {
throw new RuntimeException(e);
@@ -75,6 +92,11 @@ public class Pam implements UserSource {
}
public static final String DEFAULT_PAM_SCRIPT = "./misc/pam_authentication/pam_authentication.sh";
+
+ /*
+ * following constants are the names of the configuration parameters and parts of parameters in
+ * the user_sources.ini for the pam user source.
+ */
public static final String KEY_PAM_SCRIPT = "pam_script";
public static final String KEY_DEFAULT_USER_STATUS = "default_status";
public static final String KEY_GROUP = "group";
@@ -83,8 +105,13 @@ public class Pam implements UserSource {
public static final String KEY_INCLUDE = "include";
public static final String KEY_ROLES = "roles";
public static final String SEPARATOR = ".";
+ public static final String KEY_EMAIL = "email";
public static final String REGEX_SPLIT_CSV = "\\s*,\\s*";
- private static final String KEY_EMAIL = "email";
+
+ public static final String CACHE_REGION_GROUPS = "PAM_UnixUserGroups";
+ public static final String CACHE_REGION_USER_EXIST = "PAM_UnixUserExists";
+ public static final String CACHE_REGION_AUTH = "PAM_Authentication";
+
private String[] EXCLUDE_USERS = null;
private String[] INCLUDE_USERS = null;
private String[] EXCLUDE_GROUPS = null;
@@ -92,16 +119,28 @@ public class Pam implements UserSource {
private Map<String, String> map = null;
private PamScriptCaller pamScriptCaller = null;
+ /* Caches for groups, user-existence and password validation */
+
+ private ICacheAccess<String, HashSet<String>> groupsCache = Cache.getCache(CACHE_REGION_GROUPS);
+ private ICacheAccess<String, Boolean> userExistsCache = Cache.getCache(CACHE_REGION_USER_EXIST);
+ private ICacheAccess<String, Boolean> passwordValidCache = Cache.getCache(CACHE_REGION_AUTH);
+
@Override
public void setMap(final Map<String, String> map) {
this.map = map;
this.pamScriptCaller = null;
}
- public Map<String, String> getMap() {
+ /**
+ * Return the current configuration of this user source.
+ *
+ * @return configuration.
+ */
+ private Map<String, String> getMap() {
return this.map;
}
+ /** @see {@link UserSource#resolveRolesForUsername(String)} */
@Override
public Set<String> resolveRolesForUsername(final String username) {
final Set<String> resulting_roles = new HashSet<String>();
@@ -138,7 +177,40 @@ public class Pam implements UserSource {
return resulting_roles;
}
- private static Set<String> getGroups(final String username) {
+ /**
+ * Get the UNIX groups of the user.
+ *
+ * <p>First, try to get them from the cache. Only ask the back-end if necessary and cache the
+ * results.
+ *
+ * @param username
+ * @return A user's UNIX groups.
+ */
+ private Set<String> getGroups(final String username) {
+ Set<String> cached = groupsCache.get(username);
+ if (cached != null) {
+ return cached;
+ }
+
+ Set<String> uncached = getGroupsNoCache(username);
+ if (uncached instanceof HashSet) {
+ groupsCache.put(username, (HashSet<String>) uncached);
+ } else {
+ groupsCache.put(username, new HashSet<String>(uncached));
+ }
+ return uncached;
+ }
+
+ /**
+ * Get the UNIX groups of the user directly from the back-end.
+ *
+ * <p>If the user does not exist, an empty set is returned.
+ *
+ * @param username
+ * @return A user's UNIX groups or an empty set.
+ */
+ private Set<String> getGroupsNoCache(final String username) {
+ logger.trace("Retrieving UnixGroups", username);
if (UnixUser.exists(username)) {
try {
return new UnixUser(username).getGroups();
@@ -146,7 +218,7 @@ public class Pam implements UserSource {
throw new AuthorizationException(e);
}
}
- return null;
+ return new HashSet<String>();
}
@Override
@@ -154,11 +226,38 @@ public class Pam implements UserSource {
return "PAM";
}
+ /**
+ * Check if that user is known by the host's PAM.
+ *
+ * <p>Try the cache first. Only ask PAM directly if necessary and cache the results.
+ *
+ * @see {@link UserSource#isUserExisting(String)}.
+ * @return true iff the user is known.
+ */
@Override
public boolean isUserExisting(final String username) {
+ Boolean cached = userExistsCache.get(username);
+ if (cached != null) {
+ return cached;
+ }
+
+ boolean uncached = isUserExistingNoCache(username);
+ userExistsCache.put(username, uncached);
+ return uncached;
+ }
+
+ /**
+ * Check if that user is known by the host's PAM (without caching).
+ *
+ * @param username
+ * @return true iff the user is known.
+ */
+ private boolean isUserExistingNoCache(final String username) {
+ logger.trace("Check UnixUser.exists", username);
return username != null && UnixUser.exists(username) && isIncorporated(username);
}
+ /** @see {@link UserSource#isValid(String, String)}. */
@Override
public boolean isValid(final String username, final String password) {
if (isUserExisting(username)) {
@@ -174,8 +273,40 @@ public class Pam implements UserSource {
return this.pamScriptCaller;
}
+ /**
+ * Check the validity of the password for that user by asking the pam script caller.
+ *
+ * <p>Try the cache first, only call the pam script if necessary and cache the results.
+ *
+ * @param caller
+ * @param username
+ * @param password
+ * @return true iff the password is correct.
+ */
private boolean isValid(
final PamScriptCaller caller, final String username, final String password) {
+ String key = "<" + password + "::" + username + ">";
+ Boolean cached = passwordValidCache.get(key);
+ if (cached != null) {
+ return cached;
+ }
+
+ boolean uncached = isValidNoCache(caller, username, password);
+ passwordValidCache.put(key, uncached);
+ return uncached;
+ }
+
+ /**
+ * Check the validity of the password for that user by asking the pam script caller.
+ *
+ * @param caller
+ * @param username
+ * @param password
+ * @return true iff the password is correct.
+ */
+ private boolean isValidNoCache(
+ final PamScriptCaller caller, final String username, final String password) {
+ logger.trace("Check Password", username);
return caller.isValid(username, password);
}
@@ -278,10 +409,15 @@ public class Pam implements UserSource {
ret = n;
} else if (ret != n) {
// conflict -> ignore, go for pam-wide setting
+ ret = null;
break;
}
}
}
+
+ if (ret != null) {
+ return ret;
+ }
// by pam-wide setting
if (this.map.containsKey(KEY_DEFAULT_USER_STATUS)) {
return UserStatus.valueOf(this.map.get(KEY_DEFAULT_USER_STATUS).toUpperCase());
diff --git a/src/main/java/caosdb/server/accessControl/UserSource.java b/src/main/java/caosdb/server/accessControl/UserSource.java
index 23ce7c45ce6cb9982ae5c7fac05a3a41037a5afe..0ece8d5014227916feb1c50b986aaf4d9a9bf91b 100644
--- a/src/main/java/caosdb/server/accessControl/UserSource.java
+++ b/src/main/java/caosdb/server/accessControl/UserSource.java
@@ -4,6 +4,8 @@
*
* Copyright (C) 2018 Research Group Biomedical Physics,
* Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+ * Copyright (C) 2019 IndiScale GmbH
+ * Copyright (C) 2019 Timm Fitschen (t.fitschen@indiscale.com)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
@@ -25,19 +27,84 @@ package caosdb.server.accessControl;
import java.util.Map;
import java.util.Set;
+/**
+ * UserSources are sources for users - i.e. they authenticate users and contain basic information
+ * about users.
+ *
+ * <p>UserSources let you
+ * <li>check if a user exists - {@link #isUserExisting(String)}
+ * <li>authenticate a user via a password - {@link #isValid(String, String)}
+ * <li>get the default {@link UserStatus} - {@link #getDefaultUserStatus(String)}
+ * <li>get the default email address - {@link #getDefaultUserEmail(String)}
+ * <li>retrieve a users roles - {@link #resolveRolesForUsername(String)} The default email and
+ * default {@link UserStatus} might be overridden by other settings in CaosDB - that's why they
+ * are called "default".
+ *
+ * <p>Also, the user's roles might be overridden by the internal user source {@link
+ * InternalUserSource}.
+ *
+ * <p>A UserSource is configured via {@link #setMap(Map)}.
+ *
+ * @author Timm Fitschen (t.fitschen@indiscale.com)
+ */
public interface UserSource {
+ /**
+ * Every UserSource has a unique name, e.g. PAM, CaosDB (which is default name of the internal
+ * user source {@link InternalUserSource}).
+ *
+ * @return name
+ */
public String getName();
+ /**
+ * Check if a user exists.
+ *
+ * @param username
+ * @return true iff this user source knows a user with that name
+ */
public boolean isUserExisting(String username);
+ /**
+ * Return all roles that a user is associated with.
+ *
+ * @param username
+ * @return a user's roles
+ */
public Set<String> resolveRolesForUsername(final String username);
+ /**
+ * Configure this user source. The needed parameters are to be defined and documented by the
+ * implementations.
+ *
+ * @param map the configuration
+ */
public void setMap(Map<String, String> map);
+ /**
+ * Return the {@link UserStatus} of that user.
+ *
+ * @param username
+ * @return The user status of that user
+ */
public UserStatus getDefaultUserStatus(String username);
+ /**
+ * Return the email address of that user, or null if none is available as per this user source.
+ *
+ * <p>This method does not check if a user exists. So it will return null for unknown users.
+ *
+ * @param username
+ * @return The email address or null
+ */
public String getDefaultUserEmail(String username);
+ /**
+ * Check if the user can be authenticated by the given password.
+ *
+ * @param username
+ * @param password
+ * @return true iff the password was correct.
+ */
public boolean isValid(String username, String password);
}
diff --git a/src/main/java/caosdb/server/caching/Cache.java b/src/main/java/caosdb/server/caching/Cache.java
new file mode 100644
index 0000000000000000000000000000000000000000..338b1d853fd0610be780362a0242193fa46eb320
--- /dev/null
+++ b/src/main/java/caosdb/server/caching/Cache.java
@@ -0,0 +1,50 @@
+/*
+ * ** header v3.0
+ * This file is a part of the CaosDB Project.
+ *
+ * Copyright (C) 2018 Research Group Biomedical Physics,
+ * Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+ * Copyright (C) 2019 IndiScale GmbH
+ * Copyright (C) 2019 Timm Fitschen (t.fitschen@indiscale.com)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+ * ** end header
+ */
+package caosdb.server.caching;
+
+import java.io.Serializable;
+import org.apache.commons.jcs.access.behavior.ICacheAccess;
+
+/**
+ * Caching Helper Class used for all caches in the CaosDB Server.
+ *
+ * <p>The actual work is delegated to an instance of {@link JCSCacheHelper}. However, the delegate
+ * {@link #DELEGATE} can be be overridden for testing purposes with {@link #setDelegate(Cache)}.
+ *
+ * @author Timm Fitschen (t.fitschen@indiscale.com)
+ */
+public class Cache {
+
+ /** The default cache helper delegate. */
+ private static CacheHelper DELEGATE = new JCSCacheHelper();
+
+ public static void setDelegate(CacheHelper delegate) {
+ DELEGATE = delegate;
+ }
+
+ public static final <K, V extends Serializable> ICacheAccess<K, V> getCache(String region) {
+ return DELEGATE.getCache(region);
+ }
+}
diff --git a/src/main/java/caosdb/server/caching/CacheHelper.java b/src/main/java/caosdb/server/caching/CacheHelper.java
new file mode 100644
index 0000000000000000000000000000000000000000..32c854400310959a7834fe32a6f90de9e1b584d1
--- /dev/null
+++ b/src/main/java/caosdb/server/caching/CacheHelper.java
@@ -0,0 +1,31 @@
+/*
+ * ** header v3.0
+ * This file is a part of the CaosDB Project.
+ *
+ * Copyright (C) 2019 IndiScale GmbH
+ * Copyright (C) 2019 Timm Fitschen (t.fitschen@indiscale.com)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+ * ** end header
+ */
+package caosdb.server.caching;
+
+import java.io.Serializable;
+import org.apache.commons.jcs.access.behavior.ICacheAccess;
+
+public interface CacheHelper {
+
+ public <K, V extends Serializable> ICacheAccess<K, V> getCache(String region);
+}
diff --git a/src/main/java/caosdb/server/caching/JCSCacheHelper.java b/src/main/java/caosdb/server/caching/JCSCacheHelper.java
new file mode 100644
index 0000000000000000000000000000000000000000..55efaa191bcb777bd63881dfbd72cd292808064c
--- /dev/null
+++ b/src/main/java/caosdb/server/caching/JCSCacheHelper.java
@@ -0,0 +1,98 @@
+/*
+ * ** header v3.0
+ * This file is a part of the CaosDB Project.
+ *
+ * Copyright (C) 2018 Research Group Biomedical Physics,
+ * Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+ * Copyright (C) 2019 IndiScale GmbH
+ * Copyright (C) 2019 Timm Fitschen (t.fitschen@indiscale.com)
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+ * ** end header
+ */
+package caosdb.server.caching;
+
+import caosdb.server.CaosDBServer;
+import caosdb.server.ServerProperties;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.Serializable;
+import java.util.Properties;
+import org.apache.commons.jcs.JCS;
+import org.apache.commons.jcs.access.behavior.ICacheAccess;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+/**
+ * A CacheHelper implementation which is configured statically via the {@link
+ * ServerProperties#KEY_CACHE_CONF_LOC} properties file.
+ *
+ * @author Timm Fitschen (t.fitschen@indiscale.com)
+ */
+public class JCSCacheHelper implements CacheHelper {
+
+ protected static Logger logger = LogManager.getLogger(JCSCacheHelper.class);
+
+ // A No-Operation configuration for JCS
+ private static Properties getNOPCachingProperties() {
+ Properties ret = new Properties();
+ ret.setProperty(
+ "jcs.default.cacheattributes", "org.apache.commons.jcs.engine.CompositeCacheAttributes");
+ ret.setProperty("jcs.default.cacheattributes.MaxObjects", "0");
+ return ret;
+ }
+
+ // configure JCS
+ static {
+ init();
+ }
+
+ public static void init() {
+ init(CaosDBServer.getServerProperty(ServerProperties.KEY_CACHE_CONF_LOC));
+ }
+
+ public static void init(String configFileLocation) {
+ Properties config = null;
+ try {
+ Properties p = new Properties();
+ final InputStream is = new FileInputStream(configFileLocation);
+ p.load(is);
+ is.close();
+ config = p;
+ } catch (final FileNotFoundException e) {
+ logger.error(e);
+ config = getNOPCachingProperties();
+ } catch (final IOException e) {
+ logger.error(e);
+ config = getNOPCachingProperties();
+ }
+ logger.info("Configuring JCS Caching with {}", config);
+ JCS.setConfigProperties(config);
+ }
+
+ @Override
+ public <K, V extends Serializable> ICacheAccess<K, V> getCache(final String name) {
+ final ICacheAccess<K, V> cache = JCS.getInstance(name);
+ logger.info(
+ "Caching configuration for {}:\n+----{}\n+----{}",
+ name,
+ cache.getCacheAttributes(),
+ cache.getDefaultElementAttributes());
+
+ return cache;
+ }
+}
diff --git a/src/main/java/caosdb/server/database/BackendTransaction.java b/src/main/java/caosdb/server/database/BackendTransaction.java
index 1d5d5c2e68274e664c392a24a901aa4bcd65623b..42cce21af9f7e851d27c960b9704aae46b96350f 100644
--- a/src/main/java/caosdb/server/database/BackendTransaction.java
+++ b/src/main/java/caosdb/server/database/BackendTransaction.java
@@ -130,21 +130,21 @@ public abstract class BackendTransaction implements Undoable {
private final UndoHandler undoHandler = new UndoHandler();
private Access access;
- BackendTransaction parent = null;
+ private TransactionBenchmark benchmark;
private static HashMap<
Class<? extends BackendTransactionImpl>, Class<? extends BackendTransactionImpl>>
impl = new HashMap<>();
protected abstract void execute();
- final void executeTransaction() {
+ public final void executeTransaction() {
final long t1 = System.currentTimeMillis();
execute();
final long t2 = System.currentTimeMillis();
- TransactionBenchmark.getInstance().addBenchmark(this, t2 - t1);
+ this.addMeasurement(this, t2 - t1);
}
- private static void init() {
+ public static void init() {
if (impl.isEmpty()) {
setImpl(DeleteEntityPropertiesImpl.class, MySQLDeleteEntityProperties.class);
setImpl(DeleteSparseEntityImpl.class, MySQLDeleteSparseEntity.class);
@@ -202,20 +202,22 @@ public abstract class BackendTransaction implements Undoable {
assert t != this;
this.undoHandler.append(t);
t.setAccess(this.access);
- t.parent = this;
+ if (benchmark != null) {
+ t.setTransactionBenchmark(benchmark.getBenchmark(t.getClass()));
+ }
final long t1 = System.currentTimeMillis();
t.execute();
final long t2 = System.currentTimeMillis();
- TransactionBenchmark.getInstance().addBenchmark(t, t2 - t1);
+ this.addMeasurement(t, t2 - t1);
return t;
}
- private static <K extends BackendTransactionImpl, L extends K> void setImpl(
+ public static <K extends BackendTransactionImpl, L extends K> void setImpl(
final Class<K> k, final Class<L> l) {
impl.put(k, l);
}
- void setAccess(final Access access) {
+ public void setAccess(final Access access) {
this.access = access;
}
@@ -223,10 +225,14 @@ public abstract class BackendTransaction implements Undoable {
protected <T extends BackendTransactionImpl> T getImplementation(final Class<T> clz) {
init();
try {
- final T ret = (T) impl.get(clz).getConstructor(Access.class).newInstance(this.access);
+ Class<?> implclz = impl.get(clz);
+ final T ret = (T) implclz.getConstructor(Access.class).newInstance(this.access);
if (ret instanceof Undoable) {
this.undoHandler.append((Undoable) ret);
}
+ if (benchmark != null) {
+ ret.setTransactionBenchmark(benchmark.getBenchmark(ret.getClass()));
+ }
return ret;
} catch (final Exception e) {
throw new TransactionException(e);
@@ -253,7 +259,16 @@ public abstract class BackendTransaction implements Undoable {
@Override
public String toString() {
- return (this.parent != null ? this.parent.toString() + " -> " : "")
- + this.getClass().getSimpleName();
+ return this.getClass().getSimpleName();
+ }
+
+ public void setTransactionBenchmark(TransactionBenchmark b) {
+ this.benchmark = b;
+ }
+
+ public void addMeasurement(Object o, long time) {
+ if (this.benchmark != null) {
+ this.benchmark.addMeasurement(o, time);
+ }
}
}
diff --git a/src/main/java/caosdb/server/database/CacheableBackendTransaction.java b/src/main/java/caosdb/server/database/CacheableBackendTransaction.java
index 390a177fe899f89330aeb3d0959aab5bf1d52cbc..fc4c9659d9a8ea80393bebad945bd6badf9580c4 100644
--- a/src/main/java/caosdb/server/database/CacheableBackendTransaction.java
+++ b/src/main/java/caosdb/server/database/CacheableBackendTransaction.java
@@ -4,6 +4,8 @@
*
* Copyright (C) 2018 Research Group Biomedical Physics,
* Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+ * Copyright (C) 2019 IndiScale GmbH
+ * Copyright (C) 2019 Timm Fitschen (t.fitschen@indiscale.com)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
@@ -24,14 +26,19 @@ package caosdb.server.database;
import caosdb.server.database.exceptions.TransactionException;
import java.io.Serializable;
-import org.apache.commons.jcs.access.CacheAccess;
+import org.apache.commons.jcs.access.behavior.ICacheAccess;
public abstract class CacheableBackendTransaction<K, V extends Serializable>
extends BackendTransaction {
- public abstract V executeNoCache() throws TransactionException;
-
private Boolean cached = null;
+ private ICacheAccess<K, V> cache;
+
+ public CacheableBackendTransaction(ICacheAccess<K, V> cache) {
+ this.cache = cache;
+ }
+
+ public abstract V executeNoCache() throws TransactionException;
@Override
public final void execute() throws TransactionException {
@@ -67,8 +74,8 @@ public abstract class CacheableBackendTransaction<K, V extends Serializable>
protected abstract K getKey();
- protected CacheAccess<K, V> getCache() {
- return null;
+ protected final ICacheAccess<K, V> getCache() {
+ return cache;
}
private final boolean cacheIsEnabled() {
diff --git a/src/main/java/caosdb/server/database/Database.java b/src/main/java/caosdb/server/database/Database.java
index c30b68c7c16cecf4d59f3d6adda10e066a42763f..d67acdff9d8fbacdbdefdfdeb5013c3838156461 100644
--- a/src/main/java/caosdb/server/database/Database.java
+++ b/src/main/java/caosdb/server/database/Database.java
@@ -22,18 +22,17 @@
*/
package caosdb.server.database;
-import caosdb.server.database.access.Access;
-import caosdb.server.database.misc.RollBackHandler;
-
public class Database {
- public static <K extends BackendTransaction> K execute(final K t, final Access access) {
-
- final RollBackHandler handler = (RollBackHandler) access.getHelper("RollBack");
- handler.append(t);
- t.setAccess(access);
-
- t.executeTransaction();
- return t;
- }
+ // public static <K extends BackendTransaction> K execute(
+ // final K t, final Access access, TransactionBenchmark b) {
+ //
+ // final RollBackHandler handler = (RollBackHandler) access.getHelper("RollBack");
+ // handler.append(t);
+ // t.setAccess(access);
+ //
+ // t.setTransactionBenchmark(b);
+ // t.executeTransaction();
+ // return t;
+ // }
}
diff --git a/src/main/java/caosdb/server/database/backend/implementation/MySQL/MySQLRetrieveProperties.java b/src/main/java/caosdb/server/database/backend/implementation/MySQL/MySQLRetrieveProperties.java
index 329a4e81b211815ac3ab5059e4b789eed9dce1b9..8b9a547bac69cf221dbf7c204a93e95326bb484c 100644
--- a/src/main/java/caosdb/server/database/backend/implementation/MySQL/MySQLRetrieveProperties.java
+++ b/src/main/java/caosdb/server/database/backend/implementation/MySQL/MySQLRetrieveProperties.java
@@ -49,6 +49,7 @@ public class MySQLRetrieveProperties extends MySQLTransaction implements Retriev
final PreparedStatement prepareStatement = prepareStatement(stmtStr);
final List<FlatProperty> props = retrieveFlatPropertiesStage1(0, entity, prepareStatement);
+
final ArrayList<ProtoProperty> protos = new ArrayList<ProtoProperty>();
for (final FlatProperty p : props) {
final ProtoProperty proto = new ProtoProperty();
@@ -56,6 +57,7 @@ public class MySQLRetrieveProperties extends MySQLTransaction implements Retriev
final List<FlatProperty> subProps =
retrieveFlatPropertiesStage1(entity, p.id, prepareStatement);
+
proto.subProperties = subProps;
protos.add(proto);
@@ -80,7 +82,10 @@ public class MySQLRetrieveProperties extends MySQLTransaction implements Retriev
}
stmt.setInt(2, entity);
+ long t1 = System.currentTimeMillis();
rs = stmt.executeQuery();
+ long t2 = System.currentTimeMillis();
+ addMeasurement(this.getClass().getSimpleName() + ".retrieveFlatPropertiesStage1", t2 - t1);
final List<FlatProperty> props = DatabaseUtils.parsePropertyResultset(rs);
@@ -107,7 +112,10 @@ public class MySQLRetrieveProperties extends MySQLTransaction implements Retriev
try {
stmt2.setInt(1, domain);
stmt2.setInt(2, entity);
+ long t1 = System.currentTimeMillis();
rs = stmt2.executeQuery();
+ long t2 = System.currentTimeMillis();
+ addMeasurement(this.getClass().getSimpleName() + ".retrieveOverrides", t2 - t1);
DatabaseUtils.parseOverrides(props, rs);
} finally {
if (rs != null && !rs.isClosed()) {
diff --git a/src/main/java/caosdb/server/database/backend/implementation/MySQL/MySQLTransaction.java b/src/main/java/caosdb/server/database/backend/implementation/MySQL/MySQLTransaction.java
index 1eb398634a665672bfbd266c51d819565fbea04d..f116a136c1b3e1942ddc2400139db3c82210fac8 100644
--- a/src/main/java/caosdb/server/database/backend/implementation/MySQL/MySQLTransaction.java
+++ b/src/main/java/caosdb/server/database/backend/implementation/MySQL/MySQLTransaction.java
@@ -23,13 +23,16 @@
package caosdb.server.database.backend.implementation.MySQL;
import caosdb.server.database.access.Access;
+import caosdb.server.database.backend.interfaces.BackendTransactionImpl;
import caosdb.server.database.misc.DBHelper;
+import caosdb.server.database.misc.TransactionBenchmark;
import java.sql.PreparedStatement;
import java.sql.SQLException;
-public class MySQLTransaction {
+public class MySQLTransaction implements BackendTransactionImpl {
protected final Access access;
+ private TransactionBenchmark benchmark;
public MySQLTransaction(final Access access) {
this.access = access;
@@ -54,4 +57,20 @@ public class MySQLTransaction {
public final void undo() {}
public final void cleanUp() {}
+
+ @Override
+ public TransactionBenchmark getBenchmark() {
+ return benchmark;
+ }
+
+ @Override
+ public void setTransactionBenchmark(TransactionBenchmark b) {
+ this.benchmark = b;
+ }
+
+ protected void addMeasurement(String string, long l) {
+ if (benchmark != null) {
+ benchmark.addMeasurement(string, l);
+ }
+ }
}
diff --git a/src/main/java/caosdb/server/database/backend/implementation/UnixFileSystem/UnixFileSystemTransaction.java b/src/main/java/caosdb/server/database/backend/implementation/UnixFileSystem/UnixFileSystemTransaction.java
index 851e0b0d6bfdbe0b29f9ce8ec51d26f1b26954e8..63e87d6a4f968d3d4833111f9ca9875daf18d9cc 100644
--- a/src/main/java/caosdb/server/database/backend/implementation/UnixFileSystem/UnixFileSystemTransaction.java
+++ b/src/main/java/caosdb/server/database/backend/implementation/UnixFileSystem/UnixFileSystemTransaction.java
@@ -24,19 +24,27 @@ package caosdb.server.database.backend.implementation.UnixFileSystem;
import caosdb.server.FileSystem;
import caosdb.server.database.access.Access;
+import caosdb.server.database.backend.interfaces.BackendTransactionImpl;
import caosdb.server.database.misc.DBHelper;
+import caosdb.server.database.misc.TransactionBenchmark;
import caosdb.server.entity.Message;
import java.io.File;
import java.io.IOException;
-public abstract class UnixFileSystemTransaction {
+public abstract class UnixFileSystemTransaction implements BackendTransactionImpl {
private final Access access;
+ private TransactionBenchmark benchmark;
public UnixFileSystemTransaction(final Access access) {
this.access = access;
}
+ @Override
+ public void setTransactionBenchmark(TransactionBenchmark b) {
+ this.benchmark = b;
+ }
+
protected File getFile(final String path) throws IOException, Message {
return new File(getHelper().getBasePath() + path);
}
@@ -56,4 +64,9 @@ public abstract class UnixFileSystemTransaction {
return (UnixFileSystemHelper) dbHelper;
}
}
+
+ @Override
+ public TransactionBenchmark getBenchmark() {
+ return benchmark;
+ }
}
diff --git a/src/main/java/caosdb/server/database/backend/interfaces/BackendTransactionImpl.java b/src/main/java/caosdb/server/database/backend/interfaces/BackendTransactionImpl.java
index cc2af6e2d8a421eab23b466986413bf2819b10b5..d4751f94cda0b0bdea703d5f207527ea756681f3 100644
--- a/src/main/java/caosdb/server/database/backend/interfaces/BackendTransactionImpl.java
+++ b/src/main/java/caosdb/server/database/backend/interfaces/BackendTransactionImpl.java
@@ -22,4 +22,11 @@
*/
package caosdb.server.database.backend.interfaces;
-public interface BackendTransactionImpl {}
+import caosdb.server.database.misc.TransactionBenchmark;
+
+public interface BackendTransactionImpl {
+
+ public void setTransactionBenchmark(TransactionBenchmark b);
+
+ public TransactionBenchmark getBenchmark();
+}
diff --git a/src/main/java/caosdb/server/database/backend/transaction/DeleteSparseEntity.java b/src/main/java/caosdb/server/database/backend/transaction/DeleteSparseEntity.java
index 4bd6e5b9706fe743be6c924c3e1f86cb6ff83371..39deb1416c743f838bfc1b96c29fd845dbcef742 100644
--- a/src/main/java/caosdb/server/database/backend/transaction/DeleteSparseEntity.java
+++ b/src/main/java/caosdb/server/database/backend/transaction/DeleteSparseEntity.java
@@ -4,6 +4,8 @@
*
* Copyright (C) 2018 Research Group Biomedical Physics,
* Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+ * Copyright (C) 2019 IndiScale GmbH
+ * Copyright (C) 2019 Timm Fitschen (t.fitschen@indiscale.com)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
@@ -41,6 +43,9 @@ public class DeleteSparseEntity extends BackendTransaction {
@Override
protected void execute() {
RetrieveSparseEntity.removeCached(this.entity.getId());
+ if (entity.hasFileProperties()) {
+ GetFileRecordByPath.removeCached(this.entity.getFileProperties().getPath());
+ }
final DeleteSparseEntityImpl ret = getImplementation(DeleteSparseEntityImpl.class);
diff --git a/src/main/java/caosdb/server/database/backend/transaction/GetFileRecordByPath.java b/src/main/java/caosdb/server/database/backend/transaction/GetFileRecordByPath.java
index 195f16db5341fdcd204debbaad0217635d1fd222..6366bd2884aca38a0d54eca4a926fcc7e8b9cda4 100644
--- a/src/main/java/caosdb/server/database/backend/transaction/GetFileRecordByPath.java
+++ b/src/main/java/caosdb/server/database/backend/transaction/GetFileRecordByPath.java
@@ -4,6 +4,8 @@
*
* Copyright (C) 2018 Research Group Biomedical Physics,
* Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+ * Copyright (C) 2019 IndiScale GmbH
+ * Copyright (C) 2019 Timm Fitschen (t.fitschen@indiscale.com)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
@@ -22,20 +24,32 @@
*/
package caosdb.server.database.backend.transaction;
+import caosdb.server.caching.Cache;
import caosdb.server.database.CacheableBackendTransaction;
import caosdb.server.database.backend.interfaces.GetFileRecordByPathImpl;
+import caosdb.server.database.exceptions.EntityDoesNotExistException;
import caosdb.server.database.exceptions.TransactionException;
import caosdb.server.database.proto.SparseEntity;
+import org.apache.commons.jcs.access.behavior.ICacheAccess;
public class GetFileRecordByPath extends CacheableBackendTransaction<String, SparseEntity> {
+ private static final ICacheAccess<String, SparseEntity> cache =
+ Cache.getCache("BACKEND_SparseFileRecordsByPath");
private final String path;
private SparseEntity entity;
public GetFileRecordByPath(final String path) {
+ super(cache);
this.path = path;
}
+ public static void removeCached(final String path) {
+ if (path != null && cache != null) {
+ cache.remove(path);
+ }
+ }
+
@Override
protected String getKey() {
return this.path;
@@ -49,7 +63,11 @@ public class GetFileRecordByPath extends CacheableBackendTransaction<String, Spa
@Override
public SparseEntity executeNoCache() throws TransactionException {
final GetFileRecordByPathImpl t = getImplementation(GetFileRecordByPathImpl.class);
- return t.execute(getKey());
+ SparseEntity result = t.execute(getKey());
+ if (result == null) {
+ throw new EntityDoesNotExistException();
+ }
+ return result;
}
public Integer getId() {
diff --git a/src/main/java/caosdb/server/database/backend/transaction/RetrieveParents.java b/src/main/java/caosdb/server/database/backend/transaction/RetrieveParents.java
index 5d1545d29b9bab295c9f335fe4eb083f12f748b2..5c1b0178f51496f83a4fac6a4ce3b2447abdccd5 100644
--- a/src/main/java/caosdb/server/database/backend/transaction/RetrieveParents.java
+++ b/src/main/java/caosdb/server/database/backend/transaction/RetrieveParents.java
@@ -4,6 +4,8 @@
*
* Copyright (C) 2018 Research Group Biomedical Physics,
* Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+ * Copyright (C) 2019 IndiScale GmbH
+ * Copyright (C) 2019 Timm Fitschen (t.fitschen@indiscale.com)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
@@ -22,26 +24,21 @@
*/
package caosdb.server.database.backend.transaction;
-import caosdb.server.CaosDBServer;
-import caosdb.server.ServerProperties;
+import caosdb.server.caching.Cache;
import caosdb.server.database.CacheableBackendTransaction;
import caosdb.server.database.DatabaseUtils;
import caosdb.server.database.backend.interfaces.RetrieveParentsImpl;
import caosdb.server.database.exceptions.TransactionException;
-import caosdb.server.database.misc.Cache;
import caosdb.server.database.proto.VerySparseEntity;
import caosdb.server.entity.EntityInterface;
import java.util.ArrayList;
-import org.apache.commons.jcs.access.CacheAccess;
+import org.apache.commons.jcs.access.behavior.ICacheAccess;
public class RetrieveParents
extends CacheableBackendTransaction<Integer, ArrayList<VerySparseEntity>> {
- private static final CacheAccess<Integer, ArrayList<VerySparseEntity>> cache =
- Cache.getCache(
- "ParentsCache",
- Integer.valueOf(
- CaosDBServer.getServerProperty(ServerProperties.KEY_SPARSE_ENTITY_CACHE_CAPACITY)));
+ private static final ICacheAccess<Integer, ArrayList<VerySparseEntity>> cache =
+ Cache.getCache("BACKEND_EntityParents");
/**
* To be called by DeleteEntityProperties on execution.
@@ -54,14 +51,10 @@ public class RetrieveParents
}
}
- @Override
- protected CacheAccess<Integer, ArrayList<VerySparseEntity>> getCache() {
- return cache;
- }
-
private final EntityInterface entity;
public RetrieveParents(final EntityInterface entity) {
+ super(cache);
this.entity = entity;
}
diff --git a/src/main/java/caosdb/server/database/backend/transaction/RetrievePermissionRules.java b/src/main/java/caosdb/server/database/backend/transaction/RetrievePermissionRules.java
index c0daecbc46ac90630eac83d0c7334d7da148d429..7e9925777c21a6d041759cfdade7682b85458566 100644
--- a/src/main/java/caosdb/server/database/backend/transaction/RetrievePermissionRules.java
+++ b/src/main/java/caosdb/server/database/backend/transaction/RetrievePermissionRules.java
@@ -4,6 +4,8 @@
*
* Copyright (C) 2018 Research Group Biomedical Physics,
* Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+ * Copyright (C) 2019 IndiScale GmbH
+ * Copyright (C) 2019 Timm Fitschen (t.fitschen@indiscale.com)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
@@ -22,28 +24,24 @@
*/
package caosdb.server.database.backend.transaction;
-import caosdb.server.CaosDBServer;
-import caosdb.server.ServerProperties;
+import caosdb.server.caching.Cache;
import caosdb.server.database.CacheableBackendTransaction;
import caosdb.server.database.backend.interfaces.RetrievePermissionRulesImpl;
import caosdb.server.database.exceptions.TransactionException;
-import caosdb.server.database.misc.Cache;
import caosdb.server.permissions.PermissionRule;
import java.util.HashSet;
-import org.apache.commons.jcs.access.CacheAccess;
+import org.apache.commons.jcs.access.behavior.ICacheAccess;
public class RetrievePermissionRules
extends CacheableBackendTransaction<String, HashSet<PermissionRule>> {
- private static final CacheAccess<String, HashSet<PermissionRule>> cache =
- Cache.getCache(
- "PermissionRulesCache",
- Integer.valueOf(
- CaosDBServer.getServerProperty(ServerProperties.KEY_GROUP_CACHE_CAPACITY)));
+ private static final ICacheAccess<String, HashSet<PermissionRule>> cache =
+ Cache.getCache("BACKEND_PermissionRules");
private HashSet<PermissionRule> rules;
private final String role;
public RetrievePermissionRules(final String role) {
+ super(cache);
this.role = role;
}
@@ -51,11 +49,6 @@ public class RetrievePermissionRules
cache.remove(role);
}
- @Override
- protected CacheAccess<String, HashSet<PermissionRule>> getCache() {
- return cache;
- }
-
@Override
public HashSet<PermissionRule> executeNoCache() throws TransactionException {
final RetrievePermissionRulesImpl t = getImplementation(RetrievePermissionRulesImpl.class);
diff --git a/src/main/java/caosdb/server/database/backend/transaction/RetrieveProperties.java b/src/main/java/caosdb/server/database/backend/transaction/RetrieveProperties.java
index b75d91acba76a4d97b16395701afe42ff05c5fac..487feb56cf848e948fcacb8f33f49e4a0bc2267d 100644
--- a/src/main/java/caosdb/server/database/backend/transaction/RetrieveProperties.java
+++ b/src/main/java/caosdb/server/database/backend/transaction/RetrieveProperties.java
@@ -4,6 +4,8 @@
*
* Copyright (C) 2018 Research Group Biomedical Physics,
* Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+ * Copyright (C) 2019 IndiScale GmbH
+ * Copyright (C) 2019 Timm Fitschen (t.fitschen@indiscale.com)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
@@ -22,29 +24,25 @@
*/
package caosdb.server.database.backend.transaction;
-import caosdb.server.CaosDBServer;
-import caosdb.server.ServerProperties;
+import caosdb.server.caching.Cache;
import caosdb.server.database.CacheableBackendTransaction;
import caosdb.server.database.DatabaseUtils;
import caosdb.server.database.backend.interfaces.RetrievePropertiesImpl;
import caosdb.server.database.exceptions.TransactionException;
-import caosdb.server.database.misc.Cache;
import caosdb.server.database.proto.ProtoProperty;
import caosdb.server.entity.EntityInterface;
import caosdb.server.entity.Role;
import caosdb.server.entity.wrapper.Property;
import java.util.ArrayList;
-import org.apache.commons.jcs.access.CacheAccess;
+import org.apache.commons.jcs.access.behavior.ICacheAccess;
public class RetrieveProperties
extends CacheableBackendTransaction<Integer, ArrayList<ProtoProperty>> {
private final EntityInterface entity;
- private static final CacheAccess<Integer, ArrayList<ProtoProperty>> cache =
- Cache.getCache(
- "PropertiesCache",
- Integer.valueOf(
- CaosDBServer.getServerProperty(ServerProperties.KEY_PROPERTIES_CACHE_CAPACITY)));
+ public static final String CACHE_REGION = "BACKEND_EntityProperties";
+ private static final ICacheAccess<Integer, ArrayList<ProtoProperty>> cache =
+ Cache.getCache(CACHE_REGION);
/**
* To be called by DeleteEntityProperties on execution.
@@ -57,12 +55,8 @@ public class RetrieveProperties
}
}
- @Override
- protected CacheAccess<Integer, ArrayList<ProtoProperty>> getCache() {
- return cache;
- }
-
public RetrieveProperties(final EntityInterface entity) {
+ super(cache);
this.entity = entity;
}
diff --git a/src/main/java/caosdb/server/database/backend/transaction/RetrieveRole.java b/src/main/java/caosdb/server/database/backend/transaction/RetrieveRole.java
index e45879964ef8bafad0106238073ff8c9d87e2287..5bb6150a8640f45997a4a3432ad4ac27196cd15c 100644
--- a/src/main/java/caosdb/server/database/backend/transaction/RetrieveRole.java
+++ b/src/main/java/caosdb/server/database/backend/transaction/RetrieveRole.java
@@ -4,6 +4,8 @@
*
* Copyright (C) 2018 Research Group Biomedical Physics,
* Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+ * Copyright (C) 2019 IndiScale GmbH
+ * Copyright (C) 2019 Timm Fitschen (t.fitschen@indiscale.com)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
@@ -22,36 +24,26 @@
*/
package caosdb.server.database.backend.transaction;
-import caosdb.server.CaosDBServer;
-import caosdb.server.ServerProperties;
import caosdb.server.accessControl.Role;
+import caosdb.server.caching.Cache;
import caosdb.server.database.CacheableBackendTransaction;
import caosdb.server.database.backend.interfaces.RetrieveRoleImpl;
import caosdb.server.database.exceptions.TransactionException;
-import caosdb.server.database.misc.Cache;
-import org.apache.commons.jcs.access.CacheAccess;
+import org.apache.commons.jcs.access.behavior.ICacheAccess;
public class RetrieveRole extends CacheableBackendTransaction<String, Role> {
- private static final CacheAccess<String, Role> cache =
- Cache.getCache(
- "RolesCache",
- Integer.valueOf(
- CaosDBServer.getServerProperty(ServerProperties.KEY_GROUP_CACHE_CAPACITY)));
+ private static final ICacheAccess<String, Role> cache = Cache.getCache("BACKEND_UserRoles");
private final String role_name;
private Role role;
- @Override
- protected CacheAccess<String, Role> getCache() {
- return cache;
- }
-
public Role getRole() {
return this.role;
}
public RetrieveRole(final String role) {
+ super(cache);
this.role_name = role;
}
diff --git a/src/main/java/caosdb/server/database/backend/transaction/RetrieveSparseEntity.java b/src/main/java/caosdb/server/database/backend/transaction/RetrieveSparseEntity.java
index df9f58671184fb70fca99ca7707f9270f75c87a1..a91b5021f1fa722a8e8831234fc8ebc2032ba115 100644
--- a/src/main/java/caosdb/server/database/backend/transaction/RetrieveSparseEntity.java
+++ b/src/main/java/caosdb/server/database/backend/transaction/RetrieveSparseEntity.java
@@ -4,6 +4,8 @@
*
* Copyright (C) 2018 Research Group Biomedical Physics,
* Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+ * Copyright (C) 2019 IndiScale GmbH
+ * Copyright (C) 2019 Timm Fitschen (t.fitschen@indiscale.com)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
@@ -22,30 +24,25 @@
*/
package caosdb.server.database.backend.transaction;
-import caosdb.server.CaosDBServer;
-import caosdb.server.ServerProperties;
+import caosdb.server.caching.Cache;
import caosdb.server.database.CacheableBackendTransaction;
import caosdb.server.database.DatabaseUtils;
import caosdb.server.database.backend.interfaces.RetrieveSparseEntityImpl;
import caosdb.server.database.exceptions.TransactionException;
-import caosdb.server.database.misc.Cache;
import caosdb.server.database.proto.SparseEntity;
import caosdb.server.entity.Entity;
import caosdb.server.entity.EntityInterface;
import caosdb.server.utils.EntityStatus;
-import org.apache.commons.jcs.access.CacheAccess;
+import org.apache.commons.jcs.access.behavior.ICacheAccess;
public class RetrieveSparseEntity extends CacheableBackendTransaction<Integer, SparseEntity> {
private final EntityInterface entity;
- private static final CacheAccess<Integer, SparseEntity> cache =
- Cache.getCache(
- "SparseEntityCache",
- Integer.valueOf(
- CaosDBServer.getServerProperty(ServerProperties.KEY_SPARSE_ENTITY_CACHE_CAPACITY)));
+ private static final ICacheAccess<Integer, SparseEntity> cache =
+ Cache.getCache("BACKEND_SparseEntities");
/**
- * To be called by UpdateSparseEntity and DeleteEntity on execution.
+ * To be called by {@link UpdateSparseEntity} and {@link DeleteEntity} on execution.
*
* @param id
*/
@@ -55,17 +52,13 @@ public class RetrieveSparseEntity extends CacheableBackendTransaction<Integer, S
}
}
- @Override
- protected CacheAccess<Integer, SparseEntity> getCache() {
- return cache;
- }
-
public RetrieveSparseEntity(final EntityInterface entity) {
+ super(cache);
this.entity = entity;
}
public RetrieveSparseEntity(final int id) {
- this.entity = new Entity(id);
+ this(new Entity(id));
}
@Override
diff --git a/src/main/java/caosdb/server/database/backend/transaction/RetrieveTransactionHistory.java b/src/main/java/caosdb/server/database/backend/transaction/RetrieveTransactionHistory.java
index cb9cdd9e993b9399f2bec5c9bff7316e0754b9ed..72bc1e5b1ec3a3509cc5cbba23e38b908d4e969f 100644
--- a/src/main/java/caosdb/server/database/backend/transaction/RetrieveTransactionHistory.java
+++ b/src/main/java/caosdb/server/database/backend/transaction/RetrieveTransactionHistory.java
@@ -4,6 +4,8 @@
*
* Copyright (C) 2018 Research Group Biomedical Physics,
* Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+ * Copyright (C) 2019 IndiScale GmbH
+ * Copyright (C) 2019 Timm Fitschen (t.fitschen@indiscale.com)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
@@ -23,17 +25,15 @@
package caosdb.server.database.backend.transaction;
import caosdb.datetime.UTCDateTime;
-import caosdb.server.database.CacheableBackendTransaction;
+import caosdb.server.database.BackendTransaction;
import caosdb.server.database.backend.interfaces.RetrieveTransactionHistoryImpl;
import caosdb.server.database.exceptions.TransactionException;
import caosdb.server.database.proto.ProtoTransactionLogMessage;
import caosdb.server.entity.EntityInterface;
import caosdb.server.utils.TransactionLogMessage;
import java.util.ArrayList;
-import org.apache.commons.jcs.access.CacheAccess;
-public class RetrieveTransactionHistory
- extends CacheableBackendTransaction<Integer, ArrayList<ProtoTransactionLogMessage>> {
+public class RetrieveTransactionHistory extends BackendTransaction {
private final EntityInterface entity;
@@ -42,25 +42,13 @@ public class RetrieveTransactionHistory
}
@Override
- protected Integer getKey() {
- return this.entity.getId();
- }
-
- @Override
- protected CacheAccess<Integer, ArrayList<ProtoTransactionLogMessage>> getCache() {
- return null;
- }
-
- @Override
- public ArrayList<ProtoTransactionLogMessage> executeNoCache() throws TransactionException {
+ protected void execute() {
final RetrieveTransactionHistoryImpl t =
getImplementation(RetrieveTransactionHistoryImpl.class);
- return t.execute(getKey());
+ process(t.execute(entity.getId()));
}
- @Override
- protected void process(final ArrayList<ProtoTransactionLogMessage> l)
- throws TransactionException {
+ private void process(final ArrayList<ProtoTransactionLogMessage> l) throws TransactionException {
for (final ProtoTransactionLogMessage t : l) {
final UTCDateTime dateTime = UTCDateTime.UTCSeconds(t.seconds, t.nanos);
diff --git a/src/main/java/caosdb/server/database/backend/transaction/RetrieveUser.java b/src/main/java/caosdb/server/database/backend/transaction/RetrieveUser.java
index 7492df38adfcfd24d18f1a263be5070a8916c9c8..9f27152030f5c953ee7bd4ecfff8c83d3cbc8a73 100644
--- a/src/main/java/caosdb/server/database/backend/transaction/RetrieveUser.java
+++ b/src/main/java/caosdb/server/database/backend/transaction/RetrieveUser.java
@@ -4,6 +4,8 @@
*
* Copyright (C) 2018 Research Group Biomedical Physics,
* Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+ * Copyright (C) 2019 IndiScale GmbH
+ * Copyright (C) 2019 Timm Fitschen (t.fitschen@indiscale.com)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
@@ -22,43 +24,29 @@
*/
package caosdb.server.database.backend.transaction;
-import caosdb.server.CaosDBServer;
-import caosdb.server.ServerProperties;
import caosdb.server.accessControl.Principal;
+import caosdb.server.caching.Cache;
import caosdb.server.database.CacheableBackendTransaction;
import caosdb.server.database.backend.interfaces.RetrieveUserImpl;
import caosdb.server.database.exceptions.TransactionException;
-import caosdb.server.database.misc.Cache;
import caosdb.server.database.proto.ProtoUser;
-import org.apache.commons.jcs.access.CacheAccess;
+import org.apache.commons.jcs.access.behavior.ICacheAccess;
public class RetrieveUser extends CacheableBackendTransaction<Principal, ProtoUser> {
private ProtoUser user;
- private static final CacheAccess<Principal, ProtoUser> cache =
- Cache.getCache(
- "UserAccountCache",
- Integer.valueOf(
- CaosDBServer.getServerProperty(ServerProperties.KEY_USER_ACCOUNT_CACHE_CAPACITY)));
+ private static final ICacheAccess<Principal, ProtoUser> cache = Cache.getCache("BACKEND_Users");
private final Principal principal;
- /**
- * To be called by DeleteSparseEntity, SetPassword, and UpdateSparseEntity on execution.
- *
- * @param u
- */
+ /** To be called by DeleteSparseEntity, SetPassword, and UpdateSparseEntity on execution. */
public static void removeCached(final Principal principal) {
if (principal != null && cache != null) {
cache.remove(principal);
}
}
- @Override
- protected CacheAccess<Principal, ProtoUser> getCache() {
- return cache;
- }
-
public RetrieveUser(final Principal principal) {
+ super(cache);
this.principal = principal;
}
diff --git a/src/main/java/caosdb/server/database/backend/transaction/RuleLoader.java b/src/main/java/caosdb/server/database/backend/transaction/RuleLoader.java
index 1a8698ac1d698946e7a3e239c03e386bc200e2b6..303029b42179a697c939767e2caf4d490a803a69 100644
--- a/src/main/java/caosdb/server/database/backend/transaction/RuleLoader.java
+++ b/src/main/java/caosdb/server/database/backend/transaction/RuleLoader.java
@@ -4,6 +4,8 @@
*
* Copyright (C) 2018 Research Group Biomedical Physics,
* Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+ * Copyright (C) 2019 IndiScale GmbH
+ * Copyright (C) 2019 Timm Fitschen (t.fitschen@indiscale.com)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
@@ -22,22 +24,22 @@
*/
package caosdb.server.database.backend.transaction;
-import caosdb.server.CaosDBServer;
-import caosdb.server.ServerProperties;
+import caosdb.server.caching.Cache;
import caosdb.server.database.CacheableBackendTransaction;
import caosdb.server.database.backend.interfaces.RuleLoaderImpl;
import caosdb.server.database.exceptions.TransactionException;
-import caosdb.server.database.misc.Cache;
import caosdb.server.database.proto.Rule;
import caosdb.server.entity.EntityInterface;
import caosdb.server.entity.container.TransactionContainer;
import caosdb.server.jobs.Job;
import caosdb.server.transaction.Transaction;
import java.util.ArrayList;
-import org.apache.commons.jcs.access.CacheAccess;
+import org.apache.commons.jcs.access.behavior.ICacheAccess;
public class RuleLoader extends CacheableBackendTransaction<String, ArrayList<Rule>> {
+ private static final ICacheAccess<String, ArrayList<Rule>> cache =
+ Cache.getCache("BACKEND_JobRules");
private final Transaction<? extends TransactionContainer> transaction;
private final EntityInterface e;
private final Integer entity;
@@ -49,6 +51,7 @@ public class RuleLoader extends CacheableBackendTransaction<String, ArrayList<Ru
final Integer entity,
final EntityInterface e,
final Transaction<? extends TransactionContainer> transaction) {
+ super(cache);
this.domain = domain;
this.entity = entity;
this.e = e;
@@ -66,17 +69,6 @@ public class RuleLoader extends CacheableBackendTransaction<String, ArrayList<Ru
+ ">";
}
- @Override
- protected CacheAccess<String, ArrayList<Rule>> getCache() {
- return rulesCache;
- }
-
- private static CacheAccess<String, ArrayList<Rule>> rulesCache =
- Cache.getCache(
- "RulesCache",
- Integer.valueOf(
- CaosDBServer.getServerProperty(ServerProperties.KEY_RULES_CACHE_CAPACITY)));
-
public ArrayList<Job> getJobs() {
return this.jobs;
}
diff --git a/src/main/java/caosdb/server/database/backend/transaction/UpdateSparseEntity.java b/src/main/java/caosdb/server/database/backend/transaction/UpdateSparseEntity.java
index 57d22ef226e83caf7e3b37b48b53c1025afaa93f..c6a1c7683409e0c875484e14553e4aeac2757a71 100644
--- a/src/main/java/caosdb/server/database/backend/transaction/UpdateSparseEntity.java
+++ b/src/main/java/caosdb/server/database/backend/transaction/UpdateSparseEntity.java
@@ -4,6 +4,8 @@
*
* Copyright (C) 2018 Research Group Biomedical Physics,
* Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+ * Copyright (C) 2019 IndiScale GmbH
+ * Copyright (C) 2019 Timm Fitschen (t.fitschen@indiscale.com)
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
@@ -39,6 +41,9 @@ public class UpdateSparseEntity extends BackendTransaction {
@Override
public void execute() throws TransactionException {
RetrieveSparseEntity.removeCached(this.entity.getId());
+ if (entity.hasFileProperties()) {
+ GetFileRecordByPath.removeCached(this.entity.getFileProperties().getPath());
+ }
final UpdateSparseEntityImpl t = getImplementation(UpdateSparseEntityImpl.class);
diff --git a/src/main/java/caosdb/server/database/misc/Cache.java b/src/main/java/caosdb/server/database/misc/Cache.java
deleted file mode 100644
index 7f364d4ab3e2ddc2d3e5e0638be4fe5dfa252ccd..0000000000000000000000000000000000000000
--- a/src/main/java/caosdb/server/database/misc/Cache.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * ** header v3.0
- * This file is a part of the CaosDB Project.
- *
- * Copyright (C) 2018 Research Group Biomedical Physics,
- * Max-Planck-Institute for Dynamics and Self-Organization Göttingen
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
- * ** end header
- */
-package caosdb.server.database.misc;
-
-import caosdb.server.CaosDBServer;
-import caosdb.server.ServerProperties;
-import caosdb.server.terminal.StatLabel;
-import caosdb.server.terminal.StatsPanel;
-import caosdb.server.utils.AbstractObservable;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.Serializable;
-import java.util.Properties;
-import org.apache.commons.jcs.JCS;
-import org.apache.commons.jcs.access.CacheAccess;
-
-public class Cache {
-
- static {
- final Properties p = new Properties();
- try {
- final InputStream is =
- new FileInputStream(CaosDBServer.getServerProperty(ServerProperties.KEY_CACHE_CONF_LOC));
- p.load(is);
- is.close();
- } catch (final FileNotFoundException e) {
- e.printStackTrace();
- } catch (final IOException e) {
- e.printStackTrace();
- }
- JCS.setConfigProperties(p);
- }
-
- public static <K, V extends Serializable> CacheAccess<K, V> getCache(
- final String name, final int capacity) {
- if (capacity > 0) {
- final CacheAccess<K, V> cache = JCS.getInstance(name);
- cache.getCacheAttributes().setMaxObjects(capacity);
- StatsPanel.addStat(
- name,
- new AbstractObservable() {
- @Override
- public String toString() {
- return cache.getStats();
- };
- });
- StatsPanel.addStat(
- name,
- new StatLabel(
- "Configuration", cache.getCacheAttributes().toString().replaceAll(",", "\n")));
-
- return cache;
- }
- return null;
- }
-}
diff --git a/src/main/java/caosdb/server/database/misc/TransactionBenchmark.java b/src/main/java/caosdb/server/database/misc/TransactionBenchmark.java
index 80b60e80893e8b512abededb345e286438ec22bb..f7c9e80f811418392b6567fcda5bfb8a9948bbff 100644
--- a/src/main/java/caosdb/server/database/misc/TransactionBenchmark.java
+++ b/src/main/java/caosdb/server/database/misc/TransactionBenchmark.java
@@ -27,49 +27,122 @@ import caosdb.server.ServerProperties;
import caosdb.server.utils.CronJob;
import caosdb.server.utils.Info;
import caosdb.server.utils.ServerStat;
+import java.io.Serializable;
import java.util.Date;
import java.util.HashMap;
import java.util.Locale;
+import java.util.Map;
import java.util.Map.Entry;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
import org.jdom2.Element;
-public class TransactionBenchmark implements ServerStat {
+class Counter implements Serializable {
+ private static final long serialVersionUID = 8679355597595634790L;
+ private int c;
- private static final long serialVersionUID = -8916163825450491067L;
- private long since = System.currentTimeMillis();
- private final HashMap<String, Integer> counts = new HashMap<String, Integer>();
- private final HashMap<String, Long> acc = new HashMap<String, Long>();
- private transient boolean synced = false;
- private static final TransactionBenchmark instance = new TransactionBenchmark();
- private static final boolean isActive =
- Boolean.valueOf(
- CaosDBServer.getServerProperty(ServerProperties.KEY_TRANSACTION_BENCHMARK_ENABLED)
- .toLowerCase());
+ public Counter(int initial) {
+ this.c = initial;
+ }
- static {
- if (isActive) {
- instance.init();
- }
+ public void add(int inc) {
+ this.c += inc;
+ }
+
+ public int toInteger() {
+ return c;
+ }
+}
+
+class Timer implements Serializable {
+ private static final long serialVersionUID = 7895352555775799409L;
+ private long c;
+
+ public Timer(long initial) {
+ this.c = initial;
+ }
+
+ public void add(long inc) {
+ this.c += inc;
+ }
+
+ public long toLong() {
+ return c;
+ }
+}
+
+class Measurement implements Serializable {
+ private static final long serialVersionUID = -2429348657382168470L;
+ private final Timer timer;
+ private final Counter counter;
+ private final String name;
+
+ public Measurement(String name, long time, int count) {
+ this.name = name;
+ this.timer = new Timer(time);
+ this.counter = new Counter(count);
+ }
+
+ public void add(long time, int count) {
+ this.timer.add(time);
+ this.counter.add(count);
+ }
+
+ public double getAvg() {
+ return (double) this.getAccTime() / this.getCount();
}
+ public StringBuilder toStringBuilder() {
+ StringBuilder sb = new StringBuilder();
+ sb.append('#');
+ sb.append(name);
+ sb.append(": ");
+ sb.append('(');
+ sb.append(this.getCount());
+ sb.append(", ");
+ sb.append(this.getAccTime());
+ sb.append(", ");
+ sb.append(String.format(Locale.US, "%3.2f", this.getAvg()));
+ sb.append(')');
+ return sb;
+ }
+
+ public int getCount() {
+ return counter.toInteger();
+ }
+
+ public long getAccTime() {
+ return timer.toLong();
+ }
+
+ public String getName() {
+ return this.name;
+ }
+}
+
+class RootBenchmark extends TransactionBenchmark implements ServerStat {
+
+ private static final long serialVersionUID = 8070554107783826602L;
+ private transient boolean synced = false;
/**
* Fetch old data (from before last shutdown) and fill it into this instance.
*
* @return
*/
- private TransactionBenchmark init() {
- synchronized (this.counts) {
+ protected TransactionBenchmark init() {
+ final RootBenchmark b = this;
+ synchronized (this.measurements) {
final Runnable updater =
new Runnable() {
@Override
public void run() {
try {
- synchronized (TransactionBenchmark.this.counts) {
- Info.syncDatabase(TransactionBenchmark.this);
+ synchronized (b.measurements) {
+ Info.getInstance().syncDatabase(b);
}
} catch (final Exception e) {
- e.printStackTrace();
+ logger.error(e);
}
}
};
@@ -81,38 +154,166 @@ public class TransactionBenchmark implements ServerStat {
return this;
}
+ @Override
+ public final String getName() {
+ return "TransactionBenchmark";
+ }
+
/**
- * Add a benchmark for a certain object. The object will be toString()'ed. The string serves as a
- * key.
+ * Add data from s to this instance.
*
- * @param object
- * @param time
+ * @param s another TransactionBenchmark instance
*/
- public void addBenchmark(final Object object, final long time) {
+ @Override
+ public final void update(final ServerStat s) {
+ if (!this.synced && isActive) {
+ final RootBenchmark t = (RootBenchmark) s;
+ this.synced = true;
+ merge(this, t);
+ }
+ }
+
+ public void merge(TransactionBenchmark target, TransactionBenchmark src) {
if (isActive) {
- addBenchmark(instance, object.toString(), time, 1);
+ target.since = Long.min(target.since, src.since);
+ synchronized (target.measurements) {
+ for (final Measurement m : src.measurements.values()) {
+ // add all measurements from src
+ target.addMeasurement(m);
+ }
+ }
+
+ // merge subBenchmarks
+ synchronized (src.subBenchmarks) {
+ for (final SubBenchmark srcsb : src.getSubBenchmarks()) {
+ synchronized (target.subBenchmarks) {
+ if (target.subBenchmarks.containsKey(srcsb.getName())) {
+ merge(target.subBenchmarks.get(srcsb.getName()), srcsb);
+ } else {
+ target.subBenchmarks.put(srcsb.getName(), srcsb);
+ }
+ }
+ }
+ }
}
}
+}
- private static void addBenchmark(
- final TransactionBenchmark b, final String name, final long time, final int count) {
- synchronized (b.counts) {
- if (b.counts.containsKey(name)) {
- int c = b.counts.get(name);
- c += count;
- b.counts.put(name, c);
- long a = b.acc.get(name);
- a += time;
- b.acc.put(name, a);
- } else {
- b.counts.put(name, count);
- b.acc.put(name, time);
+class SubBenchmark extends TransactionBenchmark {
+
+ private static final long serialVersionUID = 323147917189195973L;
+ private final String name;
+
+ public SubBenchmark(String name) {
+ this.name = name;
+ }
+
+ @Override
+ public String getName() {
+ return name;
+ }
+}
+
+class JdomConverter {
+ public Element convert(Measurement m) {
+ Element ret = new Element("Measurement");
+ ret.setAttribute("name", m.getName());
+ ret.setAttribute("acc_time", Long.toString(m.getAccTime()));
+ ret.setAttribute("avg_time", String.format(Locale.US, "%3.2f", m.getAvg()));
+ ret.setAttribute("count", Integer.toString(m.getCount()));
+ return ret;
+ }
+
+ public Element convert(TransactionBenchmark b) {
+ Element ret = new Element(b.getName());
+ ret.setAttribute("since", new Date(b.since).toString());
+ Iterable<Measurement> measurements = b.getMeasurements();
+
+ synchronized (measurements) {
+ for (Measurement m : measurements) {
+ ret.addContent(this.convert(m));
}
}
+
+ // tree view of subBenchmarks
+ Iterable<SubBenchmark> subBenchmarks = b.getSubBenchmarks();
+ synchronized (subBenchmarks) {
+ for (SubBenchmark s : subBenchmarks) {
+ ret.addContent(this.convert(s));
+ }
+ }
+ return ret;
+ }
+}
+
+public abstract class TransactionBenchmark implements Serializable {
+
+ private static final long serialVersionUID = -8916163825450491067L;
+ private static final TransactionBenchmark rootService = new RootBenchmark().init();
+ public final transient Logger logger = LogManager.getLogger(getClass());
+ protected static final transient boolean isActive =
+ Boolean.valueOf(
+ CaosDBServer.getServerProperty(ServerProperties.KEY_TRANSACTION_BENCHMARK_ENABLED)
+ .toLowerCase());
+
+ long since = System.currentTimeMillis();
+
+ protected final Map<String, Measurement> measurements = new HashMap<>();
+ protected final Map<String, SubBenchmark> subBenchmarks = new HashMap<>();
+
+ protected TransactionBenchmark() {};
+
+ public Iterable<SubBenchmark> getSubBenchmarks() {
+ return this.subBenchmarks.values();
}
- public static TransactionBenchmark getInstance() {
- return instance;
+ public Iterable<Measurement> getMeasurements() {
+ return this.measurements.values();
+ }
+
+ /**
+ * Add a measurement. The object will be toString()'ed. The string serves as a key.
+ *
+ * @param object
+ * @param time
+ */
+ public void addMeasurement(final Object object, final long time) {
+ this.addMeasurement(object, time, 1);
+ }
+
+ /**
+ * Add a measurement. The object will be toString()'ed. The string serves as a key.
+ *
+ * @param object
+ * @param time
+ * @param count
+ */
+ public void addMeasurement(final Object object, final long time, int count) {
+ if (isActive) {
+ String name = object.toString();
+
+ synchronized (this.measurements) {
+ Measurement existing = this.measurements.get(name);
+ if (existing == null) {
+ this.measurements.put(name, new Measurement(name, time, count));
+ } else {
+ existing.add(time, count);
+ }
+ }
+ }
+ }
+
+ /**
+ * Add a measurement.
+ *
+ * @param measurement
+ */
+ public void addMeasurement(Measurement measurement) {
+ this.addMeasurement(measurement.getName(), measurement.getAccTime(), measurement.getCount());
+ }
+
+ public static TransactionBenchmark getRootInstance() {
+ return rootService;
}
@Override
@@ -121,23 +322,11 @@ public class TransactionBenchmark implements ServerStat {
final StringBuilder sb = new StringBuilder();
sb.append(
"\nNAME: (execution count, accumulated execution time [ms], avg time per execution [ms])\n");
- synchronized (this.counts) {
- for (final Entry<String, Integer> e : this.counts.entrySet()) {
- final int c = e.getValue();
- final long acc = this.acc.get(e.getKey());
- final String name = e.getKey();
- final double avg = (double) acc / c;
+ synchronized (this.measurements) {
+ for (final Entry<String, Measurement> e : this.measurements.entrySet()) {
+ final Measurement m = e.getValue();
sb.append('\n');
- sb.append('#');
- sb.append(name);
- sb.append(": ");
- sb.append('(');
- sb.append(c);
- sb.append(", ");
- sb.append(acc);
- sb.append(", ");
- sb.append(String.format(Locale.US, "%3.2f", avg));
- sb.append(')');
+ sb.append(m.toStringBuilder());
}
}
return sb.toString();
@@ -147,73 +336,54 @@ public class TransactionBenchmark implements ServerStat {
}
public Element toElememt() {
- final Element ret = new Element("TransactionBenchmark");
if (isActive) {
- ret.setAttribute("since", new Date(this.since).toString());
- synchronized (this.counts) {
- for (final Entry<String, Integer> e : this.counts.entrySet()) {
- final int c = e.getValue();
- final long acc = TransactionBenchmark.this.acc.get(e.getKey());
- final String name = e.getKey();
- final double avg = (double) acc / c;
-
- final Element b = new Element("Benchmark");
- b.setAttribute("name", name);
- b.setAttribute("execution_count", Integer.toString(c));
- b.setAttribute("accumulated_execution_time", Long.toString(acc) + "ms");
- b.setAttribute("avg_execution_time", String.format(Locale.US, "%3.2f", avg) + "ms");
-
- ret.addContent(b);
- }
- }
+ return new JdomConverter().convert(this).setName("TransactionBenchmark");
} else {
+ final Element ret = new Element("TransactionBenchmark");
ret.setAttribute("info", "TransactionBenchmark is disabled.");
+ return ret;
}
- return ret;
}
- @Override
- public String getName() {
- return this.getClass().getSimpleName();
- }
+ public abstract String getName();
/**
- * Add data from s to this instance
+ * Create a new independent {@link TransactionBenchmark}.
*
- * @param s another TransactionBenchmark instance
+ * <p>This is for starting a series of measurements from a clean slate.
+ *
+ * <p>However, the Measurements and Benchmarks are still recorded in the {@link RootBenchmark}
+ * too.
+ *
+ * @param name the name of the new {@link TransactionBenchmark}.
+ * @return a new {@link TransactionBenchmark}
*/
- @Override
- public void update(final ServerStat s) {
- final TransactionBenchmark t = (TransactionBenchmark) s;
- if (!this.synced) {
- this.synced = true;
- this.since = t.since;
- for (final String key : t.counts.keySet()) {
- addBenchmark(this, key, t.acc.get(key), t.counts.get(key));
- }
+ public TransactionBenchmark createBenchmark(String name) {
+ synchronized (subBenchmarks) {
+ SubBenchmark b = new SubBenchmark(name);
+ subBenchmarks.put(name, b);
+ return b;
}
}
- /**
- * Get a SubBenchmark which can be used to produce a separate benchmark (e.g. for single
- * transactions). The SubBenchmark's data will also available to the main benchmark.
- *
- * @return A SubBenchmark
- */
- public static TransactionBenchmark getSubBenchmark() {
- return new SubBenchmark();
+ public TransactionBenchmark getBenchmark(String name) {
+ synchronized (subBenchmarks) {
+ SubBenchmark existing = subBenchmarks.get(name);
+ if (existing != null) {
+ return existing;
+ } else {
+ SubBenchmark b = new SubBenchmark(name);
+ subBenchmarks.put(name, b);
+ return b;
+ }
+ }
}
- private static class SubBenchmark extends TransactionBenchmark {
-
- private static final long serialVersionUID = 1L;
+ public TransactionBenchmark getBenchmark(Class<?> class1) {
+ return this.getBenchmark(class1.getSimpleName());
+ }
- @Override
- public void addBenchmark(final Object object, final long time) {
- if (isActive) {
- super.addBenchmark(object, time);
- TransactionBenchmark.addBenchmark(this, object.toString(), time, 1);
- }
- }
+ public TransactionBenchmark createBenchmark(Class<?> class1) {
+ return this.createBenchmark(class1.getSimpleName());
}
}
diff --git a/src/main/java/caosdb/server/entity/Message.java b/src/main/java/caosdb/server/entity/Message.java
index 03e788251620b6c192eb86f1bbd172564a66235f..0ffedb4316fe372e09f3b69ca5824a72f98accaf 100644
--- a/src/main/java/caosdb/server/entity/Message.java
+++ b/src/main/java/caosdb/server/entity/Message.java
@@ -79,8 +79,8 @@ public class Message extends Exception implements Comparable<Message>, ToElement
this(type, code, null, null);
}
- public Message(final Integer code, final String description) {
- this("Message", code, description, null);
+ public Message(Integer code, String description) {
+ this(MessageType.Info, code, description);
}
public Message(final MessageType type, final Integer code, final String description) {
@@ -96,6 +96,10 @@ public class Message extends Exception implements Comparable<Message>, ToElement
this(type, code, description, null);
}
+ public Message(MessageType type, String description) {
+ this(type.toString(), 0, description);
+ }
+
public Message(
final String type, final Integer code, final String description, final String body) {
this.code = code;
diff --git a/src/main/java/caosdb/server/entity/Role.java b/src/main/java/caosdb/server/entity/Role.java
index 58cd7ba4f19ff89dad10a0a1c18fc3a417274255..79b6548f82f41568ec44c574c4ad35502379e58b 100644
--- a/src/main/java/caosdb/server/entity/Role.java
+++ b/src/main/java/caosdb/server/entity/Role.java
@@ -22,7 +22,6 @@
*/
package caosdb.server.entity;
-import caosdb.server.database.Database;
import caosdb.server.database.access.Access;
import caosdb.server.database.backend.transaction.GetIDByName;
import caosdb.server.entity.xml.EntityToElementStrategy;
@@ -44,10 +43,16 @@ public enum Role {
ids = new HashMap<Role, Integer>();
for (final Role r : Role.values()) {
- ids.put(r, Database.execute(new GetIDByName(r.name(), "ROLE"), access).getId());
+ ids.put(r, execute(new GetIDByName(r.name(), "ROLE"), access).getId());
}
}
+ private static GetIDByName execute(GetIDByName getIDByName, Access access) {
+ getIDByName.setAccess(access);
+ getIDByName.executeTransaction();
+ return getIDByName;
+ }
+
public static Role parse(final String str) {
for (final Role r : Role.values()) {
if (r.toString().equalsIgnoreCase(str)) {
diff --git a/src/main/java/caosdb/server/entity/container/TransactionContainer.java b/src/main/java/caosdb/server/entity/container/TransactionContainer.java
index ff799a4df5eb74a0d15e48a990e735af609925ec..ea4cc02e7d16bda586ff3c700e3f2fb237b15426 100644
--- a/src/main/java/caosdb/server/entity/container/TransactionContainer.java
+++ b/src/main/java/caosdb/server/entity/container/TransactionContainer.java
@@ -103,15 +103,15 @@ public class TransactionContainer extends Container<Entity> implements ToElement
@Override
public void addToElement(final Element element) {
element.setAttribute("count", Integer.toString(size()));
+ if (this.benchmark != null && CaosDBServer.isDebugMode()) {
+ element.addContent(this.benchmark.toElememt());
+ }
for (final ToElementable m : this.messages) {
m.addToElement(element);
}
for (final EntityInterface entity : this) {
entity.addToElement(element);
}
- if (this.benchmark != null && CaosDBServer.isDebugMode()) {
- element.addContent(this.benchmark.toElememt());
- }
}
public void print() {
@@ -180,7 +180,7 @@ public class TransactionContainer extends Container<Entity> implements ToElement
public TransactionBenchmark getTransactionBenchmark() {
if (this.benchmark == null) {
- this.benchmark = TransactionBenchmark.getSubBenchmark();
+ this.benchmark = TransactionBenchmark.getRootInstance().createBenchmark(getClass());
}
return this.benchmark;
}
diff --git a/src/main/java/caosdb/server/jobs/Job.java b/src/main/java/caosdb/server/jobs/Job.java
index 681098b76a9deb90bc05dbea6f13ad2acd937ea6..b3551b4dd92084e6296a57be4166b341efa68262 100644
--- a/src/main/java/caosdb/server/jobs/Job.java
+++ b/src/main/java/caosdb/server/jobs/Job.java
@@ -23,7 +23,7 @@
package caosdb.server.jobs;
import caosdb.server.CaosDBException;
-import caosdb.server.database.Database;
+import caosdb.server.database.BackendTransaction;
import caosdb.server.database.backend.transaction.GetIDByName;
import caosdb.server.database.backend.transaction.IsSubType;
import caosdb.server.database.backend.transaction.RetrieveFullEntity;
@@ -78,6 +78,10 @@ public abstract class Job extends AbstractObservable implements Observer {
return getTransaction().getTransactor();
}
+ public <K extends BackendTransaction> K execute(final K t) {
+ return getTransaction().execute(t, getTransaction().getAccess());
+ }
+
protected ScheduledJob appendJob(final EntityInterface entity, final Class<? extends Job> clazz) {
try {
final Job job = clazz.newInstance();
@@ -156,8 +160,7 @@ public abstract class Job extends AbstractObservable implements Observer {
}
protected final boolean isValidSubTypeNoCache(final int child, final int parent) {
- return child == parent
- || Database.execute(new IsSubType(child, parent), getTransaction().getAccess()).isSubType();
+ return child == parent || execute(new IsSubType(child, parent)).isSubType();
}
protected final EntityInterface retrieveValidSparseEntityByName(final String name)
@@ -166,8 +169,7 @@ public abstract class Job extends AbstractObservable implements Observer {
}
protected final EntityInterface retrieveValidSparseEntityById(final Integer id) throws Message {
- final EntityInterface ret =
- Database.execute(new RetrieveSparseEntity(id), getTransaction().getAccess()).getEntity();
+ final EntityInterface ret = execute(new RetrieveSparseEntity(id)).getEntity();
if (ret.getEntityStatus() == EntityStatus.NONEXISTENT) {
throw ServerMessages.ENTITY_DOES_NOT_EXIST;
}
@@ -175,17 +177,15 @@ public abstract class Job extends AbstractObservable implements Observer {
}
protected final EntityInterface retrieveValidEntity(Integer id) {
- return Database.execute(new RetrieveFullEntity(id), getTransaction().getAccess())
- .getContainer()
- .get(0);
+ return execute(new RetrieveFullEntity(id)).getContainer().get(0);
}
protected final Integer retrieveValidIDByName(final String name) {
- return Database.execute(new GetIDByName(name), getTransaction().getAccess()).getId();
+ return execute(new GetIDByName(name)).getId();
}
protected EntityInterface retrieveParentsOfValidEntity(final EntityInterface entity) {
- Database.execute(new RetrieveParents(entity), getTransaction().getAccess());
+ execute(new RetrieveParents(entity));
return entity;
}
@@ -257,16 +257,16 @@ public abstract class Job extends AbstractObservable implements Observer {
}
protected List<Job> loadDataTypeSpecificJobs() {
- return Job.loadDataTypeSpecificJobs(getEntity(), getTransaction());
+ return loadDataTypeSpecificJobs(getEntity(), getTransaction());
}
- public static List<Job> loadDataTypeSpecificJobs(
+ public List<Job> loadDataTypeSpecificJobs(
final EntityInterface entity, final Transaction<? extends TransactionContainer> transaction) {
return loadDataTypeSpecificJobs(entity.getDatatype(), entity, transaction);
}
- private static List<Job> loadDataTypeSpecificJobs(
+ private List<Job> loadDataTypeSpecificJobs(
final AbstractDatatype dt,
final EntityInterface entity,
final Transaction<? extends TransactionContainer> transaction) {
@@ -275,19 +275,19 @@ public abstract class Job extends AbstractObservable implements Observer {
}
if (dt instanceof ReferenceDatatype2) {
final RuleLoader t = new RuleLoader(0, 17, entity, transaction);
- return Database.execute(t, transaction.getAccess()).getJobs();
+ return execute(t).getJobs();
} else if (dt instanceof AbstractCollectionDatatype) {
final AbstractDatatype datatype = ((AbstractCollectionDatatype) dt).getDatatype();
return loadDataTypeSpecificJobs(datatype, entity, transaction);
} else if (dt.getId() != null) {
final RuleLoader t = new RuleLoader(0, dt.getId(), entity, transaction);
- return Database.execute(t, transaction.getAccess()).getJobs();
+ return execute(t).getJobs();
} else {
return null;
}
}
- public static List<Job> loadStandardJobs(
+ public List<Job> loadStandardJobs(
final EntityInterface entity, final Transaction<? extends TransactionContainer> transaction) {
final ArrayList<Job> jobs = new ArrayList<>();
@@ -302,13 +302,13 @@ public abstract class Job extends AbstractObservable implements Observer {
// load general rules
{
final RuleLoader t = new RuleLoader(0, 0, entity, transaction);
- jobs.addAll(Database.execute(t, transaction.getAccess()).getJobs());
+ jobs.addAll(execute(t).getJobs());
}
// load Role specific rules
if (entity.hasRole()) {
final RuleLoader t = new RuleLoader(0, entity.getRole().getId(), entity, transaction);
- jobs.addAll(Database.execute(t, transaction.getAccess()).getJobs());
+ jobs.addAll(execute(t).getJobs());
}
// load data type specific rules
@@ -336,7 +336,7 @@ public abstract class Job extends AbstractObservable implements Observer {
}
}
- public static List<Job> loadJobs(
+ public List<Job> loadJobs(
final EntityInterface entity, final Transaction<? extends TransactionContainer> transaction) {
final LinkedList<Job> jobs = new LinkedList<>();
diff --git a/src/main/java/caosdb/server/jobs/Schedule.java b/src/main/java/caosdb/server/jobs/Schedule.java
index 367fdcbd13c5830f46205ef05d386f98bb14492d..4e7f005f3d2f790008a991a0138d62013f6a545a 100644
--- a/src/main/java/caosdb/server/jobs/Schedule.java
+++ b/src/main/java/caosdb/server/jobs/Schedule.java
@@ -55,7 +55,7 @@ class ScheduledJob {
this.job
.getContainer()
.getTransactionBenchmark()
- .addBenchmark(this.job.getClass().getSimpleName(), this.runtime);
+ .addMeasurement(this.job.getClass().getSimpleName(), this.runtime);
}
void pause() {
diff --git a/src/main/java/caosdb/server/jobs/core/AccessControl.java b/src/main/java/caosdb/server/jobs/core/AccessControl.java
index 8b51cd582933d9c993c2427be7c399e7dc553df2..0dd2ef4740d1a17d6e6ed8b0cc04c8d66a56c918 100644
--- a/src/main/java/caosdb/server/jobs/core/AccessControl.java
+++ b/src/main/java/caosdb/server/jobs/core/AccessControl.java
@@ -22,7 +22,6 @@
*/
package caosdb.server.jobs.core;
-import caosdb.server.database.Database;
import caosdb.server.database.backend.transaction.RetrieveSparseEntity;
import caosdb.server.entity.EntityInterface;
import caosdb.server.entity.wrapper.Parent;
@@ -66,7 +65,7 @@ public class AccessControl extends ContainerJob {
if (e.hasParents() && e.getParents().size() == 1) {
final Parent par1 = e.getParents().get(0);
if (par1.hasId() && par1.getId() > 0) {
- Database.execute(new RetrieveSparseEntity(par1), getTransaction().getAccess());
+ execute(new RetrieveSparseEntity(par1));
}
if (par1.hasName()
&& par1.getName().equals("CommentAnnotation")
diff --git a/src/main/java/caosdb/server/jobs/core/CheckChildDependencyExistent.java b/src/main/java/caosdb/server/jobs/core/CheckChildDependencyExistent.java
index 407c6e549bdcb8968a7ca91093534574ec6e5e2e..a3d9f4e493b189e7fb49de69347c5b6149f9f885 100644
--- a/src/main/java/caosdb/server/jobs/core/CheckChildDependencyExistent.java
+++ b/src/main/java/caosdb/server/jobs/core/CheckChildDependencyExistent.java
@@ -22,7 +22,6 @@
*/
package caosdb.server.jobs.core;
-import caosdb.server.database.Database;
import caosdb.server.database.backend.transaction.GetChildren;
import caosdb.server.entity.EntityInterface;
import caosdb.server.jobs.EntityJob;
@@ -42,9 +41,7 @@ public class CheckChildDependencyExistent extends EntityJob {
public final void run() {
if (getEntity().getDomain() == null || getEntity().getDomain() == 0) {
- final List<Integer> children =
- Database.execute(new GetChildren(getEntity().getId()), getTransaction().getAccess())
- .getList();
+ final List<Integer> children = execute(new GetChildren(getEntity().getId())).getList();
// loop:
for (final Integer id : children) {
diff --git a/src/main/java/caosdb/server/jobs/core/CheckFileStorageConsistency.java b/src/main/java/caosdb/server/jobs/core/CheckFileStorageConsistency.java
index b74a0319c981753a52d5fb962f9bf3add3347ba1..5af12ad71b4b3c5b12ac8321131b4d9cb29013d5 100644
--- a/src/main/java/caosdb/server/jobs/core/CheckFileStorageConsistency.java
+++ b/src/main/java/caosdb/server/jobs/core/CheckFileStorageConsistency.java
@@ -28,9 +28,9 @@ import caosdb.server.database.exceptions.TransactionException;
import caosdb.server.entity.Message;
import caosdb.server.jobs.FlagJob;
import caosdb.server.jobs.JobAnnotation;
+import caosdb.server.transaction.FileStorageConsistencyCheck;
import caosdb.server.transaction.Retrieve;
import caosdb.server.transaction.Transaction;
-import caosdb.server.utils.FileStorageConsistencyCheck;
import caosdb.server.utils.FileUtils;
import caosdb.server.utils.Observable;
import caosdb.server.utils.Observer;
diff --git a/src/main/java/caosdb/server/jobs/core/CheckParOblPropPresent.java b/src/main/java/caosdb/server/jobs/core/CheckParOblPropPresent.java
index ead4248f63fe5af4bf6e35778ecc34c7e3459da4..bee056c5763593ce6ef6a71f713019fae30a8bb5 100644
--- a/src/main/java/caosdb/server/jobs/core/CheckParOblPropPresent.java
+++ b/src/main/java/caosdb/server/jobs/core/CheckParOblPropPresent.java
@@ -22,7 +22,6 @@
*/
package caosdb.server.jobs.core;
-import caosdb.server.database.Database;
import caosdb.server.database.backend.transaction.RetrieveFullEntity;
import caosdb.server.entity.EntityInterface;
import caosdb.server.entity.StatementStatus;
@@ -49,8 +48,7 @@ public class CheckParOblPropPresent extends EntityJob {
runJobFromSchedule(getEntity(), CheckParValid.class);
if (getEntity().hasParents()) {
- Database.execute(
- new RetrieveFullEntity(getEntity().getParents()), getTransaction().getAccess());
+ execute(new RetrieveFullEntity(getEntity().getParents()));
}
// inherit properties
diff --git a/src/main/java/caosdb/server/jobs/core/CheckReferenceDependencyExistent.java b/src/main/java/caosdb/server/jobs/core/CheckReferenceDependencyExistent.java
index eec99f5ed00384d4be447a7ef3cd51b19178dccf..fd5eaab0f0d302e9b5586693ef502c32d607d773 100644
--- a/src/main/java/caosdb/server/jobs/core/CheckReferenceDependencyExistent.java
+++ b/src/main/java/caosdb/server/jobs/core/CheckReferenceDependencyExistent.java
@@ -22,7 +22,6 @@
*/
package caosdb.server.jobs.core;
-import caosdb.server.database.Database;
import caosdb.server.database.backend.transaction.GetDependentEntities;
import caosdb.server.entity.EntityInterface;
import caosdb.server.jobs.EntityJob;
@@ -45,9 +44,7 @@ public class CheckReferenceDependencyExistent extends EntityJob {
// retrieve dependent entities
final List<Integer> depends =
- Database.execute(
- new GetDependentEntities(getEntity().getId()), getTransaction().getAccess())
- .getList();
+ execute(new GetDependentEntities(getEntity().getId())).getList();
// loop:
for (final Integer id : depends) {
diff --git a/src/main/java/caosdb/server/jobs/core/CheckTargetPathValid.java b/src/main/java/caosdb/server/jobs/core/CheckTargetPathValid.java
index 3dc0c4bba460dfa7f6eb2eec338e5dea738da9e2..404e6b3a0d82950061ba4168a7a6f32e5f847400 100644
--- a/src/main/java/caosdb/server/jobs/core/CheckTargetPathValid.java
+++ b/src/main/java/caosdb/server/jobs/core/CheckTargetPathValid.java
@@ -55,7 +55,8 @@ public class CheckTargetPathValid extends EntityJob {
}
try {
- FileSystem.checkTarget(getEntity(), getTransaction().getAccess());
+ FileSystem.checkTarget(
+ getEntity(), getTransaction().getAccess(), getTransaction().getTransactionBenchmark());
} catch (final Message m) {
getEntity().setEntityStatus(EntityStatus.UNQUALIFIED);
getEntity().addError(m);
diff --git a/src/main/java/caosdb/server/jobs/core/History.java b/src/main/java/caosdb/server/jobs/core/History.java
index 7e7643040196333b923536b13d8b84ae6b962311..ee72413a46e65f8b8a020315e3c2e93c7b8ccd97 100644
--- a/src/main/java/caosdb/server/jobs/core/History.java
+++ b/src/main/java/caosdb/server/jobs/core/History.java
@@ -22,7 +22,6 @@
*/
package caosdb.server.jobs.core;
-import caosdb.server.database.Database;
import caosdb.server.database.backend.transaction.RetrieveTransactionHistory;
import caosdb.server.entity.EntityInterface;
import caosdb.server.jobs.FlagJob;
@@ -44,7 +43,7 @@ public class History extends FlagJob {
try {
entity.checkPermission(EntityPermission.RETRIEVE_HISTORY);
final RetrieveTransactionHistory t = new RetrieveTransactionHistory(entity);
- Database.execute(t, getTransaction().getAccess());
+ execute(t);
} catch (final AuthorizationException e) {
entity.setEntityStatus(EntityStatus.UNQUALIFIED);
entity.addError(ServerMessages.AUTHORIZATION_ERROR);
diff --git a/src/main/java/caosdb/server/jobs/core/Inheritance.java b/src/main/java/caosdb/server/jobs/core/Inheritance.java
index ed25c64cbf02d16566addc79961e36e0fdffad81..f7149eb3c74270012db011385e17bbf01fdd01a2 100644
--- a/src/main/java/caosdb/server/jobs/core/Inheritance.java
+++ b/src/main/java/caosdb/server/jobs/core/Inheritance.java
@@ -22,7 +22,6 @@
*/
package caosdb.server.jobs.core;
-import caosdb.server.database.Database;
import caosdb.server.database.backend.transaction.RetrieveFullEntity;
import caosdb.server.entity.Entity;
import caosdb.server.entity.EntityInterface;
@@ -74,7 +73,7 @@ public class Inheritance extends EntityJob {
runJobFromSchedule(getEntity(), CheckParValid.class);
- Database.execute(new RetrieveFullEntity(parent), getTransaction().getAccess());
+ execute(new RetrieveFullEntity(parent));
if (parent.hasProperties()) {
// loop over all properties of the parent and
@@ -163,7 +162,7 @@ public class Inheritance extends EntityJob {
}
}
} else {
- Database.execute(new RetrieveFullEntity(validProperty), getTransaction().getAccess());
+ execute(new RetrieveFullEntity(validProperty));
}
if (validProperty.getEntityStatus() == EntityStatus.VALID
diff --git a/src/main/java/caosdb/server/jobs/core/InsertFilesInDir.java b/src/main/java/caosdb/server/jobs/core/InsertFilesInDir.java
index 773ac27525e0b28a1a10f23f1a2ee9cfd9fbf799..6b3144eb08b471ea90b836978c76b4ff643c313d 100644
--- a/src/main/java/caosdb/server/jobs/core/InsertFilesInDir.java
+++ b/src/main/java/caosdb/server/jobs/core/InsertFilesInDir.java
@@ -22,6 +22,7 @@
*/
package caosdb.server.jobs.core;
+import caosdb.server.CaosDBException;
import caosdb.server.CaosDBServer;
import caosdb.server.FileSystem;
import caosdb.server.ServerProperties;
@@ -61,6 +62,9 @@ public class InsertFilesInDir extends FlagJob {
private Pattern include = null;
private Pattern exclude = null;
private boolean forceSymLinks = false;
+ private Pattern valueParser =
+ Pattern.compile(
+ "(?:(?:-p\\s*([^\\s]*?)\\s+)|(?:-i\\s*([^\\s]*?)\\s+)|(?:-e\\s*([^\\s]*?)\\s+)|(--force-allow-symlinks\\s+))|([^-].*)");
/**
* @return a List of directories which subdirs are allowed to be batch-added. Needs to be
@@ -85,14 +89,10 @@ public class InsertFilesInDir extends FlagJob {
return ret;
}
- @Override
- protected void job(final String value) {
+ public String parseValue(String value) {
- String dirStr = value;
- final Pattern pattern =
- Pattern.compile(
- "(?:(?:-p\\s*([^\\s]*?)\\s+)|(?:-i\\s*([^\\s]*?)\\s+)|(?:-e\\s*([^\\s]*?)\\s+)|(--force-allow-symlinks\\s+))|([^-].*)");
- final Matcher matcher = pattern.matcher(value);
+ String ret = value;
+ final Matcher matcher = valueParser.matcher(value);
while (matcher.find()) {
if (matcher.group(1) != null) {
this.prefix = matcher.group(1).replaceFirst("/$", "") + "/";
@@ -107,9 +107,16 @@ public class InsertFilesInDir extends FlagJob {
this.forceSymLinks = true;
}
if (matcher.group(5) != null) {
- dirStr = matcher.group(5);
+ ret = matcher.group(5);
}
}
+ return ret;
+ }
+
+ @Override
+ protected void job(final String value) {
+
+ String dirStr = parseValue(value);
final File dir = new File(dirStr);
@@ -153,7 +160,7 @@ public class InsertFilesInDir extends FlagJob {
try {
final Undoable delete = FileUtils.delete(InsertFilesInDir.this.tmp, true);
delete.cleanUp();
- } catch (final IOException e) {
+ } catch (final IOException | CaosDBException | InterruptedException e) {
e.printStackTrace();
}
}
@@ -163,7 +170,7 @@ public class InsertFilesInDir extends FlagJob {
try {
final Undoable delete = FileUtils.delete(InsertFilesInDir.this.tmp, true);
delete.cleanUp();
- } catch (final IOException e) {
+ } catch (final IOException | CaosDBException | InterruptedException e) {
e.printStackTrace();
}
}
@@ -223,7 +230,10 @@ public class InsertFilesInDir extends FlagJob {
// add create symlink and file record to this
// container if the target
// path is allowed
- if (FileSystem.checkTarget(newFileEntity, getTransaction().getAccess())
+ if (FileSystem.checkTarget(
+ newFileEntity,
+ getTransaction().getAccess(),
+ getTransaction().getTransactionBenchmark())
&& newFileEntity.getEntityStatus() != EntityStatus.UNQUALIFIED) {
final File link =
@@ -243,24 +253,32 @@ public class InsertFilesInDir extends FlagJob {
return i;
}
+ boolean isExcluded(File f) throws IOException {
+ return this.exclude != null && this.exclude.matcher(f.getCanonicalPath()).find();
+ }
+
+ boolean isNotIncluded(File f) throws IOException {
+ return this.include != null && !this.include.matcher(f.getCanonicalPath()).find();
+ }
+
private boolean shouldBeProcessed(final File sub) throws IOException {
- if (this.include != null && !this.include.matcher(sub.getCanonicalPath()).matches()) {
- getContainer()
- .addMessage(
- new Message(
- MessageType.Warning,
- 1,
- "Not explicitly included directory or file: " + sub.getCanonicalPath()));
- return false;
- }
- if (this.exclude != null && this.exclude.matcher(sub.getCanonicalPath()).matches()) {
- getContainer()
- .addMessage(
- new Message(
- MessageType.Warning,
- 2,
- "Explicitly excluded directory or file: " + sub.getCanonicalPath()));
- return false;
+ if (sub.isFile()) {
+ if (this.isNotIncluded(sub)) {
+ getContainer()
+ .addMessage(
+ new Message(
+ MessageType.Warning,
+ 1,
+ "Not explicitly included file: " + sub.getCanonicalPath()));
+ return false;
+ }
+ if (this.isExcluded(sub)) {
+ getContainer()
+ .addMessage(
+ new Message(
+ MessageType.Warning, 2, "Explicitly excluded file: " + sub.getCanonicalPath()));
+ return false;
+ }
}
if (sub.isHidden()) {
getContainer()
@@ -300,7 +318,7 @@ public class InsertFilesInDir extends FlagJob {
}
private void loadJobs(final Entity e) {
- final List<Job> loadJobs = Job.loadJobs(e, getTransaction());
+ final List<Job> loadJobs = loadJobs(e, getTransaction());
getTransaction().getSchedule().addAll(loadJobs);
}
diff --git a/src/main/java/caosdb/server/jobs/core/ResolveNames.java b/src/main/java/caosdb/server/jobs/core/ResolveNames.java
index fd5498f468e44801279655ba21a1702da12c168e..f5e1695f34c30a78f136950f7aae3ae00be170e4 100644
--- a/src/main/java/caosdb/server/jobs/core/ResolveNames.java
+++ b/src/main/java/caosdb/server/jobs/core/ResolveNames.java
@@ -22,7 +22,6 @@
*/
package caosdb.server.jobs.core;
-import caosdb.server.database.Database;
import caosdb.server.database.backend.transaction.GetIDByName;
import caosdb.server.database.exceptions.EntityDoesNotExistException;
import caosdb.server.entity.Entity;
@@ -45,9 +44,7 @@ public class ResolveNames extends ContainerJob {
for (final Entity e : container) {
if (e.hasName() && !e.hasId()) {
try {
- final List<Integer> c =
- Database.execute(new GetIDByName(e.getName(), false), getTransaction().getAccess())
- .getList();
+ final List<Integer> c = execute(new GetIDByName(e.getName(), false)).getList();
e.setId(c.get(0));
e.setEntityStatus(EntityStatus.QUALIFIED);
diff --git a/src/main/java/caosdb/server/jobs/core/RetrieveAllJob.java b/src/main/java/caosdb/server/jobs/core/RetrieveAllJob.java
index f522986c6b40ed73fdbb752226a7dd66de4d040e..0fba1223a3bd0e984c5c2b6ff43dd58070b5bbc5 100644
--- a/src/main/java/caosdb/server/jobs/core/RetrieveAllJob.java
+++ b/src/main/java/caosdb/server/jobs/core/RetrieveAllJob.java
@@ -22,7 +22,6 @@
*/
package caosdb.server.jobs.core;
-import caosdb.server.database.Database;
import caosdb.server.database.backend.transaction.RetrieveAll;
import caosdb.server.jobs.FlagJob;
import caosdb.server.jobs.JobAnnotation;
@@ -37,7 +36,7 @@ public class RetrieveAllJob extends FlagJob {
if (value == null) {
value = "ENTITY";
}
- Database.execute(new RetrieveAll(getContainer(), value), getTransaction().getAccess());
+ execute(new RetrieveAll(getContainer(), value));
}
}
}
diff --git a/src/main/java/caosdb/server/jobs/core/UpdateUnitConverters.java b/src/main/java/caosdb/server/jobs/core/UpdateUnitConverters.java
index 86f5b2d23721e97cef6e1b9ebd356cbf140b0cc6..8d0118b80582e3a65b613caae2200719af428819 100644
--- a/src/main/java/caosdb/server/jobs/core/UpdateUnitConverters.java
+++ b/src/main/java/caosdb/server/jobs/core/UpdateUnitConverters.java
@@ -22,7 +22,6 @@
*/
package caosdb.server.jobs.core;
-import caosdb.server.database.Database;
import caosdb.server.database.backend.transaction.InsertLinCon;
import caosdb.server.jobs.EntityJob;
import caosdb.server.jobs.JobAnnotation;
@@ -66,7 +65,7 @@ public class UpdateUnitConverters extends EntityJob {
if (signature_from != signature_to) {
final InsertLinCon t =
new InsertLinCon(signature_from, signature_to, a, b_dividend, b_divisor, c);
- Database.execute(t, getTransaction().getAccess());
+ execute(t);
}
}
}
diff --git a/src/main/java/caosdb/server/jobs/extension/AWIBoxLoan.java b/src/main/java/caosdb/server/jobs/extension/AWIBoxLoan.java
index 65c9689623fd266dc01d77ff7b8dc97eed1eff81..8df3bc2f8b798a8495f9bb670fa7ce23c5eba9c2 100644
--- a/src/main/java/caosdb/server/jobs/extension/AWIBoxLoan.java
+++ b/src/main/java/caosdb/server/jobs/extension/AWIBoxLoan.java
@@ -11,8 +11,6 @@ import caosdb.server.entity.Message;
import caosdb.server.entity.Message.MessageType;
import caosdb.server.entity.wrapper.Property;
import caosdb.server.jobs.JobAnnotation;
-import caosdb.server.jobs.core.CheckNoAdditionalPropertiesPresent;
-import caosdb.server.jobs.core.CheckNoOverridesPresent;
import caosdb.server.jobs.core.CheckPropValid;
import caosdb.server.permissions.EntityACL;
import caosdb.server.permissions.EntityACLFactory;
@@ -392,7 +390,9 @@ public class AWIBoxLoan extends AWIBoxLoanModel {
if (!isBoxRecord(e) || !hasOnlyAllowedBoxProperties4RequestLoan(e)) {
return false;
}
- appendJob(e, CheckNoAdditionalPropertiesPresent.class);
+ // TODO this breaks the box loan functionality if any other prior changes have been made to
+ // the box
+ // appendJob(e, CheckNoAdditionalPropertiesPresent.class);
}
return true;
}
@@ -440,8 +440,10 @@ public class AWIBoxLoan extends AWIBoxLoanModel {
setCuratorAsOwner(e);
}
setLoanRequestDate(e);
- appendJob(e, CheckNoAdditionalPropertiesPresent.class);
- appendJob(e, CheckNoOverridesPresent.class);
+ // TODO this check breaks the box loan functionality if any other changes have been made to
+ // the box entity
+ // appendJob(e, CheckNoAdditionalPropertiesPresent.class);
+ // appendJob(e, CheckNoOverridesPresent.class);
}
appendJob(AWIBoxLoanRequestLoanCuratorEmail.class);
return true;
@@ -476,8 +478,10 @@ public class AWIBoxLoan extends AWIBoxLoanModel {
&& isPersonRecord(getContainer().get(0))
&& checkUniqueName(getContainer().get(0))
&& checkEmail(getContainer().get(0))) {
- appendJob(getContainer().get(0), CheckNoAdditionalPropertiesPresent.class);
- appendJob(getContainer().get(0), CheckNoOverridesPresent.class);
+ // TODO this check breaks the box loan functionality if any other changes have been made to
+ // the box entity
+ // appendJob(getContainer().get(0), CheckNoAdditionalPropertiesPresent.class);
+ // appendJob(getContainer().get(0), CheckNoOverridesPresent.class);
logger.trace("isRequestReturnSetUser: true");
return true;
}
@@ -536,7 +540,7 @@ public class AWIBoxLoan extends AWIBoxLoanModel {
*/
boolean hasOnlyAllowedLoanProperties4RequestReturn(EntityInterface e) {
runJobFromSchedule(e, CheckPropValid.class);
- appendJob(e, CheckNoOverridesPresent.class);
+ // appendJob(e, CheckNoOverridesPresent.class);
boolean foundReturnRequested = false;
for (Property p : e.getProperties()) {
diff --git a/src/main/java/caosdb/server/permissions/EntityPermission.java b/src/main/java/caosdb/server/permissions/EntityPermission.java
index 7fae8a76a8cc87cbfb2525c34102d5418d58e94e..1747a55810cf9718abc9627cad8102f88b2a5cac 100644
--- a/src/main/java/caosdb/server/permissions/EntityPermission.java
+++ b/src/main/java/caosdb/server/permissions/EntityPermission.java
@@ -65,7 +65,7 @@ public class EntityPermission extends Permission {
"This bitNumber is too big. This implementation only handles bitNumbers up to 61.");
}
if (instances.contains(this)) {
- throw new CaosDBException("This EntityPermission is defined yet.");
+ throw new CaosDBException("This EntityPermission is defined already.");
} else {
instances.add(this);
}
diff --git a/src/main/java/caosdb/server/query/Backreference.java b/src/main/java/caosdb/server/query/Backreference.java
index f8d4f86f5d04d6b724a3fdec66c1a24d739726af..c39ce43923372811caf8607550a9852d1a518c63 100644
--- a/src/main/java/caosdb/server/query/Backreference.java
+++ b/src/main/java/caosdb/server/query/Backreference.java
@@ -192,7 +192,7 @@ public class Backreference implements EntityFilterInterface, QueryInterface {
addBenchmark("initBackreference()", t2 - t1);
if (this.entitiesTable != null) {
- Query.applyQueryTemplates(query, this.entitiesTable);
+ getQuery().applyQueryTemplates(query, this.entitiesTable);
addBenchmark("applyQueryTemplates()", System.currentTimeMillis() - t2);
}
}
diff --git a/src/main/java/caosdb/server/query/POV.java b/src/main/java/caosdb/server/query/POV.java
index 01d2cb1b9bb2bd51e18ccacc438ff8ac75788762..6a57caa780d76c5279a37eaba09d44c021215d7c 100644
--- a/src/main/java/caosdb/server/query/POV.java
+++ b/src/main/java/caosdb/server/query/POV.java
@@ -412,7 +412,7 @@ public class POV implements EntityFilterInterface {
query.addBenchmark(getClass().getSimpleName() + ".initPOVPropertiesTable()", t3 - t2);
if (this.refIdsTable != null) {
- Query.applyQueryTemplates(query, this.refIdsTable);
+ query.getQuery().applyQueryTemplates(query, this.refIdsTable);
query.addBenchmark(
getClass().getSimpleName() + ".applyQueryTemplates()", System.currentTimeMillis() - t3);
}
diff --git a/src/main/java/caosdb/server/query/Query.java b/src/main/java/caosdb/server/query/Query.java
index c9bff5a5f11aa5775582c613d80062419f9adb4c..a62a6acae9bcc413e7ab06c812312f8e8b8fe323 100644
--- a/src/main/java/caosdb/server/query/Query.java
+++ b/src/main/java/caosdb/server/query/Query.java
@@ -26,7 +26,6 @@ import static caosdb.server.database.DatabaseUtils.bytes2UTF8;
import caosdb.server.CaosDBServer;
import caosdb.server.ServerProperties;
-import caosdb.server.database.Database;
import caosdb.server.database.access.Access;
import caosdb.server.database.backend.implementation.MySQL.ConnectionException;
import caosdb.server.database.backend.implementation.MySQL.MySQLHelper;
@@ -284,7 +283,7 @@ public class Query implements QueryInterface, ToElementable, TransactionInterfac
* @throws SQLException
* @throws QueryException
*/
- public static void applyQueryTemplates(final QueryInterface query, final String resultSet)
+ public void applyQueryTemplates(final QueryInterface query, final String resultSet)
throws QueryException {
try {
final Map<Integer, String> queryTemplates = getQueryTemplates(query, resultSet);
@@ -300,7 +299,7 @@ public class Query implements QueryInterface, ToElementable, TransactionInterfac
// ... check for RETRIEVE:ENTITY permission...
final EntityInterface e =
- Database.execute(new RetrieveSparseEntity(q.getKey()), query.getAccess()).getEntity();
+ execute(new RetrieveSparseEntity(q.getKey()), query.getAccess()).getEntity();
final EntityACL entityACL = e.getEntityACL();
if (!entityACL.isPermitted(query.getUser(), EntityPermission.RETRIEVE_ENTITY)) {
// ... and ignore if not.
@@ -415,6 +414,7 @@ public class Query implements QueryInterface, ToElementable, TransactionInterfac
}
public void parse() throws ParsingException {
+ final long t1 = System.currentTimeMillis();
CQLLexer lexer;
lexer = new CQLLexer(CharStreams.fromString(this.query));
final CommonTokenStream tokens = new CommonTokenStream(lexer);
@@ -434,6 +434,10 @@ public class Query implements QueryInterface, ToElementable, TransactionInterfac
this.type = cq.t;
this.filter = cq.filter;
this.selections = cq.s;
+ final long t2 = System.currentTimeMillis();
+ if (t2 - t1 > 1000) {
+ addBenchmark("LONG_PARSING: " + this.query, t2 - t1);
+ }
}
private String executeStrategy() throws QueryException {
@@ -528,8 +532,6 @@ public class Query implements QueryInterface, ToElementable, TransactionInterfac
this.access = access;
}
- private final TransactionBenchmark benchmark = TransactionBenchmark.getSubBenchmark();
-
/**
* Filter out all entities which may not be retrieved by this user due to a missing RETRIEVE
* permission. This one is also designed for filtering of intermediate results.
@@ -539,7 +541,7 @@ public class Query implements QueryInterface, ToElementable, TransactionInterfac
* @throws SQLException
* @throws TransactionException
*/
- public static void filterEntitiesWithoutRetrievePermission(
+ public void filterEntitiesWithoutRetrievePermission(
final QueryInterface query, final String resultSet)
throws SQLException, TransactionException {
if (!filterEntitiesWithoutRetrievePermisions) {
@@ -551,7 +553,7 @@ public class Query implements QueryInterface, ToElementable, TransactionInterfac
while (rs.next()) {
final long t1 = System.currentTimeMillis();
final Integer id = rs.getInt("id");
- if (!Database.execute(new RetrieveSparseEntity(id), query.getAccess())
+ if (!execute(new RetrieveSparseEntity(id), query.getAccess())
.getEntity()
.getEntityACL()
.isPermitted(query.getUser(), EntityPermission.RETRIEVE_ENTITY)) {
@@ -584,7 +586,7 @@ public class Query implements QueryInterface, ToElementable, TransactionInterfac
while (iterator.hasNext()) {
final long t1 = System.currentTimeMillis();
final Integer id = iterator.next();
- if (!Database.execute(new RetrieveSparseEntity(id), getAccess())
+ if (!execute(new RetrieveSparseEntity(id), getAccess())
.getEntity()
.getEntityACL()
.isPermitted(getUser(), EntityPermission.RETRIEVE_ENTITY)) {
@@ -638,6 +640,8 @@ public class Query implements QueryInterface, ToElementable, TransactionInterfac
private int targetSetCount = -1;
+ private TransactionBenchmark benchmark;
+
@Override
public void addToElement(final Element parent) {
final Element ret = new Element("Query");
@@ -692,8 +696,6 @@ public class Query implements QueryInterface, ToElementable, TransactionInterfac
ret.addContent(selection);
}
- ret.addContent(this.benchmark.toElememt());
-
parent.addContent(ret);
}
@@ -723,6 +725,19 @@ public class Query implements QueryInterface, ToElementable, TransactionInterfac
@Override
public void addBenchmark(final String str, final long time) {
- this.benchmark.addBenchmark(this.getClass().getSimpleName().toString() + "." + str, time);
+ getTransactionBenchmark()
+ .addMeasurement(this.getClass().getSimpleName().toString() + "." + str, time);
+ }
+
+ @Override
+ public TransactionBenchmark getTransactionBenchmark() {
+ if (benchmark == null) {
+ if (container != null) {
+ benchmark = container.getTransactionBenchmark().getBenchmark(getClass());
+ } else {
+ benchmark = TransactionBenchmark.getRootInstance().getBenchmark(getClass());
+ }
+ }
+ return benchmark;
}
}
diff --git a/src/main/java/caosdb/server/query/SubProperty.java b/src/main/java/caosdb/server/query/SubProperty.java
index a5abe0b7b6cd3940f16cfe29461c13a019416d41..12629640ec4e7780c1b1d5933e7d6e1711f635f7 100644
--- a/src/main/java/caosdb/server/query/SubProperty.java
+++ b/src/main/java/caosdb/server/query/SubProperty.java
@@ -87,7 +87,7 @@ public class SubProperty implements QueryInterface, EntityFilterInterface {
this.filter.apply(this);
- Query.filterEntitiesWithoutRetrievePermission(this, this.sourceSet);
+ getQuery().filterEntitiesWithoutRetrievePermission(this, this.sourceSet);
final CallableStatement callFinishSubProperty =
getConnection().prepareCall("call finishSubProperty(?,?,?)");
diff --git a/src/main/java/caosdb/server/resource/AbstractCaosDBServerResource.java b/src/main/java/caosdb/server/resource/AbstractCaosDBServerResource.java
index c88bbd75a34aea81198b2c9d48702ca6f66a41e0..ea4e65f0a60d72cb5da6cb03b2cec44848dbc3c6 100644
--- a/src/main/java/caosdb/server/resource/AbstractCaosDBServerResource.java
+++ b/src/main/java/caosdb/server/resource/AbstractCaosDBServerResource.java
@@ -33,6 +33,7 @@ import caosdb.server.accessControl.UserSources;
import caosdb.server.database.backend.implementation.MySQL.ConnectionException;
import caosdb.server.entity.Message;
import caosdb.server.utils.ServerMessages;
+import caosdb.server.utils.WebinterfaceUtils;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
@@ -55,7 +56,6 @@ import org.jdom2.JDOMException;
import org.jdom2.input.SAXBuilder;
import org.restlet.data.Form;
import org.restlet.data.Header;
-import org.restlet.data.MediaType;
import org.restlet.data.Parameter;
import org.restlet.data.Status;
import org.restlet.representation.Representation;
@@ -81,7 +81,12 @@ public abstract class AbstractCaosDBServerResource extends ServerResource {
private String[] requestedItems = null;
private ArrayList<Integer> requestedIDs = new ArrayList<Integer>();
private ArrayList<String> requestedNames = new ArrayList<String>();
- private String xslScript = "webcaosdb.xsl";
+ private WebinterfaceUtils utils;
+
+ /** Return the {@link WebinterfaceUtils} instance for this resource. */
+ public WebinterfaceUtils getUtils() {
+ return this.utils;
+ }
public static class xmlNotWellFormedException extends Exception {
private static final long serialVersionUID = -6836378704013776849L;
@@ -101,7 +106,8 @@ public abstract class AbstractCaosDBServerResource extends ServerResource {
}
public Subject getUser() {
- return SecurityUtils.getSubject();
+ Subject ret = SecurityUtils.getSubject();
+ return ret;
}
/**
@@ -115,6 +121,8 @@ public abstract class AbstractCaosDBServerResource extends ServerResource {
getRequest().setEntity(r);
}
+ this.utils = WebinterfaceUtils.getInstance(getHostRef());
+
this.timestamp = getRequest().getDate().getTime();
this.sRID = getRequest().getAttributes().get("SRID").toString();
@@ -324,29 +332,36 @@ public abstract class AbstractCaosDBServerResource extends ServerResource {
}
protected String getXSLScript() {
- return this.xslScript;
- }
-
- protected void setXSLScript(final String s) {
- this.xslScript = s;
+ return this.utils.getWebinterfaceURI("webcaosdb.xsl");
}
+ /** Wrap an element for an "OK" response. */
protected JdomRepresentation ok(Element root) {
return ok(new Document(root));
}
+ /**
+ * Return a JDomRepresentation of the document and leave the {@link Response}'s {@link Status}
+ * where it is - which is usually 200 - OK when this method happens to be called.
+ */
protected JdomRepresentation ok(final Document doc) {
- return new JdomRepresentation(doc, MediaType.TEXT_XML, " ", getReference(), getXSLScript());
+ return new JdomRepresentation(doc, " ", getXSLScript());
}
+ /**
+ * Return a Representation containing the error message and set the {@link Response}'s {@link
+ * Status}.
+ *
+ * @param m - the error message.
+ * @param status - the response status
+ * @return A Representation of the error.
+ */
protected Representation error(final Message m, final Status status) {
final Document doc = new Document();
final Element root = generateRootElement();
root.addContent(m.toElement().setName("Error"));
doc.setRootElement(root);
- return error(
- new JdomRepresentation(doc, MediaType.TEXT_XML, " ", getReference(), getXSLScript()),
- status);
+ return error(new JdomRepresentation(doc, " ", getXSLScript()), status);
}
protected Representation error(Representation entity, Status status) {
@@ -362,12 +377,19 @@ public abstract class AbstractCaosDBServerResource extends ServerResource {
return error((Representation) null, status);
}
+ /**
+ * Return a Representation containing the warning message but leave the {@link Response}'s {@link
+ * Status} where it is - which is usually 200 - OK when this method happens to be called.
+ *
+ * @param m - the warning message.
+ * @return A Representation of the warning.
+ */
protected JdomRepresentation warning(final Message m) {
final Document doc = new Document();
final Element root = generateRootElement();
root.addContent(m.toElement().setName("Warning"));
doc.setRootElement(root);
- return new JdomRepresentation(doc, MediaType.TEXT_XML, " ", getReference(), getXSLScript());
+ return new JdomRepresentation(doc, " ", getXSLScript());
}
protected Representation noWellFormedNess() {
diff --git a/src/main/java/caosdb/server/resource/FileSystemResource.java b/src/main/java/caosdb/server/resource/FileSystemResource.java
index 0e8d59a107d3f8dbb83d966a9e251641ac9bdd8a..37f0e9ffda3f81b84c6001c5521ba5c8451b473e 100644
--- a/src/main/java/caosdb/server/resource/FileSystemResource.java
+++ b/src/main/java/caosdb/server/resource/FileSystemResource.java
@@ -26,9 +26,12 @@ import static caosdb.server.FileSystem.getFromFileSystem;
import static java.net.URLDecoder.decode;
import caosdb.server.database.backend.implementation.MySQL.ConnectionException;
+import caosdb.server.database.exceptions.EntityDoesNotExistException;
import caosdb.server.database.misc.TransactionBenchmark;
import caosdb.server.entity.Entity;
import caosdb.server.entity.FileProperties;
+import caosdb.server.entity.Message;
+import caosdb.server.entity.Message.MessageType;
import caosdb.server.entity.RetrieveEntity;
import caosdb.server.entity.container.TransactionContainer;
import caosdb.server.permissions.EntityPermission;
@@ -56,6 +59,11 @@ import org.restlet.representation.Representation;
*/
public class FileSystemResource extends AbstractCaosDBServerResource {
+ public static Message ORPHANED_FILE_WARNING =
+ new Message(
+ MessageType.Warning,
+ "Orphaned file. The file is not tracked. This is probably a harmless inconsistency but it might be a sign of other problems.");
+
/**
* Download a File from the CaosDBFileSystem. Only one File per Request.
*
@@ -117,13 +125,20 @@ public class FileSystemResource extends AbstractCaosDBServerResource {
rootElem.addContent(folder);
final long t2 = System.currentTimeMillis();
- getBenchmark().addBenchmark(this.getClass().getSimpleName(), t2 - t1);
+ getBenchmark().addMeasurement(this.getClass().getSimpleName(), t2 - t1);
rootElem.addContent(getBenchmark().toElememt());
final Document doc = new Document(rootElem);
return ok(doc);
} else {
+ try {
+ getEntity(specifier).checkPermission(EntityPermission.RETRIEVE_FILE);
+ } catch (EntityDoesNotExistException exception) {
+ // This file in the file system has no corresponding File record.
+ return error(ServerMessages.NOT_PERMITTED, Status.CLIENT_ERROR_FORBIDDEN);
+ }
+
final MediaType mt = MediaType.valueOf(FileUtils.getMimeType(file));
final FileRepresentation ret = new FileRepresentation(file, mt);
ret.setDisposition(new Disposition(Disposition.TYPE_ATTACHMENT));
@@ -132,6 +147,10 @@ public class FileSystemResource extends AbstractCaosDBServerResource {
}
}
+ TransactionBenchmark getBenchmark() {
+ return TransactionBenchmark.getRootInstance().getBenchmark(getClass());
+ }
+
protected Attribute getThumbnailAttribute(
final File file, final File child, final String referenceString) {
final String thpath =
@@ -145,28 +164,39 @@ public class FileSystemResource extends AbstractCaosDBServerResource {
return null;
}
- private TransactionBenchmark benchmark;
-
- private TransactionBenchmark getBenchmark() {
- if (this.benchmark == null) {
- this.benchmark = TransactionBenchmark.getSubBenchmark();
- }
- return this.benchmark;
- }
-
+ /**
+ * Return an element for the given file on a file system.
+ *
+ * <p>If there is no File entity for this file, an element without `id` is returned, instead a
+ * corresponding message is added to the element.
+ */
Element getFileElement(final String directory, final File file) throws Exception {
final Element celem = new Element("file");
- celem.setAttribute(
- "id",
- getEntityID((directory.endsWith("/") ? directory : directory + "/") + file.getName()));
celem.setAttribute("name", file.getName());
+
+ try {
+ final String entId =
+ getEntityID((directory.endsWith("/") ? directory : directory + "/") + file.getName());
+ celem.setAttribute("id", entId);
+ } catch (EntityDoesNotExistException exception) {
+ // This file in the file system has no corresponding File record.
+ ORPHANED_FILE_WARNING.addToElement(celem);
+ }
return celem;
}
protected String getEntityID(final String path) throws Exception {
- return getEntity(path).getId().toString();
+ final Entity fileEnt = getEntity(path);
+ return fileEnt.getId().toString();
}
+ /**
+ * Throws EntityDoesNotExistException when there is not entity with that path.
+ *
+ * @param path
+ * @return
+ * @throws Exception
+ */
private Entity getEntity(final String path) throws Exception {
final long t1 = System.currentTimeMillis();
final TransactionContainer c = new TransactionContainer();
@@ -177,25 +207,15 @@ public class FileSystemResource extends AbstractCaosDBServerResource {
final Transaction<?> t = new RetrieveSparseEntityByPath(c);
t.execute();
final long t2 = System.currentTimeMillis();
- getBenchmark().addBenchmark(this.getClass().getSimpleName() + ".getEntity", t2 - t1);
+ getBenchmark().addMeasurement(this.getClass().getSimpleName() + ".getEntity", t2 - t1);
return e;
}
protected File getFile(final String path) throws Exception {
final File ret = getFromFileSystem(path);
- if (ret != null && ret.isFile()) {
- checkPermissions(path);
- }
return ret;
}
- private final void checkPermissions(final String path) throws Exception {
- final long t1 = System.currentTimeMillis();
- getEntity(path).checkPermission(EntityPermission.RETRIEVE_FILE);
- final long t2 = System.currentTimeMillis();
- getBenchmark().addBenchmark(this.getClass().getSimpleName() + ".checkPermissions", t2 - t1);
- }
-
@Override
protected Representation httpPostInChildClass(final Representation entity)
throws ConnectionException, JDOMException {
diff --git a/src/main/java/caosdb/server/resource/InfoResource.java b/src/main/java/caosdb/server/resource/InfoResource.java
index 0b55f9f3e54f0a63c6e573e50bb03753fd98fbac..924d66980e5da27ec998fd361b95a8f0769e4229 100644
--- a/src/main/java/caosdb/server/resource/InfoResource.java
+++ b/src/main/java/caosdb/server/resource/InfoResource.java
@@ -30,43 +30,30 @@ import org.jdom2.Document;
import org.jdom2.Element;
import org.jdom2.JDOMException;
import org.restlet.representation.Representation;
-import org.restlet.resource.Options;
-/**
- * This class represents the information retrieved by the /Info GET request to CaosDB.
- */
+/** This class represents the information retrieved by /Info requests (only GET) to CaosDB. */
public class InfoResource extends AbstractCaosDBServerResource {
- /**
- * The response to the HTTP GET request is generated here.
- * @return The response code for OK and the resulting info XML document.
- */
+ /**
+ * The response to the HTTP GET request is generated here.
+ *
+ * @return The response code for OK and the resulting info XML document.
+ */
@Override
protected Representation httpGetInChildClass() throws Exception {
final Document doc = new Document();
final Element root = generateRootElement();
root.addContent(Info.toElement());
root.addContent(FlagInfo.toElement());
- root.addContent(TransactionBenchmark.getInstance().toElememt());
+ root.addContent(TransactionBenchmark.getRootInstance().toElememt());
doc.setRootElement(root);
return ok(doc);
}
- /**
- * There is no POST request specified for the /Info resource.
- */
+ /** There is no POST request specified for the /Info resource. */
@Override
protected Representation httpPostInChildClass(final Representation entity)
throws ConnectionException, JDOMException {
return null;
}
-
- /**
- * TODO: Can this stub be removed?
- */
- @Deprecated
- @Options
- public static Representation describeResource() {
- return null;
- }
}
diff --git a/src/main/java/caosdb/server/resource/JdomRepresentation.java b/src/main/java/caosdb/server/resource/JdomRepresentation.java
index 2443b95c20b3f659d6a2f9e88c0dd9fabb563088..04fbdac3fdd13f9e5c4f025cb2364386deac71f0 100644
--- a/src/main/java/caosdb/server/resource/JdomRepresentation.java
+++ b/src/main/java/caosdb/server/resource/JdomRepresentation.java
@@ -56,30 +56,21 @@ public class JdomRepresentation extends WriterRepresentation {
private final Document document;
private final String indent;
- private final Reference reference;
- private final String xslPath;
/**
* Constructor.
*
- * @param mediaType This should be TEXT_XML, but we'll leave it open to use others.
* @param document A JDOM Document parsed from elsewhere.
- * @param indent To make that representation better to read by a human.
- * @param reference
+ * @param indent Usually a number of whitespaces for better human readbility.
* @param xslPath A String containing a meaningful path to a xslt file.
*/
- public JdomRepresentation(
- final Document document,
- final MediaType mediaType,
- final String indent,
- final Reference reference,
- final String xslPath) {
- super(mediaType);
- this.xslPath = xslPath;
- this.reference = reference;
+ public JdomRepresentation(final Document document, final String indent, final String xslPath) {
+ super(MediaType.TEXT_XML);
this.indent = indent;
this.document = document;
- addStyleSheet();
+ if (xslPath != null && document != null) {
+ addStyleSheet(document, xslPath);
+ }
}
public static final String getWebUIRef(final Reference reference) {
@@ -89,14 +80,10 @@ public class JdomRepresentation extends WriterRepresentation {
}
/** adds the xslt processing instruction to the document. */
- protected final void addStyleSheet() {
- if (this.document != null) {
- final ProcessingInstruction pi =
- new ProcessingInstruction(
- "xml-stylesheet",
- "type=\"text/xsl\" href=\"" + getWebUIRef(this.reference) + this.xslPath + "\" ");
- this.document.getContent().add(0, pi);
- }
+ protected final void addStyleSheet(Document document, String xslPath) {
+ final ProcessingInstruction pi =
+ new ProcessingInstruction("xml-stylesheet", "type=\"text/xsl\" href=\"" + xslPath + "\" ");
+ document.getContent().add(0, pi);
}
@Override
diff --git a/src/main/java/caosdb/server/resource/ScriptingResource.java b/src/main/java/caosdb/server/resource/ScriptingResource.java
index 81a84ab631d9cd2206fd163b5c64eaac4c967286..b23a9593918ead75a6b38216838995e09ac67bc7 100644
--- a/src/main/java/caosdb/server/resource/ScriptingResource.java
+++ b/src/main/java/caosdb/server/resource/ScriptingResource.java
@@ -27,6 +27,7 @@ package caosdb.server.resource;
import caosdb.server.FileSystem;
import caosdb.server.accessControl.Principal;
import caosdb.server.accessControl.SessionToken;
+import caosdb.server.accessControl.UserSources;
import caosdb.server.entity.FileProperties;
import caosdb.server.entity.Message;
import caosdb.server.scripting.CallerSerializer;
@@ -82,6 +83,9 @@ public class ScriptingResource extends AbstractCaosDBServerResource {
@Override
protected Representation httpPostInChildClass(Representation entity) throws Exception {
+ if (isAnonymous()) {
+ return error(ServerMessages.AUTHORIZATION_ERROR, Status.CLIENT_ERROR_FORBIDDEN);
+ }
MediaType mediaType = entity.getMediaType();
try {
if (mediaType.equals(MediaType.MULTIPART_FORM_DATA, true)) {
@@ -202,9 +206,15 @@ public class ScriptingResource extends AbstractCaosDBServerResource {
return SessionToken.generate((Principal) getUser().getPrincipal(), null);
}
+ boolean isAnonymous() {
+ boolean ret = getUser().hasRole(UserSources.ANONYMOUS_ROLE);
+ return ret;
+ }
+
public int callScript(
List<String> commandLine, Integer timeoutMs, List<FileProperties> files, Object authToken)
throws Message {
+ // TODO getUser().checkPermission("SCRIPTING:EXECUTE:" + commandLine.get(0));
caller =
new ServerSideScriptingCaller(
commandLine.toArray(new String[commandLine.size()]), timeoutMs, files, authToken);
diff --git a/src/main/java/caosdb/server/resource/ServerPropertiesResource.java b/src/main/java/caosdb/server/resource/ServerPropertiesResource.java
index 9b878e3338a9d0b0c74704acb412ac92ceb1f0c4..347919d9c7d0f604734790f2ea09b5e2bac54042 100644
--- a/src/main/java/caosdb/server/resource/ServerPropertiesResource.java
+++ b/src/main/java/caosdb/server/resource/ServerPropertiesResource.java
@@ -13,7 +13,11 @@ public class ServerPropertiesResource extends AbstractCaosDBServerResource {
@Override
protected void doInit() {
super.doInit();
- setXSLScript("xsl/server_properties.xsl");
+ }
+
+ @Override
+ protected String getXSLScript() {
+ return getUtils().getWebinterfaceURI("xsl/server_properties.xsl");
}
@Override
diff --git a/src/main/java/caosdb/server/resource/TestCaseFileSystemResource.java b/src/main/java/caosdb/server/resource/TestCaseFileSystemResource.java
deleted file mode 100644
index 903ef0d5aca9a91eaeaff3693b7f77ef51e6cc46..0000000000000000000000000000000000000000
--- a/src/main/java/caosdb/server/resource/TestCaseFileSystemResource.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * ** header v3.0
- * This file is a part of the CaosDB Project.
- *
- * Copyright (C) 2018 Research Group Biomedical Physics,
- * Max-Planck-Institute for Dynamics and Self-Organization Göttingen
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
- * ** end header
- */
-package caosdb.server.resource;
-
-import caosdb.server.accessControl.Principal;
-import caosdb.server.entity.FileProperties;
-import caosdb.server.utils.FileUtils;
-import java.io.File;
-import org.jdom2.Element;
-
-public class TestCaseFileSystemResource extends FileSystemResource {
-
- @Override
- protected void doInit() {
- super.doInit();
- setXSLScript("webcaosdb.xsl");
- }
-
- private static final String BASEPATH = "./testfiles/";
-
- @Override
- protected File getFile(final String path) throws Exception {
- final File f = new File(BASEPATH + path);
- return (f != null && f.exists() ? f : null);
- }
-
- @Override
- protected String getEntityID(final String path) throws Exception {
- return "123412341234";
- }
-
- @Override
- protected Element generateRootElement() {
- final Element retRoot = new Element("Response");
-
- if (getUser() != null && getUser().isAuthenticated()) {
- retRoot.setAttribute("username", ((Principal) getUser().getPrincipal()).getUsername());
- retRoot.setAttribute("realm", ((Principal) getUser().getPrincipal()).getRealm());
- }
- retRoot.setAttribute("srid", getSRID());
- if (this.getCRID() != null) {
- retRoot.setAttribute("crid", this.getCRID());
- }
- retRoot.setAttribute("timestamp", getTimestamp().toString());
- retRoot.setAttribute("baseuri", getRootRef().toString() + "/TestCase/");
- return retRoot;
- }
-
- public static FileProperties getFileProperties(final String path) {
- final File f = new File(BASEPATH + path);
- return new FileProperties(FileUtils.getChecksum(f), path, f.length());
- }
-}
diff --git a/src/main/java/caosdb/server/resource/TestCaseResource.java b/src/main/java/caosdb/server/resource/TestCaseResource.java
deleted file mode 100644
index e8aef1f00e57335185b7b204c8de081570e6e188..0000000000000000000000000000000000000000
--- a/src/main/java/caosdb/server/resource/TestCaseResource.java
+++ /dev/null
@@ -1,481 +0,0 @@
-/*
- * ** header v3.0
- * This file is a part of the CaosDB Project.
- *
- * Copyright (C) 2018 Research Group Biomedical Physics,
- * Max-Planck-Institute for Dynamics and Self-Organization Göttingen
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <https://www.gnu.org/licenses/>.
- *
- * ** end header
- */
-package caosdb.server.resource;
-
-import caosdb.datetime.DateTimeFactory2;
-import caosdb.datetime.UTCDateTime;
-import caosdb.server.CaosDBException;
-import caosdb.server.database.backend.implementation.MySQL.ConnectionException;
-import caosdb.server.datatype.AbstractCollectionDatatype;
-import caosdb.server.datatype.AbstractDatatype;
-import caosdb.server.datatype.BooleanDatatype;
-import caosdb.server.datatype.BooleanValue;
-import caosdb.server.datatype.CollectionValue;
-import caosdb.server.datatype.DateTimeDatatype;
-import caosdb.server.datatype.DoubleDatatype;
-import caosdb.server.datatype.GenericValue;
-import caosdb.server.datatype.IntegerDatatype;
-import caosdb.server.datatype.ListDatatype;
-import caosdb.server.datatype.ReferenceDatatype2;
-import caosdb.server.datatype.ReferenceValue;
-import caosdb.server.datatype.TextDatatype;
-import caosdb.server.datatype.Value;
-import caosdb.server.entity.Entity;
-import caosdb.server.entity.EntityInterface;
-import caosdb.server.entity.MagicTypes;
-import caosdb.server.entity.Message;
-import caosdb.server.entity.Message.MessageType;
-import caosdb.server.entity.RetrieveEntity;
-import caosdb.server.entity.Role;
-import caosdb.server.entity.StatementStatus;
-import caosdb.server.entity.container.PropertyContainer;
-import caosdb.server.entity.wrapper.Parent;
-import caosdb.server.entity.wrapper.Property;
-import caosdb.server.entity.xml.ToElementable;
-import caosdb.server.query.Query;
-import caosdb.server.query.Query.ParsingException;
-import caosdb.server.utils.ServerMessages;
-import caosdb.server.utils.TransactionLogMessage;
-import java.io.IOException;
-import java.security.NoSuchAlgorithmException;
-import java.sql.SQLException;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.GregorianCalendar;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import org.jdom2.Document;
-import org.jdom2.Element;
-import org.restlet.representation.Representation;
-
-public class TestCaseResource extends AbstractCaosDBServerResource {
-
- @Override
- protected void doInit() {
- super.doInit();
- setXSLScript("webcaosdb.xsl");
- }
-
- @Override
- protected Element generateRootElement() {
- final Element retRoot = new Element("Response");
-
- retRoot.setAttribute("username", "TestUser");
- retRoot.setAttribute("realm", "PAM");
- retRoot.setAttribute("srid", getSRID());
- if (this.getCRID() != null) {
- retRoot.setAttribute("crid", this.getCRID());
- }
- retRoot.setAttribute("timestamp", getTimestamp().toString());
- retRoot.setAttribute("baseuri", getRootRef().toString() + "/TestCase/");
- return retRoot;
- }
-
- @Override
- protected synchronized Representation httpGetInChildClass()
- throws ConnectionException, IOException, SQLException, CaosDBException,
- NoSuchAlgorithmException, Exception {
-
- final Element rootElem = generateRootElement();
- final Document doc = new Document();
-
- final Collection<ToElementable> selectedTestCases = getSelectedTestCase();
- rootElem.setAttribute("count", Integer.toString(selectedTestCases.size()));
-
- int from = 0;
- int len = Integer.MAX_VALUE;
- if (getFlags().containsKey("query")) {
- final String queryStr = getFlags().get("query");
- if (queryStr != null && !queryStr.isEmpty()) {
- final Query query = new Query(queryStr);
- try {
- query.parse();
- query.addToElement(rootElem);
- } catch (final ParsingException e) {
- query.addToElement(rootElem);
- rootElem.addContent(ServerMessages.QUERY_PARSING_ERROR.toElement());
- doc.setRootElement(rootElem);
- return ok(doc);
- }
- }
- }
- if (getFlags().containsKey("P")) {
- final String[] split = getFlags().get("P").split("L");
- from = Integer.valueOf(split[0]);
- len = Integer.valueOf(split[1]);
- }
- int i = 0;
- for (final ToElementable e : selectedTestCases) {
- if (i++ >= from && i <= from + len) {
- e.addToElement(rootElem);
- }
- }
- // testEntities = null;
- // entityCounter = 1;
-
- doc.setRootElement(rootElem);
- return ok(doc);
- }
-
- private Collection<ToElementable> getSelectedTestCase() {
- final HashMap<String, ToElementable> testCases = getTestCase();
- if (getFlags().containsKey("query")) {
- final String queryStr = getFlags().get("query");
- if (queryStr.startsWith("FIND Annotation WHICH REFERENCES")) {
- final LinkedList<ToElementable> ret = new LinkedList<ToElementable>();
- for (final ToElementable e : testCases.values()) {
- if (e instanceof Entity
- && ((Entity) e).hasParents()
- && ((Entity) e).getParents().get(0).getName().equals("Annotation")) {
- ret.add(e);
- }
- }
- return ret;
- }
- }
- if (getFlags().containsKey("all")) {
- final LinkedList<ToElementable> ret = new LinkedList<ToElementable>();
- for (final ToElementable e : testCases.values()) {
- if (e instanceof Entity) {
- if (getFlags().get("all").equalsIgnoreCase("Entity")) {
- ret.add(e);
- } else {
- final Entity entity = (Entity) e;
- if (entity.getRole().toString().equalsIgnoreCase(getFlags().get("all"))) {
- ret.add(e);
- }
- }
- }
- }
- return ret;
- }
- if (!getRequestedIDs().isEmpty() || !getRequestedNames().isEmpty()) {
- final LinkedList<ToElementable> ret = new LinkedList<ToElementable>();
- for (final Integer id : getRequestedIDs()) {
- for (final ToElementable e : testCases.values()) {
- if (e instanceof Entity) {
- if (((Entity) e).getId().equals(id)) {
- ret.add(e);
- }
- }
- }
- }
- for (final String name : getRequestedNames()) {
- for (final ToElementable e : testCases.values()) {
- if (e instanceof Entity) {
- if (((Entity) e).getName().equals(name)) {
- ret.add(e);
- }
- }
- }
- }
- return ret;
- }
- return testCases.values();
- }
-
- private HashMap<String, ToElementable> getTestCase() {
- if (testEntities == null) {
- testEntities = new HashMap<String, ToElementable>();
-
- final Entity lp1 =
- createProperty(
- "LIST"
- + AbstractCollectionDatatype.LEFT_DELIMITER
- + "TEXT"
- + AbstractCollectionDatatype.RIGHT_DELIMITER);
- final Entity lp2 =
- createProperty(
- "LIST"
- + AbstractCollectionDatatype.LEFT_DELIMITER
- + "INTEGER"
- + AbstractCollectionDatatype.RIGHT_DELIMITER,
- "meter");
- final Entity lp4 =
- createProperty(
- "LIST"
- + AbstractCollectionDatatype.LEFT_DELIMITER
- + "BOOLEAN"
- + AbstractCollectionDatatype.RIGHT_DELIMITER);
- final Entity tp1 = createProperty("TEXT");
- final Entity dp1 = createProperty("DOUBLE", "meter");
- final Entity dtp1 = createProperty("DATETIME");
- final Entity tp2 = createProperty("TEXT");
- final Entity ip1 = createProperty("INTEGER", "meter");
- final Entity bp1 = createProperty("BOOLEAN");
- final Entity fp1 = createProperty("FILE");
-
- final Entity rt1 = createRecordType(new Entity[] {}, new Entity[] {});
- final Entity rt2 = createRecordType(new Entity[] {}, new Entity[] {tp2, ip1, bp1});
-
- final Entity rp1 = createProperty(rt2);
- final Entity lp3 =
- createProperty(
- "LIST"
- + AbstractCollectionDatatype.LEFT_DELIMITER
- + rt2.getName()
- + AbstractCollectionDatatype.RIGHT_DELIMITER);
- final Entity rt3 =
- createRecordType(
- new Entity[] {rt1}, new Entity[] {lp1, tp1, dp1, dtp1, rp1, lp2, lp3, lp4});
-
- createRecordFromRecordType(rt2);
- createRecordFromRecordType(rt3);
-
- createFileRecord("lorem_ipsum.txt");
- createFileRecord("testsubfolder/lorem_ipsum.txt");
- final Entity pngFile = createFileRecord("testimg.png");
- createFileRecord("testimg.jpg");
- createFileRecord("testimg.svg");
- createFileRecord("testimg.gif");
-
- createRecordWithPngFile(pngFile, fp1, rt1);
-
- /* Test Error, Warning, Info */
- createError();
- createWarning();
- createInfo();
-
- /* Entities with Error, Warning, Info */
- createRecordFromRecordType(rt2)
- .addError(
- new Message(
- MessageType.Error,
- 666,
- "Description of this error",
- "Further information about this error."));
- createRecordFromRecordType(rt2)
- .addWarning(
- new Message(
- MessageType.Warning,
- 346,
- "Description of this warning",
- "Further information about this warning."));
- createRecordFromRecordType(rt2)
- .addInfo(
- new Message(
- MessageType.Info,
- 0,
- "Description of this info",
- "Further information about this info."));
-
- createAnnotation();
- }
- return testEntities;
- }
-
- private void createAnnotation() {
- final Entity annotationOf = createEntity(Role.Property, "annotationOf");
- annotationOf.setDatatype("REFERENCE");
- final Entity comment = createEntity(Role.Property, "comment");
- comment.setDatatype("TEXT");
- final Entity annotation = createEntity(Role.RecordType, "Annotation");
- annotation.addProperty(new Property(annotationOf));
- annotation.addProperty(new Property(comment));
-
- final Entity rec = createRecordFromRecordType(annotation);
- rec.addTransactionLog(
- new TransactionLogMessage(
- "INSERT",
- rec,
- "Adam",
- UTCDateTime.SystemMillisToUTCDateTime(System.currentTimeMillis())));
- }
-
- private Entity createRecordWithPngFile(
- final Entity pngFile, final Entity property, final Entity parent) {
- final Entity entity = createRecordFromRecordType(parent);
- final Property newP = new Property(property.toElement());
- newP.setValue(new ReferenceValue(pngFile));
- entity.addProperty(newP);
- return entity;
- }
-
- private Entity createFileRecord(final String path) {
- final Entity entity = createEntity(Role.File, "TestFile");
- entity.setFileProperties(TestCaseFileSystemResource.getFileProperties(path));
- return entity;
- }
-
- private Entity createProperty(final Entity rt2) {
- final Entity entity = createProperty("REFERENCE");
- entity.setDatatype(rt2.getName());
- return entity;
- }
-
- private Value generateValue(final AbstractDatatype dt) {
- if (dt instanceof IntegerDatatype) {
- return new GenericValue(1337);
- } else if (dt instanceof DoubleDatatype) {
- return new GenericValue(3.14);
- } else if (dt instanceof DateTimeDatatype) {
- return DateTimeFactory2.valueOf(new GregorianCalendar());
- } else if (dt instanceof BooleanDatatype) {
- return BooleanValue.valueOf(entityCounter % 2 == 0);
- } else if (dt instanceof TextDatatype) {
- return new GenericValue("Don't panic.");
- } else if (dt instanceof ReferenceDatatype2) {
- final ReferenceDatatype2 dt2 = (ReferenceDatatype2) dt;
- dt2.getId();
- for (final ToElementable e : testEntities.values()) {
- if (e instanceof Entity && ((Entity) e).getRole() == Role.Record) {
- return new ReferenceValue((Entity) e);
- }
- }
- } else if (dt instanceof ListDatatype) {
- final AbstractDatatype dt2 = ((ListDatatype) dt).getDatatype();
- final CollectionValue v = new CollectionValue();
- for (int i = 0; i < 25; i++) {
- v.add(generateValue(dt2));
- }
- return v;
- }
- return null;
- }
-
- private Entity createRecordFromRecordType(final Entity rt2) {
- final Entity entity = createEntity(Role.Record, "TestRecord");
- addParentsAndProperties(entity, rt2, rt2.getProperties(), "FIX");
- for (final Property p : entity.getProperties()) {
- p.setValue(generateValue(p.getDatatype()));
- }
- return entity;
- }
-
- private void addParentsAndProperties(
- final Entity entity,
- final Entity parent,
- final PropertyContainer properties,
- final String importance) {
- final List<Entity> par = new LinkedList<Entity>();
- par.add(parent);
- addParentsAndProperties(entity, par, properties, importance);
- }
-
- private void addParentsAndProperties(
- final Entity entity,
- final Collection<? extends EntityInterface> parents,
- final Collection<? extends EntityInterface> properties,
- final String importance) {
- for (final EntityInterface p : properties) {
- final Property newP = new Property(p.toElement());
- if (importance != null) {
- newP.setStatementStatus(StatementStatus.valueOf(importance));
- }
- entity.addProperty(newP);
- }
- for (final EntityInterface p : parents) {
- entity.addParent(new Parent(p.toElement()));
- }
- }
-
- private Message createError() {
- return createMessage(
- MessageType.Error.toString(),
- "This is a description of the error",
- "Here is additional info about the error.",
- 666);
- }
-
- private Message createWarning() {
- return createMessage(
- MessageType.Warning.toString(),
- "This is a description of the warning",
- "Here is additional info about the warning.",
- 333);
- }
-
- private Message createInfo() {
- return createMessage(
- MessageType.Info.toString(),
- "Here is useful information about the last activity of the user.",
- "Here is additional information.",
- null);
- }
-
- private Message createMessage(
- String type, final String description, final String body, final Integer code) {
- final Message m = new Message(type, code, description, body);
- if (testEntities.containsKey(type)) {
- int i = 0;
- while (testEntities.containsKey(type + "-" + i)) {
- i++;
- }
- type += "-" + i;
- }
- testEntities.put(type, m);
- return m;
- }
-
- private static HashMap<String, ToElementable> testEntities = null;
- private static int entityCounter = 1;
-
- private Entity createProperty(final String datatype) {
- return createProperty(datatype, null);
- }
-
- private Entity createRecordType(final Entity[] parents, final Entity[] properties) {
- final Entity entity = createEntity(Role.RecordType, "TestRecordType");
- addParentsAndProperties(entity, parents, properties, "OBLIGATORY");
- return entity;
- }
-
- private void addParentsAndProperties(
- final Entity entity,
- final EntityInterface[] parents,
- final EntityInterface[] properties,
- final String importance) {
- addParentsAndProperties(entity, Arrays.asList(parents), Arrays.asList(properties), importance);
- }
-
- private Entity createProperty(final String datatype, final String unit) {
- final Entity entity = createEntity(Role.Property, "Test" + datatype + "Property");
- entity.setDatatype(datatype);
- if (unit != null) {
- final EntityInterface magicUnit = MagicTypes.UNIT.getEntity();
- final Property unitProp = new Property();
- unitProp.setId(magicUnit.getId());
- unitProp.setValue(new GenericValue(unit));
- entity.addProperty(unitProp);
- }
- return entity;
- }
-
- private Entity createEntity(final Role role, String name) {
- final Entity entity = new RetrieveEntity(entityCounter++);
- entity.setRole(role);
- entity.setDescription(
- "An informative description of the idea, meaning, and purpose of this entity.");
- if (testEntities.containsKey(name)) {
- int i = 0;
- while (testEntities.containsKey(name + "-" + i)) {
- i++;
- }
- name += "-" + i;
- }
- testEntities.put(name, entity);
- entity.setName(name);
- return entity;
- }
-}
diff --git a/src/main/java/caosdb/server/resource/UserResource.java b/src/main/java/caosdb/server/resource/UserResource.java
index 5cae0da2efa4716293407497fd896bf0b880f233..b77ff8a49035d2be930a3f957e743ed2f0dae067 100644
--- a/src/main/java/caosdb/server/resource/UserResource.java
+++ b/src/main/java/caosdb/server/resource/UserResource.java
@@ -42,7 +42,6 @@ import org.jdom2.Document;
import org.jdom2.Element;
import org.jdom2.JDOMException;
import org.restlet.data.Form;
-import org.restlet.data.MediaType;
import org.restlet.data.Status;
import org.restlet.representation.Representation;
@@ -84,7 +83,7 @@ public class UserResource extends AbstractCaosDBServerResource {
}
doc.setRootElement(rootElem);
- return new JdomRepresentation(doc, MediaType.TEXT_XML, " ", getReference(), getXSLScript());
+ return ok(doc);
}
@Override
@@ -125,7 +124,7 @@ public class UserResource extends AbstractCaosDBServerResource {
rootElem.addContent(t.getUserElement());
doc.setRootElement(rootElem);
- return new JdomRepresentation(doc, MediaType.TEXT_XML, " ", getReference(), getXSLScript());
+ return ok(doc);
} catch (final Message m) {
if (m == ServerMessages.ACCOUNT_DOES_NOT_EXIST) {
return error(m, Status.CLIENT_ERROR_NOT_FOUND);
@@ -177,7 +176,7 @@ public class UserResource extends AbstractCaosDBServerResource {
rootElem.addContent(t.getUserElement());
doc.setRootElement(rootElem);
- return new JdomRepresentation(doc, MediaType.TEXT_XML, " ", getReference(), getXSLScript());
+ return ok(doc);
}
@Override
@@ -200,6 +199,6 @@ public class UserResource extends AbstractCaosDBServerResource {
rootElem.addContent(t.getUserElement());
doc.setRootElement(rootElem);
- return new JdomRepresentation(doc, MediaType.TEXT_XML, " ", getReference(), getXSLScript());
+ return ok(doc);
}
}
diff --git a/src/main/java/caosdb/server/resource/Webinterface.java b/src/main/java/caosdb/server/resource/Webinterface.java
index 7343fc5ff681a56f1dcba633bd7f3de4ba27f813..f05ae4e6b9a98aba1c925677e0fc8e90a74e57e6 100644
--- a/src/main/java/caosdb/server/resource/Webinterface.java
+++ b/src/main/java/caosdb/server/resource/Webinterface.java
@@ -24,6 +24,7 @@ package caosdb.server.resource;
import caosdb.server.CaosDBServer;
import caosdb.server.ServerProperties;
+import caosdb.server.utils.WebinterfaceUtils;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
@@ -42,13 +43,14 @@ import org.restlet.util.Series;
public class Webinterface extends ServerResource {
+ private WebinterfaceUtils utils;
+
@Override
protected void doInit() throws ResourceException {
+ this.utils = WebinterfaceUtils.getInstance(getHostRef());
super.doInit();
}
- private static final File PUBLIC_DIRECTORY = new File("caosdb-webui/public/").getAbsoluteFile();
-
@Get
public Representation deliver() throws IOException {
final String path = (String) getRequest().getAttributes().get("path");
@@ -57,23 +59,15 @@ public class Webinterface extends ServerResource {
getResponse().setStatus(Status.CLIENT_ERROR_NOT_FOUND);
return null;
}
- final File file = new File(this.PUBLIC_DIRECTORY.getAbsolutePath() + "/" + path);
- // TODO if (!FileUtils.isSubDirSymlinkSave(file, this.PUBLIC_DIRECTORY.getParentFile())) {
- // getResponse().setStatus(Status.CLIENT_ERROR_FORBIDDEN);
- // return null;
- // }
- if (!file.exists()) {
+
+ final File file = utils.getPublicFile(path);
+
+ if (file == null) {
getResponse().setStatus(Status.CLIENT_ERROR_NOT_FOUND);
return null;
}
- Series<Header> headers = getRequest().getHeaders();
- if (headers == null) {
- headers = new Series<Header>(Header.class);
- getResponse().getAttributes().put(HeaderConstants.ATTRIBUTE_HEADERS, headers);
- }
- headers.set("Access-Control-Allow-Origin", getHostRef().toString());
-
+ // TODO move to FileUtils and use a third-party library
final MediaType mt =
path.endsWith(".json")
? MediaType.APPLICATION_JSON
@@ -91,6 +85,13 @@ public class Webinterface extends ServerResource {
final FileRepresentation ret = new FileRepresentation(file, mt);
+ Series<Header> headers = getRequest().getHeaders();
+ if (headers == null) {
+ headers = new Series<Header>(Header.class);
+ getResponse().getAttributes().put(HeaderConstants.ATTRIBUTE_HEADERS, headers);
+ }
+ headers.set("Access-Control-Allow-Origin", getHostRef().toString());
+
List<CacheDirective> cacheDirectives = new ArrayList<>();
cacheDirectives.add(
new CacheDirective(
diff --git a/src/main/java/caosdb/server/resource/WebinterfaceBuildNumber.java b/src/main/java/caosdb/server/resource/WebinterfaceBuildNumber.java
new file mode 100644
index 0000000000000000000000000000000000000000..bb0e47f40164a7480a547b74cda33c83d375fb76
--- /dev/null
+++ b/src/main/java/caosdb/server/resource/WebinterfaceBuildNumber.java
@@ -0,0 +1,63 @@
+/**
+ * ** header v3.0 This file is a part of the CaosDB Project.
+ *
+ * <p>Copyright (c) 2019 IndiScale GmbH Copyright (c) 2019 Daniel Hornung <d.hornung@indiscale.com>
+ *
+ * <p>This program is free software: you can redistribute it and/or modify it under the terms of the
+ * GNU Affero General Public License as published by the Free Software Foundation, either version 3
+ * of the License, or (at your option) any later version.
+ *
+ * <p>This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
+ * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * <p>You should have received a copy of the GNU Affero General Public License along with this
+ * program. If not, see <https://www.gnu.org/licenses/>.
+ *
+ * <p>** end header
+ */
+package caosdb.server.resource;
+
+import caosdb.server.utils.WebinterfaceUtils;
+import org.restlet.data.Status;
+import org.restlet.resource.Get;
+import org.restlet.resource.ResourceException;
+import org.restlet.resource.ServerResource;
+
+/**
+ * This {@link ServerResource} exposes the current build number of the web interface.
+ *
+ * <p>It is mainly used for testing and debugging. A User can determine the current build number of
+ * the web interface.
+ *
+ * @author Timm Fitschen (t.fitschen@indiscale.com)
+ */
+public class WebinterfaceBuildNumber extends ServerResource {
+
+ private WebinterfaceUtils utils;
+
+ @Override
+ protected void doInit() throws ResourceException {
+ super.doInit();
+ this.utils = WebinterfaceUtils.getInstance(getHostRef());
+ }
+
+ /**
+ * Return the current build number of the web interface.
+ *
+ * <p>If the build number could not be determined the server responds with 404 - Not Found.
+ * Reasons for this include an out-dated web interface, the web interface has not been build yet
+ * or the web interface is not available at all.
+ *
+ * @return the current build number.
+ */
+ @Get("text")
+ public String getBuildNumber() {
+ String buildNumber = utils.getBuildNumber();
+
+ if (buildNumber == null) {
+ setStatus(Status.CLIENT_ERROR_NOT_FOUND, "Build number could not be determined.");
+ }
+ return buildNumber;
+ }
+}
diff --git a/src/main/java/caosdb/server/resource/transaction/EntityResource.java b/src/main/java/caosdb/server/resource/transaction/EntityResource.java
index 0e78c46a1f20fd870aca972f5c272c38c78438d0..0c853096c1af71dcacf2ef775c46f4dceecb1673 100644
--- a/src/main/java/caosdb/server/resource/transaction/EntityResource.java
+++ b/src/main/java/caosdb/server/resource/transaction/EntityResource.java
@@ -29,7 +29,6 @@ import caosdb.server.entity.container.InsertContainer;
import caosdb.server.entity.container.RetrieveContainer;
import caosdb.server.entity.container.UpdateContainer;
import caosdb.server.resource.AbstractCaosDBServerResource;
-import caosdb.server.resource.JdomRepresentation;
import caosdb.server.resource.transaction.handlers.FileUploadHandler;
import caosdb.server.resource.transaction.handlers.IDHandler;
import caosdb.server.resource.transaction.handlers.RequestHandler;
@@ -99,6 +98,7 @@ public class EntityResource extends AbstractCaosDBServerResource {
throws ConnectionException, IOException, SQLException, CaosDBException,
NoSuchAlgorithmException, Exception {
+ final long t1 = System.currentTimeMillis();
if (!this.get) {
getResponse().setStatus(Status.CLIENT_ERROR_METHOD_NOT_ALLOWED);
return null;
@@ -112,10 +112,14 @@ public class EntityResource extends AbstractCaosDBServerResource {
final Retrieve retrieve = new Retrieve(entityContainer);
retrieve.execute();
+ final long t2 = System.currentTimeMillis();
+ entityContainer
+ .getTransactionBenchmark()
+ .addMeasurement(getClass().getSimpleName() + ".httpGetInChildClass", t2 - t1);
final Element rootElem = generateRootElement();
entityContainer.addToElement(rootElem);
doc.setRootElement(rootElem);
- return new JdomRepresentation(doc, MediaType.TEXT_XML, " ", getReference(), getXSLScript());
+ return ok(doc);
}
@Override
@@ -138,7 +142,7 @@ public class EntityResource extends AbstractCaosDBServerResource {
entityContainer.addToElement(rootElem);
doc.setRootElement(rootElem);
- return new JdomRepresentation(doc, MediaType.TEXT_XML, " ", getReference(), getXSLScript());
+ return ok(doc);
}
@Override
@@ -162,7 +166,8 @@ public class EntityResource extends AbstractCaosDBServerResource {
final Element rootElem = generateRootElement();
entityContainer.addToElement(rootElem);
doc.setRootElement(rootElem);
- return new JdomRepresentation(doc, MediaType.TEXT_XML, " ", getReference(), getXSLScript());
+
+ return ok(doc);
}
@Override
@@ -186,6 +191,7 @@ public class EntityResource extends AbstractCaosDBServerResource {
final Element rootElem = generateRootElement();
entityContainer.addToElement(rootElem);
doc.setRootElement(rootElem);
- return new JdomRepresentation(doc, MediaType.TEXT_XML, " ", getReference(), getXSLScript());
+
+ return ok(doc);
}
}
diff --git a/src/main/java/caosdb/server/utils/ChecksumUpdater.java b/src/main/java/caosdb/server/transaction/ChecksumUpdater.java
similarity index 92%
rename from src/main/java/caosdb/server/utils/ChecksumUpdater.java
rename to src/main/java/caosdb/server/transaction/ChecksumUpdater.java
index 6a1e0e927002eff7669d3a6dc7bcae39e761d859..30313478c263a131658005daa39da3b495c85eb2 100644
--- a/src/main/java/caosdb/server/utils/ChecksumUpdater.java
+++ b/src/main/java/caosdb/server/transaction/ChecksumUpdater.java
@@ -20,10 +20,9 @@
*
* ** end header
*/
-package caosdb.server.utils;
+package caosdb.server.transaction;
import caosdb.server.CaosDBException;
-import caosdb.server.database.Database;
import caosdb.server.database.DatabaseMonitor;
import caosdb.server.database.access.Access;
import caosdb.server.database.backend.transaction.GetUpdateableChecksums;
@@ -34,7 +33,7 @@ import caosdb.server.entity.EntityInterface;
import caosdb.server.entity.FileProperties;
import caosdb.server.entity.Message;
import caosdb.server.entity.container.TransactionContainer;
-import caosdb.server.transaction.WriteTransaction;
+import caosdb.server.utils.FileUtils;
import java.io.IOException;
import java.security.NoSuchAlgorithmException;
@@ -86,7 +85,7 @@ public class ChecksumUpdater extends WriteTransaction<TransactionContainer> impl
DatabaseMonitor.getInstance().acquireStrongAccess(this);
// update
- Database.execute(new UpdateSparseEntity(fileEntity), strongAccess);
+ execute(new UpdateSparseEntity(fileEntity), strongAccess);
strongAccess.commit();
} catch (final InterruptedException e) {
@@ -108,7 +107,7 @@ public class ChecksumUpdater extends WriteTransaction<TransactionContainer> impl
final EntityInterface fileEntity, final Access access, final long lastModified)
throws NoSuchAlgorithmException, IOException, CaosDBException, Message {
final EntityInterface checkEntity =
- Database.execute(new RetrieveSparseEntity(fileEntity), access).getEntity();
+ execute(new RetrieveSparseEntity(fileEntity), access).getEntity();
final FileProperties thatFP = checkEntity.getFileProperties();
final FileProperties thisFP = fileEntity.getFileProperties();
@@ -136,13 +135,13 @@ public class ChecksumUpdater extends WriteTransaction<TransactionContainer> impl
synchronized (instance.running) {
// test for updatable checksums
- final Integer id = Database.execute(new GetUpdateableChecksums(), weakAccess).getID();
+ final Integer id = execute(new GetUpdateableChecksums(), weakAccess).getID();
if (id == null) {
// nothing to be updated...
instance.running = false;
return null;
}
- return Database.execute(new RetrieveSparseEntity(id), weakAccess).getEntity();
+ return execute(new RetrieveSparseEntity(id), weakAccess).getEntity();
}
} catch (final Exception e) {
e.printStackTrace();
diff --git a/src/main/java/caosdb/server/transaction/Delete.java b/src/main/java/caosdb/server/transaction/Delete.java
index f371f0d64e32ddc5a6f4812943b2c9be08b5bcbc..1d3aacbc78043fef6982b2b7663dc5d033e50e50 100644
--- a/src/main/java/caosdb/server/transaction/Delete.java
+++ b/src/main/java/caosdb/server/transaction/Delete.java
@@ -22,7 +22,6 @@
*/
package caosdb.server.transaction;
-import caosdb.server.database.Database;
import caosdb.server.database.access.Access;
import caosdb.server.database.backend.transaction.DeleteEntity;
import caosdb.server.database.backend.transaction.RetrieveFullEntity;
@@ -53,7 +52,7 @@ public class Delete extends WriteTransaction<DeleteContainer> {
setAccess(getMonitor().allocateStrongAccess(this));
// retrieve all entities which are to be deleted.
- Database.execute(new RetrieveFullEntity(getContainer()), getAccess());
+ execute(new RetrieveFullEntity(getContainer()), getAccess());
for (final EntityInterface e : getContainer()) {
if (e.getEntityStatus() == EntityStatus.NONEXISTENT) {
@@ -104,10 +103,9 @@ public class Delete extends WriteTransaction<DeleteContainer> {
delete(getContainer(), getAccess());
}
- private static void delete(final TransactionContainer container, final Access access)
- throws Exception {
+ private void delete(final TransactionContainer container, final Access access) throws Exception {
if (container.getStatus().ordinal() >= EntityStatus.QUALIFIED.ordinal()) {
- Database.execute(new DeleteEntity(container), access);
+ execute(new DeleteEntity(container), access);
}
}
diff --git a/src/main/java/caosdb/server/transaction/DeleteRoleTransaction.java b/src/main/java/caosdb/server/transaction/DeleteRoleTransaction.java
index 3f3758823a2b554edbfb1e400c2a8f2f14916ba1..e1ba3bec80663cb3b30d2dbdc48d9f9024c8f2d7 100644
--- a/src/main/java/caosdb/server/transaction/DeleteRoleTransaction.java
+++ b/src/main/java/caosdb/server/transaction/DeleteRoleTransaction.java
@@ -23,7 +23,6 @@
package caosdb.server.transaction;
import caosdb.server.accessControl.ACMPermissions;
-import caosdb.server.database.Database;
import caosdb.server.database.backend.transaction.DeleteRole;
import caosdb.server.database.backend.transaction.RetrieveRole;
import caosdb.server.database.backend.transaction.SetPermissionRules;
@@ -42,10 +41,10 @@ public class DeleteRoleTransaction extends AccessControlTransaction {
protected void transaction() throws Exception {
SecurityUtils.getSubject().checkPermission(ACMPermissions.PERMISSION_DELETE_ROLE(this.name));
- if (Database.execute(new RetrieveRole(this.name), getAccess()).getRole() == null) {
+ if (execute(new RetrieveRole(this.name), getAccess()).getRole() == null) {
throw ServerMessages.ROLE_DOES_NOT_EXIST;
}
- Database.execute(new SetPermissionRules(this.name, null), getAccess());
- Database.execute(new DeleteRole(this.name), getAccess());
+ execute(new SetPermissionRules(this.name, null), getAccess());
+ execute(new DeleteRole(this.name), getAccess());
}
}
diff --git a/src/main/java/caosdb/server/transaction/DeleteUserTransaction.java b/src/main/java/caosdb/server/transaction/DeleteUserTransaction.java
index 729512a2a6dfae734b152e2ddd4c1e71362681d5..cdedce31b34153494ceb0be28e4c3a9da360bb6e 100644
--- a/src/main/java/caosdb/server/transaction/DeleteUserTransaction.java
+++ b/src/main/java/caosdb/server/transaction/DeleteUserTransaction.java
@@ -25,7 +25,6 @@ package caosdb.server.transaction;
import caosdb.server.accessControl.ACMPermissions;
import caosdb.server.accessControl.CredentialsValidator;
import caosdb.server.accessControl.UserSources;
-import caosdb.server.database.Database;
import caosdb.server.database.backend.transaction.DeletePassword;
import caosdb.server.database.backend.transaction.DeleteUser;
import caosdb.server.database.backend.transaction.RetrievePasswordValidator;
@@ -51,14 +50,14 @@ public class DeleteUserTransaction extends AccessControlTransaction {
.checkPermission(ACMPermissions.PERMISSION_DELETE_USER(this.realm, this.user));
final CredentialsValidator<String> validator =
- Database.execute(new RetrievePasswordValidator(this.user), getAccess()).getValidator();
+ execute(new RetrievePasswordValidator(this.user), getAccess()).getValidator();
if (validator == null) {
throw ServerMessages.ACCOUNT_DOES_NOT_EXIST;
}
- Database.execute(new DeletePassword(this.user), getAccess());
- Database.execute(new DeleteUser(this.realm, this.user), getAccess());
+ execute(new DeletePassword(this.user), getAccess());
+ execute(new DeleteUser(this.realm, this.user), getAccess());
}
public Element getUserElement() {
diff --git a/src/main/java/caosdb/server/utils/FileStorageConsistencyCheck.java b/src/main/java/caosdb/server/transaction/FileStorageConsistencyCheck.java
similarity index 84%
rename from src/main/java/caosdb/server/utils/FileStorageConsistencyCheck.java
rename to src/main/java/caosdb/server/transaction/FileStorageConsistencyCheck.java
index c750bb4e6f4de884cf39289d82df79138206f057..26cab30e898d2cd9f163ba3c9773b96c02110cbb 100644
--- a/src/main/java/caosdb/server/utils/FileStorageConsistencyCheck.java
+++ b/src/main/java/caosdb/server/transaction/FileStorageConsistencyCheck.java
@@ -20,10 +20,9 @@
*
* ** end header
*/
-package caosdb.server.utils;
+package caosdb.server.transaction;
import caosdb.datetime.UTCDateTime;
-import caosdb.server.database.Database;
import caosdb.server.database.DatabaseMonitor;
import caosdb.server.database.access.Access;
import caosdb.server.database.backend.transaction.FileConsistencyCheck;
@@ -31,10 +30,11 @@ import caosdb.server.database.backend.transaction.GetFileIterator;
import caosdb.server.database.backend.transaction.GetFileRecordByPath;
import caosdb.server.database.backend.transaction.RetrieveAllUncheckedFiles;
import caosdb.server.database.backend.transaction.SetFileCheckedTimestamp;
+import caosdb.server.database.exceptions.EntityDoesNotExistException;
import caosdb.server.database.proto.SparseEntity;
import caosdb.server.entity.Message;
import caosdb.server.entity.xml.ToElementable;
-import caosdb.server.transaction.TransactionInterface;
+import caosdb.server.utils.SHA512;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
@@ -69,7 +69,7 @@ public class FileStorageConsistencyCheck extends Thread
// test all files in file system.
final Iterator<String> iterator =
- Database.execute(new GetFileIterator(this.location), this.access).getIterator();
+ execute(new GetFileIterator(this.location), this.access).getIterator();
this.ts = System.currentTimeMillis();
while (iterator != null && iterator.hasNext()) {
@@ -82,40 +82,43 @@ public class FileStorageConsistencyCheck extends Thread
}
final String path = iterator.next();
- // FIXME this prevents all files with ".thumbnail" from being checked.
- if (path.contains(".thumbnail")) {
+ // this prevents all thumbnails from being checked.
+ if (path.contains(".thumbnails/")) {
continue;
}
- final GetFileRecordByPath t = Database.execute(new GetFileRecordByPath(path), this.access);
-
- if (t.getEntity() == null) {
+ try {
+ final GetFileRecordByPath t = execute(new GetFileRecordByPath(path), this.access);
+ final int result =
+ execute(
+ new FileConsistencyCheck(
+ path,
+ t.getSize(),
+ t.getHash(),
+ t.getLastConsistencyCheck(),
+ new SHA512()),
+ this.access)
+ .getResult();
+
+ if (result != FileConsistencyCheck.OK) {
+ this.results.put(path, result);
+ }
+
+ execute(new SetFileCheckedTimestamp(t.getId(), this.ts), this.access);
+ } catch (EntityDoesNotExistException e) {
this.results.put(path, FileConsistencyCheck.UNKNOWN_FILE);
continue;
}
- final int result =
- Database.execute(
- new FileConsistencyCheck(
- path, t.getSize(), t.getHash(), t.getLastConsistencyCheck(), new SHA512()),
- this.access)
- .getResult();
-
- if (result != FileConsistencyCheck.OK) {
- this.results.put(path, result);
- }
-
- Database.execute(new SetFileCheckedTimestamp(t.getId(), this.ts), this.access);
}
// test all remaining file records
final Iterator<SparseEntity> iterator2 =
- Database.execute(new RetrieveAllUncheckedFiles(this.ts, this.location), this.access)
- .getIterator();
+ execute(new RetrieveAllUncheckedFiles(this.ts, this.location), this.access).getIterator();
while (iterator2 != null && iterator2.hasNext()) {
final SparseEntity entity = iterator2.next();
final int result =
- Database.execute(
+ execute(
new FileConsistencyCheck(
entity.filePath,
entity.fileSize,
@@ -129,7 +132,7 @@ public class FileStorageConsistencyCheck extends Thread
this.results.put(entity.filePath, result);
}
- Database.execute(new SetFileCheckedTimestamp(entity.id, this.ts), this.access);
+ execute(new SetFileCheckedTimestamp(entity.id, this.ts), this.access);
}
} catch (final Exception e) {
diff --git a/src/main/java/caosdb/server/transaction/Insert.java b/src/main/java/caosdb/server/transaction/Insert.java
index 0dbda080ce33ea1fbcfdd4f43f36860f44e55db1..627ada52655c5e7802e98172d1c32f8280a95fb4 100644
--- a/src/main/java/caosdb/server/transaction/Insert.java
+++ b/src/main/java/caosdb/server/transaction/Insert.java
@@ -22,7 +22,6 @@
*/
package caosdb.server.transaction;
-import caosdb.server.database.Database;
import caosdb.server.database.access.Access;
import caosdb.server.database.backend.transaction.InsertEntity;
import caosdb.server.entity.EntityInterface;
@@ -101,10 +100,9 @@ public class Insert extends WriteTransaction<InsertContainer> {
insert(getContainer(), getAccess());
}
- public static void insert(final TransactionContainer container, final Access access)
- throws Exception {
+ public void insert(final TransactionContainer container, final Access access) throws Exception {
if (container.getStatus().ordinal() >= EntityStatus.QUALIFIED.ordinal()) {
- Database.execute(new InsertEntity(container), access);
+ execute(new InsertEntity(container), access);
}
}
diff --git a/src/main/java/caosdb/server/transaction/InsertLogRecordTransaction.java b/src/main/java/caosdb/server/transaction/InsertLogRecordTransaction.java
index 89b48418843f326ac42bf49f96b0e6dfa8d0f17a..3762b0bed3f726dd521f754043d0e99a5d2a3913 100644
--- a/src/main/java/caosdb/server/transaction/InsertLogRecordTransaction.java
+++ b/src/main/java/caosdb/server/transaction/InsertLogRecordTransaction.java
@@ -22,7 +22,6 @@
*/
package caosdb.server.transaction;
-import caosdb.server.database.Database;
import caosdb.server.database.DatabaseMonitor;
import caosdb.server.database.access.Access;
import caosdb.server.database.backend.transaction.InsertLogRecord;
@@ -41,7 +40,7 @@ public class InsertLogRecordTransaction implements TransactionInterface {
public void execute() throws Exception {
final Access access = DatabaseMonitor.getInstance().acquiredWeakAccess(this);
try {
- Database.execute(new InsertLogRecord(this.toBeFlushed), access);
+ execute(new InsertLogRecord(this.toBeFlushed), access);
} finally {
access.release();
}
diff --git a/src/main/java/caosdb/server/transaction/InsertRoleTransaction.java b/src/main/java/caosdb/server/transaction/InsertRoleTransaction.java
index 250ca37bba0eedaba25d105072617b20fbf94cd5..8377210690692c8819e5197e35a8e549d27f2a30 100644
--- a/src/main/java/caosdb/server/transaction/InsertRoleTransaction.java
+++ b/src/main/java/caosdb/server/transaction/InsertRoleTransaction.java
@@ -24,7 +24,6 @@ package caosdb.server.transaction;
import caosdb.server.accessControl.ACMPermissions;
import caosdb.server.accessControl.Role;
-import caosdb.server.database.Database;
import caosdb.server.database.backend.transaction.InsertRole;
import caosdb.server.database.backend.transaction.RetrieveRole;
import caosdb.server.utils.ServerMessages;
@@ -42,10 +41,10 @@ public class InsertRoleTransaction extends AccessControlTransaction {
protected void transaction() throws Exception {
SecurityUtils.getSubject().checkPermission(ACMPermissions.PERMISSION_INSERT_ROLE());
- if (Database.execute(new RetrieveRole(this.role.name), getAccess()).getRole() != null) {
+ if (execute(new RetrieveRole(this.role.name), getAccess()).getRole() != null) {
throw ServerMessages.ROLE_NAME_IS_NOT_UNIQUE;
}
- Database.execute(new InsertRole(this.role), getAccess());
+ execute(new InsertRole(this.role), getAccess());
}
}
diff --git a/src/main/java/caosdb/server/transaction/InsertUserTransaction.java b/src/main/java/caosdb/server/transaction/InsertUserTransaction.java
index d6cc2f46b397cf194a32f6ab5d7ec7e7fdc7407c..75b876b8bc291ea246746b4a8e85ccd26055c173 100644
--- a/src/main/java/caosdb/server/transaction/InsertUserTransaction.java
+++ b/src/main/java/caosdb/server/transaction/InsertUserTransaction.java
@@ -25,7 +25,6 @@ package caosdb.server.transaction;
import caosdb.server.accessControl.ACMPermissions;
import caosdb.server.accessControl.UserSources;
import caosdb.server.accessControl.UserStatus;
-import caosdb.server.database.Database;
import caosdb.server.database.backend.transaction.RetrievePasswordValidator;
import caosdb.server.database.backend.transaction.SetPassword;
import caosdb.server.database.backend.transaction.UpdateUser;
@@ -67,14 +66,14 @@ public class InsertUserTransaction extends AccessControlTransaction {
UpdateUserTransaction.checkEntityExists(this.user.entity);
}
- if (Database.execute(new RetrievePasswordValidator(this.user.name), getAccess()).getValidator()
+ if (execute(new RetrievePasswordValidator(this.user.name), getAccess()).getValidator()
== null) {
if (this.password != null) {
Utils.checkPasswordStrength(this.password);
}
- Database.execute(new SetPassword(this.user.name, this.password), getAccess());
- Database.execute(new UpdateUser(this.user), getAccess());
+ execute(new SetPassword(this.user.name, this.password), getAccess());
+ execute(new UpdateUser(this.user), getAccess());
} else {
throw ServerMessages.ACCOUNT_NAME_NOT_UNIQUE;
}
diff --git a/src/main/java/caosdb/server/transaction/Retrieve.java b/src/main/java/caosdb/server/transaction/Retrieve.java
index 3110ee97e68e38a955ad0ae146228a27eb7677d6..0de1b8e06b66a4293668fb499fb28f33fa398187 100644
--- a/src/main/java/caosdb/server/transaction/Retrieve.java
+++ b/src/main/java/caosdb/server/transaction/Retrieve.java
@@ -22,11 +22,9 @@
*/
package caosdb.server.transaction;
-import caosdb.server.database.Database;
import caosdb.server.database.access.Access;
import caosdb.server.database.backend.transaction.RetrieveFullEntity;
import caosdb.server.entity.EntityInterface;
-import caosdb.server.entity.container.Container;
import caosdb.server.entity.container.RetrieveContainer;
import caosdb.server.entity.xml.SetFieldStrategy;
import caosdb.server.entity.xml.ToElementStrategy;
@@ -137,9 +135,9 @@ public class Retrieve extends Transaction<RetrieveContainer> {
retrieveFullEntities(getContainer(), getAccess());
}
- private static void retrieveFullEntities(
- final Container<? extends EntityInterface> container, final Access access) throws Exception {
- Database.execute(new RetrieveFullEntity(container), access);
+ private void retrieveFullEntities(final RetrieveContainer container, final Access access)
+ throws Exception {
+ execute(new RetrieveFullEntity(container), access);
}
@Override
diff --git a/src/main/java/caosdb/server/transaction/RetrieveLogRecordTransaction.java b/src/main/java/caosdb/server/transaction/RetrieveLogRecordTransaction.java
index 14b5ea3140b685db77be79ff680180fea6e379b3..bbe2e1f11a7bd4c7705a790dec78903fb0e7c20a 100644
--- a/src/main/java/caosdb/server/transaction/RetrieveLogRecordTransaction.java
+++ b/src/main/java/caosdb/server/transaction/RetrieveLogRecordTransaction.java
@@ -23,7 +23,6 @@
package caosdb.server.transaction;
import caosdb.server.accessControl.ACMPermissions;
-import caosdb.server.database.Database;
import caosdb.server.database.DatabaseMonitor;
import caosdb.server.database.access.Access;
import caosdb.server.database.backend.transaction.RetrieveLogRecord;
@@ -64,7 +63,7 @@ public class RetrieveLogRecordTransaction implements TransactionInterface {
final Access access = DatabaseMonitor.getInstance().acquiredWeakAccess(this);
try {
this.logRecords =
- Database.execute(new RetrieveLogRecord(this.logger, this.level, this.message), access)
+ execute(new RetrieveLogRecord(this.logger, this.level, this.message), access)
.getLogRecords();
} finally {
access.release();
diff --git a/src/main/java/caosdb/server/transaction/RetrievePasswordValidatorTransaction.java b/src/main/java/caosdb/server/transaction/RetrievePasswordValidatorTransaction.java
index 15817a5b405d18f80b23e26ac8b16d3a545483dd..d18b5c306373d6b89aa924bd154db444fbdb8f9a 100644
--- a/src/main/java/caosdb/server/transaction/RetrievePasswordValidatorTransaction.java
+++ b/src/main/java/caosdb/server/transaction/RetrievePasswordValidatorTransaction.java
@@ -23,7 +23,6 @@
package caosdb.server.transaction;
import caosdb.server.accessControl.CredentialsValidator;
-import caosdb.server.database.Database;
import caosdb.server.database.backend.transaction.RetrievePasswordValidator;
import caosdb.server.utils.ServerMessages;
@@ -38,8 +37,7 @@ public class RetrievePasswordValidatorTransaction extends AccessControlTransacti
@Override
protected void transaction() throws Exception {
- this.validator =
- Database.execute(new RetrievePasswordValidator(this.name), getAccess()).getValidator();
+ this.validator = execute(new RetrievePasswordValidator(this.name), getAccess()).getValidator();
if (this.validator == null) {
throw ServerMessages.ACCOUNT_DOES_NOT_EXIST;
}
diff --git a/src/main/java/caosdb/server/transaction/RetrievePermissionRulesTransaction.java b/src/main/java/caosdb/server/transaction/RetrievePermissionRulesTransaction.java
index 187a277bf8724eb343e52b84364b7b10fc781641..d02722fd9e89e70c26e8782252c65fb9e199158c 100644
--- a/src/main/java/caosdb/server/transaction/RetrievePermissionRulesTransaction.java
+++ b/src/main/java/caosdb/server/transaction/RetrievePermissionRulesTransaction.java
@@ -22,7 +22,6 @@
*/
package caosdb.server.transaction;
-import caosdb.server.database.Database;
import caosdb.server.database.backend.transaction.RetrievePermissionRules;
import caosdb.server.database.backend.transaction.RetrieveRole;
import caosdb.server.permissions.PermissionRule;
@@ -40,10 +39,10 @@ public class RetrievePermissionRulesTransaction extends AccessControlTransaction
@Override
protected void transaction() throws Exception {
- if (Database.execute(new RetrieveRole(this.role), getAccess()).getRole() == null) {
+ if (execute(new RetrieveRole(this.role), getAccess()).getRole() == null) {
throw ServerMessages.ROLE_DOES_NOT_EXIST;
}
- this.rules = Database.execute(new RetrievePermissionRules(this.role), getAccess()).getRules();
+ this.rules = execute(new RetrievePermissionRules(this.role), getAccess()).getRules();
}
public HashSet<PermissionRule> getRules() {
diff --git a/src/main/java/caosdb/server/transaction/RetrieveRoleTransaction.java b/src/main/java/caosdb/server/transaction/RetrieveRoleTransaction.java
index bcadbac85942a4b1e66e7676c1675647427251ff..4d019f87fb572ce8e75393302764e8146034ee22 100644
--- a/src/main/java/caosdb/server/transaction/RetrieveRoleTransaction.java
+++ b/src/main/java/caosdb/server/transaction/RetrieveRoleTransaction.java
@@ -23,7 +23,6 @@
package caosdb.server.transaction;
import caosdb.server.accessControl.Role;
-import caosdb.server.database.Database;
import caosdb.server.database.backend.transaction.RetrieveRole;
import caosdb.server.utils.ServerMessages;
@@ -38,7 +37,7 @@ public class RetrieveRoleTransaction extends AccessControlTransaction {
@Override
protected void transaction() throws Exception {
- this.role = Database.execute(new RetrieveRole(this.name), getAccess()).getRole();
+ this.role = execute(new RetrieveRole(this.name), getAccess()).getRole();
if (this.role == null) {
throw ServerMessages.ROLE_DOES_NOT_EXIST;
}
diff --git a/src/main/java/caosdb/server/transaction/RetrieveSparseEntityByPath.java b/src/main/java/caosdb/server/transaction/RetrieveSparseEntityByPath.java
index 3530d6dad9ec5a134a9613a0a7ece61f53a20edb..2b936247bba9ece2d4bb9ab5cb1a5a1b2cbada74 100644
--- a/src/main/java/caosdb/server/transaction/RetrieveSparseEntityByPath.java
+++ b/src/main/java/caosdb/server/transaction/RetrieveSparseEntityByPath.java
@@ -22,7 +22,6 @@
*/
package caosdb.server.transaction;
-import caosdb.server.database.Database;
import caosdb.server.database.backend.transaction.GetFileRecordByPath;
import caosdb.server.database.backend.transaction.RetrieveSparseEntity;
import caosdb.server.entity.EntityInterface;
@@ -58,9 +57,9 @@ public class RetrieveSparseEntityByPath extends Transaction<TransactionContainer
protected void transaction() throws Exception {
for (final EntityInterface e : getContainer()) {
final GetFileRecordByPath r =
- Database.execute(new GetFileRecordByPath(e.getFileProperties().getPath()), getAccess());
+ execute(new GetFileRecordByPath(e.getFileProperties().getPath()), getAccess());
e.setId(r.getId());
- Database.execute(new RetrieveSparseEntity(e), getAccess());
+ execute(new RetrieveSparseEntity(e), getAccess());
}
}
diff --git a/src/main/java/caosdb/server/transaction/RetrieveUserTransaction.java b/src/main/java/caosdb/server/transaction/RetrieveUserTransaction.java
index 93c5eb5c638ff8d4ee36d0c1416bd7f980c10378..f2728a64664ad72f113e55a91ef87496810ad7c6 100644
--- a/src/main/java/caosdb/server/transaction/RetrieveUserTransaction.java
+++ b/src/main/java/caosdb/server/transaction/RetrieveUserTransaction.java
@@ -25,7 +25,6 @@ package caosdb.server.transaction;
import caosdb.server.accessControl.Principal;
import caosdb.server.accessControl.UserSources;
import caosdb.server.accessControl.UserStatus;
-import caosdb.server.database.Database;
import caosdb.server.database.backend.transaction.RetrieveUser;
import caosdb.server.database.proto.ProtoUser;
import caosdb.server.utils.ServerMessages;
@@ -46,7 +45,7 @@ public class RetrieveUserTransaction extends AccessControlTransaction {
if (!UserSources.isUserExisting(this.principal)) {
throw ServerMessages.ACCOUNT_DOES_NOT_EXIST;
}
- this.user = Database.execute(new RetrieveUser(this.principal), getAccess()).getUser();
+ this.user = execute(new RetrieveUser(this.principal), getAccess()).getUser();
if (this.user == null) {
this.user = new ProtoUser();
diff --git a/src/main/java/caosdb/server/transaction/Transaction.java b/src/main/java/caosdb/server/transaction/Transaction.java
index de1e5e279ee4b1cd252dda0fdf6bb23e8f93e57f..28b8276e2b93699f2315fb754303194208bb37e5 100644
--- a/src/main/java/caosdb/server/transaction/Transaction.java
+++ b/src/main/java/caosdb/server/transaction/Transaction.java
@@ -25,11 +25,11 @@ package caosdb.server.transaction;
import caosdb.datetime.UTCDateTime;
import caosdb.server.accessControl.AuthenticationUtils;
import caosdb.server.accessControl.Principal;
-import caosdb.server.database.Database;
import caosdb.server.database.DatabaseMonitor;
import caosdb.server.database.access.Access;
import caosdb.server.database.backend.transaction.InsertTransactionHistory;
import caosdb.server.database.exceptions.TransactionException;
+import caosdb.server.database.misc.TransactionBenchmark;
import caosdb.server.entity.EntityInterface;
import caosdb.server.entity.Message;
import caosdb.server.entity.Message.MessageType;
@@ -55,6 +55,11 @@ public abstract class Transaction<C extends TransactionContainer> extends Abstra
public static final Message ERROR_INTEGRITY_VIOLATION =
new Message(MessageType.Error, 0, "This entity caused an unexpected integrity violation.");
+ @Override
+ public TransactionBenchmark getTransactionBenchmark() {
+ return getContainer().getTransactionBenchmark();
+ }
+
private static final DatabaseMonitor monitor = DatabaseMonitor.getInstance();
public static final String CLEAN_UP = "TransactionCleanUp";
@@ -93,7 +98,7 @@ public abstract class Transaction<C extends TransactionContainer> extends Abstra
this.schedule.addAll(Job.loadPermanentContainerJobs(this));
for (final EntityInterface e : getContainer()) {
- final List<Job> loadJobs = Job.loadJobs(e, this);
+ final List<Job> loadJobs = loadContainerFlags.loadJobs(e, this);
this.schedule.addAll(loadJobs);
// additionally load datatype job
@@ -126,21 +131,22 @@ public abstract class Transaction<C extends TransactionContainer> extends Abstra
this.schedule.runJobs(JobExecutionTime.INIT);
getContainer()
.getTransactionBenchmark()
- .addBenchmark(this.getClass().getSimpleName() + ".init", System.currentTimeMillis() - t1);
+ .addMeasurement(
+ this.getClass().getSimpleName() + ".init", System.currentTimeMillis() - t1);
t1 = System.currentTimeMillis();
preCheck();
this.schedule.runJobs(JobExecutionTime.PRE_CHECK);
getContainer()
.getTransactionBenchmark()
- .addBenchmark(
+ .addMeasurement(
this.getClass().getSimpleName() + ".pre_check", System.currentTimeMillis() - t1);
t1 = System.currentTimeMillis();
check();
getContainer()
.getTransactionBenchmark()
- .addBenchmark(
+ .addMeasurement(
this.getClass().getSimpleName() + ".check", System.currentTimeMillis() - t1);
t1 = System.currentTimeMillis();
@@ -148,7 +154,7 @@ public abstract class Transaction<C extends TransactionContainer> extends Abstra
postCheck();
getContainer()
.getTransactionBenchmark()
- .addBenchmark(
+ .addMeasurement(
this.getClass().getSimpleName() + ".post_check", System.currentTimeMillis() - t1);
t1 = System.currentTimeMillis();
@@ -156,7 +162,7 @@ public abstract class Transaction<C extends TransactionContainer> extends Abstra
this.schedule.runJobs(JobExecutionTime.PRE_TRANSACTION);
getContainer()
.getTransactionBenchmark()
- .addBenchmark(
+ .addMeasurement(
this.getClass().getSimpleName() + ".pre_transaction",
System.currentTimeMillis() - t1);
@@ -164,7 +170,7 @@ public abstract class Transaction<C extends TransactionContainer> extends Abstra
transaction();
getContainer()
.getTransactionBenchmark()
- .addBenchmark(
+ .addMeasurement(
this.getClass().getSimpleName() + ".transaction", System.currentTimeMillis() - t1);
t1 = System.currentTimeMillis();
@@ -172,7 +178,7 @@ public abstract class Transaction<C extends TransactionContainer> extends Abstra
postTransaction();
getContainer()
.getTransactionBenchmark()
- .addBenchmark(
+ .addMeasurement(
this.getClass().getSimpleName() + ".post_transaction",
System.currentTimeMillis() - t1);
@@ -180,18 +186,17 @@ public abstract class Transaction<C extends TransactionContainer> extends Abstra
writeHistory();
getContainer()
.getTransactionBenchmark()
- .addBenchmark(
+ .addMeasurement(
this.getClass().getSimpleName() + ".history", System.currentTimeMillis() - t1);
t1 = System.currentTimeMillis();
commit();
getContainer()
.getTransactionBenchmark()
- .addBenchmark(
+ .addMeasurement(
this.getClass().getSimpleName() + ".commit", System.currentTimeMillis() - t1);
} catch (final Exception e) {
- e.printStackTrace();
rollBack();
throw e;
} finally {
@@ -199,7 +204,7 @@ public abstract class Transaction<C extends TransactionContainer> extends Abstra
cleanUp();
getContainer()
.getTransactionBenchmark()
- .addBenchmark(
+ .addMeasurement(
this.getClass().getSimpleName() + ".cleanup", System.currentTimeMillis() - t1);
notifyObservers(CLEAN_UP);
}
@@ -230,7 +235,7 @@ public abstract class Transaction<C extends TransactionContainer> extends Abstra
getTransactor().getPrincipal() == AuthenticationUtils.ANONYMOUS_USER.getPrincipal()
? "anonymous"
: ((Principal) getTransactor().getPrincipal()).getUsername();
- Database.execute(
+ execute(
new InsertTransactionHistory(
getContainer(), this.getClass().getSimpleName(), realm, username, getTimestamp()),
getAccess());
diff --git a/src/main/java/caosdb/server/transaction/TransactionInterface.java b/src/main/java/caosdb/server/transaction/TransactionInterface.java
index 2dfb3d4201a864795ca76bcdbe6872fa76ab8e9d..1aaf2786cc613709c59c92b7102474cead00ca1c 100644
--- a/src/main/java/caosdb/server/transaction/TransactionInterface.java
+++ b/src/main/java/caosdb/server/transaction/TransactionInterface.java
@@ -22,7 +22,26 @@
*/
package caosdb.server.transaction;
+import caosdb.server.database.BackendTransaction;
+import caosdb.server.database.access.Access;
+import caosdb.server.database.misc.RollBackHandler;
+import caosdb.server.database.misc.TransactionBenchmark;
+
public interface TransactionInterface {
public void execute() throws Exception;
+
+ public default TransactionBenchmark getTransactionBenchmark() {
+ return TransactionBenchmark.getRootInstance().getBenchmark(getClass());
+ }
+
+ public default <K extends BackendTransaction> K execute(K t, Access access) {
+ final RollBackHandler handler = (RollBackHandler) access.getHelper("RollBack");
+ handler.append(t);
+ t.setAccess(access);
+
+ t.setTransactionBenchmark(getTransactionBenchmark().getBenchmark(t.getClass()));
+ t.executeTransaction();
+ return t;
+ }
}
diff --git a/src/main/java/caosdb/server/transaction/Update.java b/src/main/java/caosdb/server/transaction/Update.java
index 8cb14a66716c711656acf6af2dd037110b0f5025..8edd27d0aa0250ffebadc8d9a9639490daf1e2c7 100644
--- a/src/main/java/caosdb/server/transaction/Update.java
+++ b/src/main/java/caosdb/server/transaction/Update.java
@@ -23,7 +23,6 @@
package caosdb.server.transaction;
import caosdb.server.CaosDBException;
-import caosdb.server.database.Database;
import caosdb.server.database.access.Access;
import caosdb.server.database.backend.transaction.RetrieveFullEntity;
import caosdb.server.database.backend.transaction.UpdateEntity;
@@ -82,7 +81,7 @@ public class Update extends WriteTransaction<UpdateContainer> {
// retrieve a container which contains all id of those entities
// which are to be updated.
- Database.execute(new RetrieveFullEntity(oldContainer), getAccess());
+ execute(new RetrieveFullEntity(oldContainer), getAccess());
// Check if any updates are to be processed.
for (final EntityInterface newEntity : getContainer()) {
@@ -169,10 +168,9 @@ public class Update extends WriteTransaction<UpdateContainer> {
update(getContainer(), getAccess());
}
- public static void update(final TransactionContainer container, final Access access)
- throws Exception {
+ private void update(final TransactionContainer container, final Access access) throws Exception {
if (container.getStatus().ordinal() >= EntityStatus.QUALIFIED.ordinal()) {
- Database.execute(new UpdateEntity(container), access);
+ execute(new UpdateEntity(container), access);
}
}
diff --git a/src/main/java/caosdb/server/transaction/UpdatePermissionRulesTransaction.java b/src/main/java/caosdb/server/transaction/UpdatePermissionRulesTransaction.java
index 450e89e0d900a34964976d668ff5cb0a3e500cf4..e537b001f2af65e35c65e8c8ad88d37a2fd2bc01 100644
--- a/src/main/java/caosdb/server/transaction/UpdatePermissionRulesTransaction.java
+++ b/src/main/java/caosdb/server/transaction/UpdatePermissionRulesTransaction.java
@@ -23,7 +23,6 @@
package caosdb.server.transaction;
import caosdb.server.accessControl.ACMPermissions;
-import caosdb.server.database.Database;
import caosdb.server.database.backend.transaction.RetrieveRole;
import caosdb.server.database.backend.transaction.SetPermissionRules;
import caosdb.server.permissions.PermissionRule;
@@ -46,10 +45,10 @@ public class UpdatePermissionRulesTransaction extends AccessControlTransaction {
SecurityUtils.getSubject()
.checkPermission(ACMPermissions.PERMISSION_UPDATE_ROLE_PERMISSIONS(this.role));
- if (Database.execute(new RetrieveRole(this.role), getAccess()).getRole() == null) {
+ if (execute(new RetrieveRole(this.role), getAccess()).getRole() == null) {
throw ServerMessages.ROLE_DOES_NOT_EXIST;
}
- Database.execute(new SetPermissionRules(this.role, this.rules), getAccess());
+ execute(new SetPermissionRules(this.role, this.rules), getAccess());
}
}
diff --git a/src/main/java/caosdb/server/transaction/UpdateRoleTransaction.java b/src/main/java/caosdb/server/transaction/UpdateRoleTransaction.java
index e4ca75051e757dfb227585b11153b2ae5c802a82..dfec776005800469654f144a0c5ae6f0213a04ad 100644
--- a/src/main/java/caosdb/server/transaction/UpdateRoleTransaction.java
+++ b/src/main/java/caosdb/server/transaction/UpdateRoleTransaction.java
@@ -25,7 +25,6 @@ package caosdb.server.transaction;
import caosdb.server.accessControl.ACMPermissions;
import caosdb.server.accessControl.Role;
import caosdb.server.accessControl.UserSources;
-import caosdb.server.database.Database;
import caosdb.server.database.backend.transaction.InsertRole;
import caosdb.server.database.backend.transaction.RetrieveRole;
import caosdb.server.utils.ServerMessages;
@@ -48,7 +47,7 @@ public class UpdateRoleTransaction extends AccessControlTransaction {
throw ServerMessages.ROLE_DOES_NOT_EXIST;
}
- Database.execute(new InsertRole(this.role), getAccess());
+ execute(new InsertRole(this.role), getAccess());
RetrieveRole.removeCached(this.role.name);
}
}
diff --git a/src/main/java/caosdb/server/transaction/UpdateUserRolesTransaction.java b/src/main/java/caosdb/server/transaction/UpdateUserRolesTransaction.java
index 6b8a93b0c3da1674e1ab9b1589c85b9b0116f1ec..257e8c06cc9d9c9f24774e541cea611663f5ff0d 100644
--- a/src/main/java/caosdb/server/transaction/UpdateUserRolesTransaction.java
+++ b/src/main/java/caosdb/server/transaction/UpdateUserRolesTransaction.java
@@ -25,7 +25,6 @@ package caosdb.server.transaction;
import caosdb.server.accessControl.ACMPermissions;
import caosdb.server.accessControl.Principal;
import caosdb.server.accessControl.UserSources;
-import caosdb.server.database.Database;
import caosdb.server.database.backend.transaction.RetrieveRole;
import caosdb.server.database.backend.transaction.UpdateUserRoles;
import caosdb.server.utils.ServerMessages;
@@ -56,7 +55,7 @@ public class UpdateUserRolesTransaction extends AccessControlTransaction {
SecurityUtils.getSubject().checkPermission(ACMPermissions.PERMISSION_ASSIGN_ROLE(role));
// test if role is existent
- if (Database.execute(new RetrieveRole(role), getAccess()).getRole() == null) {
+ if (execute(new RetrieveRole(role), getAccess()).getRole() == null) {
throw ServerMessages.ROLE_DOES_NOT_EXIST;
}
}
@@ -65,7 +64,7 @@ public class UpdateUserRolesTransaction extends AccessControlTransaction {
throw ServerMessages.ACCOUNT_DOES_NOT_EXIST;
}
- Database.execute(new UpdateUserRoles(this.realm, this.user, this.roles), getAccess());
+ execute(new UpdateUserRoles(this.realm, this.user, this.roles), getAccess());
}
public Element getUserRolesElement() {
diff --git a/src/main/java/caosdb/server/transaction/UpdateUserTransaction.java b/src/main/java/caosdb/server/transaction/UpdateUserTransaction.java
index 663367c21a34c14392bcd0b60113d1e82e1e0539..ca29bb7977da8fc677c774b30a1f3fa78879c5a7 100644
--- a/src/main/java/caosdb/server/transaction/UpdateUserTransaction.java
+++ b/src/main/java/caosdb/server/transaction/UpdateUserTransaction.java
@@ -26,7 +26,6 @@ import caosdb.server.accessControl.ACMPermissions;
import caosdb.server.accessControl.Principal;
import caosdb.server.accessControl.UserSources;
import caosdb.server.accessControl.UserStatus;
-import caosdb.server.database.Database;
import caosdb.server.database.backend.transaction.RetrieveUser;
import caosdb.server.database.backend.transaction.SetPassword;
import caosdb.server.database.backend.transaction.UpdateUser;
@@ -88,23 +87,22 @@ public class UpdateUserTransaction extends AccessControlTransaction {
if (this.password.isEmpty()) {
// reset password
- Database.execute(new SetPassword(this.user.name, null), getAccess());
+ execute(new SetPassword(this.user.name, null), getAccess());
} else {
Utils.checkPasswordStrength(this.password);
- Database.execute(new SetPassword(this.user.name, this.password), getAccess());
+ execute(new SetPassword(this.user.name, this.password), getAccess());
}
}
if (isToBeUpdated()) {
- Database.execute(new UpdateUser(this.user), getAccess());
+ execute(new UpdateUser(this.user), getAccess());
}
}
private boolean isToBeUpdated() throws Exception {
boolean isToBeUpdated = false;
final ProtoUser validUser =
- Database.execute(
- new RetrieveUser(new Principal(this.user.realm, this.user.name)), getAccess())
+ execute(new RetrieveUser(new Principal(this.user.realm, this.user.name)), getAccess())
.getUser();
if (this.user.status != null && (validUser == null || this.user.status != validUser.status)) {
SecurityUtils.getSubject()
diff --git a/src/main/java/caosdb/server/transaction/WriteTransaction.java b/src/main/java/caosdb/server/transaction/WriteTransaction.java
index a666e9737a50af039d124178ddf507905e38d7ef..e67ae0600bc573efbdf67b9b341299fe1fe3ca00 100644
--- a/src/main/java/caosdb/server/transaction/WriteTransaction.java
+++ b/src/main/java/caosdb/server/transaction/WriteTransaction.java
@@ -25,7 +25,6 @@ package caosdb.server.transaction;
import caosdb.server.database.misc.RollBackHandler;
import caosdb.server.entity.FileProperties;
import caosdb.server.entity.container.TransactionContainer;
-import caosdb.server.utils.ChecksumUpdater;
public abstract class WriteTransaction<C extends TransactionContainer> extends Transaction<C> {
diff --git a/src/main/java/caosdb/server/utils/FileUtils.java b/src/main/java/caosdb/server/utils/FileUtils.java
index 7441e73c1223f39d31ce4017618572486dfd8a64..824646caab6983863acd7d35a276a1213b393235 100644
--- a/src/main/java/caosdb/server/utils/FileUtils.java
+++ b/src/main/java/caosdb/server/utils/FileUtils.java
@@ -303,17 +303,34 @@ public class FileUtils {
return java.nio.file.Files.isDirectory(file.toPath(), LinkOption.NOFOLLOW_LINKS);
}
- private static void moveReplace(File file, File target) throws IOException {
+ private static void moveReplace(File file, File target)
+ throws IOException, CaosDBException, InterruptedException {
if (exists(target)) {
org.apache.commons.io.FileUtils.forceDelete(target);
}
if (isDir(file)) {
org.apache.commons.io.FileUtils.moveDirectory(file, target);
+ } else if (isSymlink(file)) {
+ moveSymlink(file, target);
} else {
org.apache.commons.io.FileUtils.moveFile(file, target);
}
}
+ /**
+ * @param oldlink
+ * @param newlink
+ * @throws IOException
+ * @throws InterruptedException
+ * @throws CaosDBException
+ */
+ private static void moveSymlink(File oldlink, File newlink)
+ throws IOException, CaosDBException, InterruptedException {
+ File target = oldlink.toPath().toRealPath().toFile();
+ createSymlink(newlink, target);
+ callPosixUnlink(oldlink);
+ }
+
public static Undoable rename(final File file, final File target) throws Message, IOException {
final File backup;
if (target.getAbsoluteFile().equals(file.getAbsoluteFile())) {
@@ -325,16 +342,16 @@ public class FileUtils {
public void cleanUp() {}
};
}
- if (exists(target)) {
- // in case this is a update transaction, the old version of the file
- // must be stored somewhere until the transaction is done.
- final File tmp = getTmpFile(target.getName());
- moveReplace(target, tmp);
- backup = tmp;
- } else {
- backup = null;
- }
try {
+ if (exists(target)) {
+ // in case this is a update transaction, the old version of the file
+ // must be stored somewhere until the transaction is done.
+ final File tmp = getTmpFile(target.getName());
+ moveReplace(target, tmp);
+ backup = tmp;
+ } else {
+ backup = null;
+ }
moveReplace(file, target);
return new UndoHandler() {
@@ -383,7 +400,8 @@ public class FileUtils {
org.apache.commons.io.FileUtils.forceDelete(file);
}
- public static Undoable delete(final File file) throws IOException {
+ public static Undoable delete(final File file)
+ throws IOException, CaosDBException, InterruptedException {
return delete(file, false);
}
@@ -397,8 +415,11 @@ public class FileUtils {
* @param recursive
* @return
* @throws IOException
+ * @throws InterruptedException
+ * @throws CaosDBException
*/
- public static Undoable delete(final File file, final boolean recursive) throws IOException {
+ public static Undoable delete(final File file, final boolean recursive)
+ throws IOException, CaosDBException, InterruptedException {
if (file.exists() && file.isFile()) {
// delete file
final File backup = getTmpFile(file.getName());
@@ -496,18 +517,22 @@ public class FileUtils {
return isSymbolicLink(file.toPath());
}
- public static Undoable unlink(final File file)
- throws IOException, InterruptedException, CaosDBException {
- final String target = file.getCanonicalPath();
+ private static void callPosixUnlink(File file) throws IOException, InterruptedException {
final Process cmd = Runtime.getRuntime().exec("unlink " + file.getAbsolutePath());
if (cmd.waitFor() != 0) {
throw new CaosDBException("could not unlink " + file.getAbsolutePath());
}
+ }
+
+ public static Undoable unlink(final File file) throws IOException, InterruptedException {
+ callPosixUnlink(file);
+
+ final String path = file.getCanonicalPath();
return new Undoable() {
@Override
public void undo() {
try {
- Runtime.getRuntime().exec("ln -s " + target + " " + file.getAbsolutePath());
+ Runtime.getRuntime().exec("ln -s " + path + " " + file.getAbsolutePath());
} catch (final IOException e) {
e.printStackTrace();
}
diff --git a/src/main/java/caosdb/server/utils/Info.java b/src/main/java/caosdb/server/utils/Info.java
index e0747548b3ff6bb4675355065567393f2b9c989c..bf85546d8303c072a322afcb9433c77eda928fec 100644
--- a/src/main/java/caosdb/server/utils/Info.java
+++ b/src/main/java/caosdb/server/utils/Info.java
@@ -24,7 +24,6 @@ package caosdb.server.utils;
import caosdb.server.CaosDBServer;
import caosdb.server.FileSystem;
-import caosdb.server.database.Database;
import caosdb.server.database.DatabaseMonitor;
import caosdb.server.database.access.Access;
import caosdb.server.database.backend.transaction.GetInfo;
@@ -37,6 +36,8 @@ import java.io.File;
import java.io.IOException;
import java.sql.SQLException;
import java.util.LinkedList;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
import org.jdom2.Element;
public class Info extends AbstractObservable implements Observer, TransactionInterface {
@@ -44,40 +45,39 @@ public class Info extends AbstractObservable implements Observer, TransactionInt
private static Info instance = new Info();
public static final String SYNC_DATABASE_EVENT = "SyncDatabaseEvent";
private final Access access;
+ public Logger logger = LogManager.getLogger(getClass());
@Override
public boolean notifyObserver(final String e, final Observable o) {
try {
syncDatabase();
} catch (final Exception exc) {
- exc.printStackTrace();
+ logger.error(exc);
}
return true;
}
- static {
+ private Info() {
+ this.access = DatabaseMonitor.getInfoAccess(this);
try {
syncDatabase();
- } catch (final Exception e) {
- e.printStackTrace();
+ } catch (final Exception exc) {
+ logger.error(exc);
}
}
- private Info() {
- this.access = DatabaseMonitor.getInfoAccess(this);
- }
-
public static Info getInstance() {
return instance;
}
+ private Long lastSync = 0L;
private static File dropOffBox = null;
private static int filesCount = -1;
private static int propertiesCount = -1;
private static int recordsCount = -1;
private static int recordTypesCount = -1;
private static int tmpFilesCount;
- private static String fssize;
+ private static String fssize = "";
public static Integer getRecordsCount() throws Exception {
return recordsCount;
@@ -95,17 +95,18 @@ public class Info extends AbstractObservable implements Observer, TransactionInt
return recordTypesCount;
}
- /**
- * Utility function that takes an array of file objects and returns a linked list containing
- * XML representations of all files within each contained directory.
- * All files in the array that are directories are recursively traversed.
- * This function is used for files within the drop off box only.
- *
- * @param files An array of file objects.
- * @return A linked list containing XML elements. The path attribute of these elements is set
- * to the absolute path of the filenames where the path to the drop off box is removed
- * at the beginning.
- */
+ /**
+ * Utility function that takes an array of file objects and returns a linked list containing XML
+ * representations of all files within each contained directory. All files in the array that are
+ * directories are recursively traversed. This function is used for files within the drop off box
+ * only.
+ *
+ * @param files An array of file objects.
+ * @return A linked list containing XML elements. The path attribute of these elements is set to
+ * the absolute path of the filenames where the path to the drop off box is removed at the
+ * beginning.
+ * @fixme Should check if the files are inside the DropOffBox path.
+ */
private static LinkedList<Element> getFlatList(final File[] files) {
try {
final LinkedList<Element> ret = new LinkedList<Element>();
@@ -115,7 +116,7 @@ public class Info extends AbstractObservable implements Observer, TransactionInt
} else {
final Element element = new Element("file");
final String tempPath =
- file.getAbsolutePath().substring(dropOffBox.getCanonicalPath().length() + 1);
+ file.getCanonicalPath().substring(dropOffBox.getCanonicalPath().length() + 1);
element.setAttribute("path", tempPath);
ret.add(element);
}
@@ -144,12 +145,22 @@ public class Info extends AbstractObservable implements Observer, TransactionInt
return ret;
}
- public static void syncDatabase() throws Exception {
- final Access access = getInstance().access;
+ public long timeSinceSync() {
+ long time = System.currentTimeMillis();
+ return time - lastSync;
+ }
+
+ public void syncDatabase() throws Exception {
+ synchronized (lastSync) {
+ if (timeSinceSync() < 10000) {
+ return;
+ }
+ lastSync = System.currentTimeMillis();
+ }
final ProtoInfo i = new ProtoInfo();
final GetInfo t = new GetInfo(i);
- Database.execute(t, access);
+ execute(t, access);
filesCount = i.filesCount;
recordsCount = i.recordsCount;
recordTypesCount = i.recordTypesCount;
@@ -157,7 +168,7 @@ public class Info extends AbstractObservable implements Observer, TransactionInt
fssize = Utils.getReadableByteSize(i.fssize);
tmpFilesCount = countTmpFiles();
- getInstance().notifyObservers(SYNC_DATABASE_EVENT);
+ notifyObservers(SYNC_DATABASE_EVENT);
}
private static int countTmpFiles() throws IOException {
@@ -169,21 +180,26 @@ public class Info extends AbstractObservable implements Observer, TransactionInt
return toElement(false);
}
- /**
- * Generates an XML element showing the information for the /Info resource in CaosDB.
- *
- * @return An XML element containing:
- * - The number of records
- * - The number of properties
- * - The number of record types
- * - The number of files
- * - The total file size
- * - The number of temp files
- * - The path of the DropOffBox
- * - A tree of files in the DropOffBox
- * TODO: The error format for missing or not readable drop off box has to be specified.
- */
+ /**
+ * Generates an XML element showing the information for the /Info resource in CaosDB.
+ *
+ * @return An XML element containing:
+ * <ul>
+ * <li>The number of records
+ * <li>The number of properties
+ * <li>The number of record types
+ * <li>The number of files
+ * <li>The total file size
+ * <li>The number of temp files
+ * <li>The path of the DropOffBox
+ * <li>A tree of files in the DropOffBox
+ * </ul>
+ * TODO: The error format for missing or not readable drop off box has to be specified.
+ */
public static Element toElement(final boolean tree) throws Exception, SQLException {
+ if (filesCount == -1 && recordsCount == -1) {
+ getInstance().syncDatabase();
+ }
dropOffBox = new File(FileSystem.getDropOffBox());
final Element info = new Element("Stats");
final Element counts = new Element("counts");
@@ -198,31 +214,32 @@ public class Info extends AbstractObservable implements Observer, TransactionInt
}
final Element e = new Element("dropOffBox");
if (dropOffBox.isDirectory()) {
- if (dropOffBox.isReadable()) {
- if (tree) {
- e.setAttribute("path", dropOffBox.getAbsolutePath());
- e.addContent(getTree(dropOffBox.listFiles()));
- } else {
- e.setAttribute("path", dropOffBox.getAbsolutePath());
- e.addContent(getFlatList(dropOffBox.listFiles()));
- }
+ if (dropOffBox.canRead()) {
+ if (tree) {
+ e.setAttribute("path", dropOffBox.getAbsolutePath());
+ e.addContent(getTree(dropOffBox.listFiles()));
} else {
- // TODO: return a message that the DropOffBox is not readable.
+ e.setAttribute("path", dropOffBox.getAbsolutePath());
+ e.addContent(getFlatList(dropOffBox.listFiles()));
}
+ } else {
+ // TODO: return a message that the DropOffBox is not readable.
+ }
} else {
- // TODO: This function should at least return a message that the DropOffBox is diabled or not present.
+ // TODO: This function should at least return a message that the DropOffBox is disabled or not
+ // present.
}
info.addContent(counts);
info.addContent(e);
return info;
}
- public static void syncDatabase(final ServerStat stat) throws TransactionException, Message {
+ public void syncDatabase(final ServerStat stat) throws TransactionException, Message {
final Access access = getInstance().access;
final SyncStats t = new SyncStats(stat);
- Database.execute(t, access);
+ execute(t, access);
}
@Override
diff --git a/src/main/java/caosdb/server/utils/WebinterfaceUtils.java b/src/main/java/caosdb/server/utils/WebinterfaceUtils.java
new file mode 100644
index 0000000000000000000000000000000000000000..ecf0b08630e58662c0abad5c5fe71a0c5b1a8889
--- /dev/null
+++ b/src/main/java/caosdb/server/utils/WebinterfaceUtils.java
@@ -0,0 +1,293 @@
+/**
+ * ** header v3.0 This file is a part of the CaosDB Project.
+ *
+ * <p>Copyright (c) 2019 IndiScale GmbH Copyright (c) 2019 Daniel Hornung <d.hornung@indiscale.com>
+ *
+ * <p>This program is free software: you can redistribute it and/or modify it under the terms of the
+ * GNU Affero General Public License as published by the Free Software Foundation, either version 3
+ * of the License, or (at your option) any later version.
+ *
+ * <p>This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY;
+ * without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * <p>You should have received a copy of the GNU Affero General Public License along with this
+ * program. If not, see <https://www.gnu.org/licenses/>.
+ *
+ * <p>** end header
+ */
+package caosdb.server.utils;
+
+import caosdb.server.CaosDBServer;
+import caosdb.server.ServerProperties;
+import caosdb.server.resource.AbstractCaosDBServerResource;
+import caosdb.server.resource.Webinterface;
+import java.io.File;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.stream.Stream;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.restlet.data.Reference;
+
+/**
+ * This class is responsible for two things:
+ * <li>Resolving relative paths of files of the web interface to {@link File} objects.
+ * <li>Generating URIs to public files of the web interface from relative string paths.
+ *
+ * <p>Especially the {@link AbstractCaosDBServerResource} and the {@link Webinterface} use this
+ * class for these tasks.
+ *
+ * @author Timm Fitschen (t.fitschen@indiscale.com)
+ */
+public class WebinterfaceUtils {
+
+ private Logger logger = LogManager.getLogger(this.getClass());
+ public static final String DEFAULT_WEBUI_PUBLIC = "caosdb-webui/public";
+ public static final String DEFAULT_BUILD_NUMBER_FILE = ".build_number";
+ public static final String DEFAULT_ROUTE = "webinterface";
+ private final String host;
+ private final String publicDir;
+ private final String route;
+ private final Path buildNumberFile;
+ private final String contextRoot;
+ private String buildNumber = null;
+ private long buildNumberDate;
+ private static final Map<String, WebinterfaceUtils> instances = new HashMap<>();
+
+ /**
+ * Retrieve an instance of {@link WebinterfaceUtils} for the host. The instance can be shared with
+ * other callers.
+ *
+ * @param host
+ * @return a shared instance of {@link WebinterfaceUtils}.
+ */
+ public static WebinterfaceUtils getInstance(Reference host) {
+ return getInstance(host.getHostIdentifier());
+ }
+
+ /**
+ * Retrieve an instance of {@link WebinterfaceUtils} for the host. The instance can be shared with
+ * other callers.
+ *
+ * @param host
+ * @return a shared instance of {@link WebinterfaceUtils}.
+ */
+ public static WebinterfaceUtils getInstance(String host) {
+ synchronized (instances) {
+ WebinterfaceUtils ret = instances.get(host);
+ if (ret == null) {
+ ret = new WebinterfaceUtils(host);
+ instances.put(host, ret);
+ }
+ return ret;
+ }
+ }
+
+ /**
+ * Create a utility class for creating references to resources of the web interface.
+ *
+ * @param host
+ * @param contextRoot
+ * @param publicDir
+ * @param route
+ * @param buildNumberFile
+ */
+ WebinterfaceUtils(
+ String host, String contextRoot, String publicDir, String route, String buildNumberFile) {
+ this.host = host;
+ this.contextRoot = contextRoot;
+ this.publicDir = publicDir;
+ this.route = route;
+ this.buildNumberFile = getPublicRootPath().resolve(buildNumberFile);
+ }
+
+ /**
+ * Create a utility class for creating references to resources of the web interface.
+ *
+ * <p>The route to the web interface, the build number file and the public folder of the web
+ * interface build directory are set to default values ({@link #DEFAULT_BUILD_NUMBER_FILE}, {@link
+ * #DEFAULT_ROUTE}, and {@link #DEFAULT_WEBUI_PUBLIC}).
+ *
+ * <p>The contextRoute defaults to the server property {@link ServerProperties.KEY_CONTEXT_ROOT}.
+ *
+ * @param host - the host of the web interface.
+ * @see {@link WebinterfaceUtils#WebinterfaceUtils(String, String, String, String)}.
+ */
+ WebinterfaceUtils(Reference host) {
+ this(host.getHostIdentifier());
+ }
+
+ /**
+ * Create a utility class for creating references to resources of the web interface.
+ *
+ * <p>The route to the web interface, the build number file and the public folder of the web
+ * interface build directory are set to default values ({@link #DEFAULT_BUILD_NUMBER_FILE}, {@link
+ * #DEFAULT_ROUTE}, and {@link #DEFAULT_WEBUI_PUBLIC}).
+ *
+ * <p>The contextRoute defaults to the server property {@link ServerProperties.KEY_CONTEXT_ROOT}.
+ *
+ * @param host - the host of the web interface.
+ * @see {@link WebinterfaceUtils#WebinterfaceUtils(String, String, String, String)}.
+ */
+ WebinterfaceUtils(String host) {
+ this(
+ host,
+ CaosDBServer.getServerProperty(ServerProperties.KEY_CONTEXT_ROOT),
+ DEFAULT_WEBUI_PUBLIC,
+ DEFAULT_ROUTE,
+ DEFAULT_BUILD_NUMBER_FILE);
+ }
+
+ /**
+ * Return the server root reference.
+ *
+ * <p>The server root is composed of the scheme (https://) the host name (example.com) the port if
+ * necessary (:10443) and the context root of the application (beta).
+ *
+ * @return the server root reference e.g. https://example.com:10443/beta.
+ */
+ public String getServerRootURI() {
+ if (contextRoot == null || contextRoot.length() == 0) {
+ return host;
+ }
+ return host + "/" + contextRoot;
+ }
+
+ /**
+ * Return the root reference for the web interface.
+ *
+ * <p>The root reference for the web interface is the server root reference ({@link
+ * #getServerRootURI()}) plus the route to the resource which routes to the individual resources
+ * of the web interface ({@link Webinterface}).
+ *
+ * @return root reference for the web interface.
+ */
+ public String getWebinterfaceRootURI() {
+ return getServerRootURI() + "/" + this.route + "/" + getBuildNumber();
+ }
+
+ /**
+ * Return the absolute path of the root of the public directory of the webui.
+ *
+ * @return Public directory of webui.
+ */
+ public Path getPublicRootPath() {
+ return Paths.get(this.publicDir).toAbsolutePath();
+ }
+
+ /**
+ * return the Path of the build number file.
+ *
+ * @return
+ */
+ public Path getBuildNumberFile() {
+ return this.buildNumberFile;
+ }
+
+ /**
+ * Determine the build number of the webui. Returns null if the build number could not be
+ * determined.
+ *
+ * <p>Because opening and reading the {@link #buildNumberFile} content is expensive, the
+ * buildNumber is cached.
+ *
+ * <p>The cached buildNumber invalidates when the files modified time stamp is younger than the
+ * {@link #buildNumberDate}. The validity of the cache is checked once every minute with a {@link
+ * CronJob}.
+ *
+ * @return Build number or null.
+ */
+ public String getBuildNumber() {
+ validateBuildNumber();
+ if (this.buildNumber == null) {
+ this.buildNumberDate = System.currentTimeMillis();
+ this.buildNumber = readBuildNumber();
+ }
+ return this.buildNumber;
+ }
+
+ /**
+ * Return the latest buildNumber, freshly read from the buildNumberfile.
+ *
+ * @return fresh build number.
+ */
+ public String readBuildNumber() {
+ Path buildNumberFile = getBuildNumberFile();
+ StringBuilder contentBuilder = new StringBuilder();
+ try (Stream<String> stream = Files.lines(buildNumberFile, StandardCharsets.UTF_8)) {
+ stream.forEach(s -> contentBuilder.append(s));
+ } catch (IOException e) {
+ logger.error(e);
+ return null;
+ }
+ return contentBuilder.toString();
+ }
+
+ /**
+ * Return the public file or null if the file does not exists or is not in the public directory of
+ * the web interface.
+ *
+ * <p>This method is used to resolve requests for files of the web interface.
+ *
+ * @param path
+ * @return The file or null
+ */
+ public File getPublicFile(String path) {
+ Path publicFilePath = getPublicFilePath(path);
+ if (publicFilePath != null) {
+ File publicFile = publicFilePath.toFile();
+ if (publicFile.exists() && !publicFile.isDirectory() && publicFile.canRead())
+ return publicFile;
+ }
+ return null;
+ }
+
+ /**
+ * Return the absolute path to the public file denoted by the path.
+ *
+ * <p>This method is used to resolve requests for files of the web interface.
+ *
+ * @param path - path to the file of the web interface, relative to the public directory of the
+ * web interface.
+ * @return The absolute path of the web interface file.
+ */
+ public Path getPublicFilePath(String path) {
+ Path root = getPublicRootPath();
+ Path resolved = root.resolve(path).toAbsolutePath().normalize();
+ if (resolved.startsWith(root)) {
+ return resolved;
+ }
+ return null;
+ }
+
+ /**
+ * Return a URI to a public file of the web interface.
+ *
+ * <p>This method is used to generate references for *.xsl files or other resources of the web
+ * interface.
+ *
+ * @param publicFile - the path of the public file, relative to the public directory of the web
+ * interface.
+ * @return a URI for the public file of the web interface.
+ */
+ public String getWebinterfaceURI(String publicFile) {
+ return getWebinterfaceRootURI() + "/" + publicFile;
+ }
+
+ /**
+ * Check the validity of the buildNumber and invalidate the buildNumber if the buildNumberDate is
+ * older than the {@link File#lastModified} time stamp of the buildNumberFile.
+ */
+ public void validateBuildNumber() {
+ if (buildNumber != null
+ && getBuildNumberFile().toFile().lastModified() > this.buildNumberDate) {
+ this.buildNumber = null;
+ }
+ }
+}
diff --git a/src/test/docker/Dockerfile b/src/test/docker/Dockerfile
index 9e7bd9a73a3a52e052f4b5953adf1178254f6f03..d8b831eebbda33a3143f89a8da1edb770788bce5 100644
--- a/src/test/docker/Dockerfile
+++ b/src/test/docker/Dockerfile
@@ -1,4 +1,4 @@
FROM debian:stretch
RUN apt-get update && \
apt-get install git make mariadb-server maven openjdk-8-jdk-headless \
- python3-pip screen libpam0g-dev unzip curl -y
+ python3-pip screen libpam0g-dev unzip curl shunit2 -y
diff --git a/src/test/java/caosdb/server/Misc.java b/src/test/java/caosdb/server/Misc.java
index e94d91513be40459910a05559a6f526bb94e9872..bcdc1729f008dbb1393fe238a3eb763c67d69e0f 100644
--- a/src/test/java/caosdb/server/Misc.java
+++ b/src/test/java/caosdb/server/Misc.java
@@ -25,7 +25,6 @@ package caosdb.server;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertTrue;
import caosdb.server.database.misc.TransactionBenchmark;
@@ -37,12 +36,10 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
-import java.util.Properties;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import org.apache.commons.jcs.JCS;
-import org.apache.commons.jcs.access.CacheAccess;
import org.apache.shiro.SecurityUtils;
+import org.apache.shiro.config.Ini;
import org.apache.shiro.config.IniSecurityManagerFactory;
import org.apache.shiro.mgt.SecurityManager;
import org.apache.shiro.subject.Subject;
@@ -204,26 +201,6 @@ public class Misc {
assertEquals(null, m.group(2));
}
- @Test
- public void testCache() throws IOException {
- final Properties p = new Properties();
- p.load(
- new FileInputStream(
- System.getProperty("user.dir") + "/src/test/java/caosdb/server/cache.ccf"));
- JCS.setConfigProperties(p);
-
- final CacheAccess<Object, Object> cache = JCS.getInstance("default");
- JCS.getInstance("default");
-
- final String key = "KEY";
- final String value = "VALUE";
-
- cache.put(key, value);
-
- assertEquals(value, cache.get(key));
- assertSame(value, cache.get(key));
- }
-
@Test
public void testCrobJob() throws InterruptedException {
new CronJob(
@@ -294,11 +271,11 @@ public class Misc {
@Test
public void benchmarkSerialization() throws IOException, ClassNotFoundException {
- TransactionBenchmark.getInstance().addBenchmark("bla", 1000);
+ TransactionBenchmark.getRootInstance().addMeasurement("bla", 1000);
final FileOutputStream fileOut = new FileOutputStream("/tmp/benchmark.java.ser");
final ObjectOutputStream out = new ObjectOutputStream(fileOut);
- out.writeObject(TransactionBenchmark.getInstance());
+ out.writeObject(TransactionBenchmark.getRootInstance());
out.close();
fileOut.close();
System.out.printf("Serialized data is saved in /tmp/benchmark.java.ser");
@@ -316,7 +293,8 @@ public class Misc {
@Test
public void testShiro() {
- final Factory<SecurityManager> factory = new IniSecurityManagerFactory();
+ Ini config = CaosDBServer.getShiroConfig();
+ final Factory<SecurityManager> factory = new IniSecurityManagerFactory(config);
final SecurityManager securityManager = factory.getInstance();
diff --git a/src/test/java/caosdb/server/cache.ccf b/src/test/java/caosdb/server/cache.ccf
deleted file mode 100644
index d57b1c9f63d44f6436532e686b3609dd33df00d3..0000000000000000000000000000000000000000
--- a/src/test/java/caosdb/server/cache.ccf
+++ /dev/null
@@ -1,4 +0,0 @@
-jcs.default=DC
-jcs.default.cacheattributes=org.apache.commons.jcs.engine.CompositeCacheAttributes
-jcs.default.cacheattributes.MaxObjects=1000
-jcs.default.cacheattributes.MemoryCacheName=org.apache.commons.jcs.engine.memory.lru.LRUMemoryCache
\ No newline at end of file
diff --git a/src/test/java/caosdb/server/caching/TestCaching.java b/src/test/java/caosdb/server/caching/TestCaching.java
new file mode 100644
index 0000000000000000000000000000000000000000..56d8e750ab75663e29808a0f97962df88006d080
--- /dev/null
+++ b/src/test/java/caosdb/server/caching/TestCaching.java
@@ -0,0 +1,52 @@
+package caosdb.server.caching;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertSame;
+
+import caosdb.server.CaosDBServer;
+import caosdb.server.accessControl.Pam;
+import caosdb.server.database.backend.transaction.RetrieveProperties;
+import java.io.IOException;
+import org.apache.commons.jcs.JCS;
+import org.apache.commons.jcs.access.CacheAccess;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestCaching {
+
+ @BeforeClass
+ public static void init() throws IOException {
+ CaosDBServer.initServerProperties();
+ JCSCacheHelper.init();
+ }
+
+ @Test
+ public void testCacheConfig() {
+ CacheAccess<String, String> retrieve_properties_cache =
+ JCS.getInstance(RetrieveProperties.CACHE_REGION);
+ assertEquals(1003, retrieve_properties_cache.getCacheAttributes().getMaxObjects());
+
+ CacheAccess<String, String> retrieve_entities_cache = JCS.getInstance("BACKEND_SparseEntities");
+ assertEquals(1002, retrieve_entities_cache.getCacheAttributes().getMaxObjects());
+
+ CacheAccess<String, String> pam_groups_cache = JCS.getInstance(Pam.CACHE_REGION_GROUPS);
+ assertEquals(1000, pam_groups_cache.getCacheAttributes().getMaxObjects());
+ assertEquals(false, pam_groups_cache.getCacheAttributes().isUseMemoryShrinker());
+ assertEquals(false, pam_groups_cache.getDefaultElementAttributes().getIsEternal());
+ assertEquals(61, pam_groups_cache.getDefaultElementAttributes().getIdleTime());
+ assertEquals(601, pam_groups_cache.getDefaultElementAttributes().getMaxLife());
+ }
+
+ @Test
+ public void testCacheElements() throws IOException {
+ final CacheAccess<Object, Object> cache = JCS.getInstance("default");
+
+ final String key = "KEY";
+ final String value = "VALUE";
+
+ cache.put(key, value);
+
+ assertEquals(value, cache.get(key));
+ assertSame(value, cache.get(key));
+ }
+}
diff --git a/src/test/java/caosdb/server/jobs/core/TestInsertFilesInDir.java b/src/test/java/caosdb/server/jobs/core/TestInsertFilesInDir.java
new file mode 100644
index 0000000000000000000000000000000000000000..37cad0eca28a50b006985a73edb07e45b60fdf9e
--- /dev/null
+++ b/src/test/java/caosdb/server/jobs/core/TestInsertFilesInDir.java
@@ -0,0 +1,19 @@
+package caosdb.server.jobs.core;
+
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.IOException;
+import org.junit.Test;
+
+public class TestInsertFilesInDir {
+
+ @Test
+ public void testExclude() throws IOException {
+ InsertFilesInDir job = new InsertFilesInDir();
+ job.init(null, null, null);
+ job.parseValue("-e ^.*test.*$ test");
+ File testFile = new File("test.dat");
+ assertTrue(job.isExcluded(testFile));
+ }
+}
diff --git a/src/test/java/caosdb/server/logging/TestLogging.java b/src/test/java/caosdb/server/logging/TestLogging.java
index 35667f35363af5165596ea1f32fa2f96963595aa..6e35d96585683d99c81a0fc2c669d4dc977bc09a 100644
--- a/src/test/java/caosdb/server/logging/TestLogging.java
+++ b/src/test/java/caosdb/server/logging/TestLogging.java
@@ -51,6 +51,8 @@ public class TestLogging {
@Test
public void testRequestTimeLogger() {
+ Assert.assertTrue(CaosDBServer.isDebugMode());
Assert.assertFalse(request_time_logger.isErrorEnabled());
+ Assert.assertFalse(request_time_logger.isTraceEnabled());
}
}
diff --git a/src/test/java/caosdb/server/query/TestCQL.java b/src/test/java/caosdb/server/query/TestCQL.java
index cae8b26175ebd0d0fceca0753e2bb5629203dbf8..d5a939577b274f4a01355be0a8a8fd1b2a948756 100644
--- a/src/test/java/caosdb/server/query/TestCQL.java
+++ b/src/test/java/caosdb/server/query/TestCQL.java
@@ -230,6 +230,8 @@ public class TestCQL {
String filepath_pat03 = "/foo\\\\\\\\*/"; // -> \\\\* (4)
String filepath_pat04 = "/foo**/";
+ String referenceByLikePattern = "FIND ENTITY WHICH IS REFERENCED BY *name*";
+
@Test
public void testQuery1()
throws InterruptedException, SQLException, ConnectionException, QueryException {
@@ -6154,4 +6156,26 @@ public class TestCQL {
assertTrue(StoredAt.requiresPatternMatching(filepath_pat03));
assertTrue(StoredAt.requiresPatternMatching(filepath_pat04));
}
+
+ /** String referenceByLikePattern = "FIND ENTITY WHICH IS REFERENCED BY *name*"; */
+ @Test
+ public void testReferencedByLike() {
+ CQLLexer lexer;
+ lexer = new CQLLexer(CharStreams.fromString(this.referenceByLikePattern));
+ final CommonTokenStream tokens = new CommonTokenStream(lexer);
+
+ final CQLParser parser = new CQLParser(tokens);
+ final CqContext sfq = parser.cq();
+
+ assertEquals(Query.Role.ENTITY, sfq.r);
+ assertNotNull(sfq.filter);
+ assertEquals(Backreference.class.getName(), sfq.filter.getClass().getName());
+ final EntityFilterInterface f = sfq.filter;
+
+ assertNotNull(f);
+ assertEquals("@(%name%,null)", f.toString());
+
+ assertTrue(f instanceof Backreference);
+ assertEquals("%name%", ((Backreference) f).getEntity());
+ }
}
diff --git a/src/test/java/caosdb/server/resource/TestScriptingResource.java b/src/test/java/caosdb/server/resource/TestScriptingResource.java
index 3b035fe703409605c59a77df70421e7516230376..2edfb82b53b56324996293a469fadf807264d86c 100644
--- a/src/test/java/caosdb/server/resource/TestScriptingResource.java
+++ b/src/test/java/caosdb/server/resource/TestScriptingResource.java
@@ -23,23 +23,147 @@
package caosdb.server.resource;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import caosdb.server.CaosDBServer;
+import caosdb.server.accessControl.AuthenticationUtils;
+import caosdb.server.accessControl.CredentialsValidator;
+import caosdb.server.accessControl.Principal;
+import caosdb.server.accessControl.Role;
+import caosdb.server.accessControl.SessionToken;
+import caosdb.server.database.BackendTransaction;
+import caosdb.server.database.access.Access;
+import caosdb.server.database.backend.interfaces.RetrievePasswordValidatorImpl;
+import caosdb.server.database.backend.interfaces.RetrievePermissionRulesImpl;
+import caosdb.server.database.backend.interfaces.RetrieveRoleImpl;
+import caosdb.server.database.backend.interfaces.RetrieveUserImpl;
+import caosdb.server.database.exceptions.TransactionException;
+import caosdb.server.database.misc.TransactionBenchmark;
+import caosdb.server.database.proto.ProtoUser;
import caosdb.server.entity.Message;
+import caosdb.server.permissions.PermissionRule;
import java.io.IOException;
import java.util.Date;
+import java.util.HashSet;
import java.util.List;
+import org.apache.shiro.SecurityUtils;
+import org.apache.shiro.config.Ini;
+import org.apache.shiro.config.IniSecurityManagerFactory;
+import org.apache.shiro.mgt.SecurityManager;
+import org.apache.shiro.subject.Subject;
+import org.apache.shiro.util.Factory;
+import org.junit.BeforeClass;
import org.junit.Test;
import org.restlet.Request;
import org.restlet.Response;
import org.restlet.data.Form;
import org.restlet.data.MediaType;
import org.restlet.data.Method;
+import org.restlet.data.Reference;
import org.restlet.data.Status;
import org.restlet.representation.Representation;
import org.restlet.representation.StringRepresentation;
public class TestScriptingResource {
+ public static class RetrieveRole implements RetrieveRoleImpl {
+
+ public RetrieveRole(Access a) {}
+
+ @Override
+ public Role retrieve(String role) throws TransactionException {
+ Role ret = new Role();
+ ret.name = "anonymous";
+ ret.description = "bla";
+ return ret;
+ }
+
+ @Override
+ public TransactionBenchmark getBenchmark() {
+ return null;
+ }
+
+ @Override
+ public void setTransactionBenchmark(TransactionBenchmark b) {}
+ }
+
+ public static class RetrievePermissionRules implements RetrievePermissionRulesImpl {
+
+ public RetrievePermissionRules(Access a) {}
+
+ @Override
+ public HashSet<PermissionRule> retrievePermissionRule(String role) throws TransactionException {
+ return new HashSet<>();
+ }
+
+ @Override
+ public TransactionBenchmark getBenchmark() {
+ return null;
+ }
+
+ @Override
+ public void setTransactionBenchmark(TransactionBenchmark b) {}
+ }
+
+ public static class RetrieveUser implements RetrieveUserImpl {
+
+ public RetrieveUser(Access a) {}
+
+ @Override
+ public ProtoUser execute(Principal principal) throws TransactionException {
+ return new ProtoUser();
+ }
+
+ @Override
+ public TransactionBenchmark getBenchmark() {
+ return null;
+ }
+
+ @Override
+ public void setTransactionBenchmark(TransactionBenchmark b) {}
+ }
+
+ public static class RetrievePasswordValidator implements RetrievePasswordValidatorImpl {
+
+ public RetrievePasswordValidator(Access a) {}
+
+ @Override
+ public CredentialsValidator<String> execute(String name) throws TransactionException {
+ return new CredentialsValidator<String>() {
+
+ @Override
+ public boolean isValid(String credential) {
+ return true;
+ }
+ };
+ }
+
+ @Override
+ public void setTransactionBenchmark(TransactionBenchmark b) {}
+
+ @Override
+ public TransactionBenchmark getBenchmark() {
+ return null;
+ }
+ }
+
+ @BeforeClass
+ public static void setupShiro() throws IOException {
+ BackendTransaction.setImpl(RetrieveRoleImpl.class, RetrieveRole.class);
+ BackendTransaction.setImpl(RetrievePermissionRulesImpl.class, RetrievePermissionRules.class);
+ BackendTransaction.setImpl(RetrieveUserImpl.class, RetrieveUser.class);
+ BackendTransaction.setImpl(
+ RetrievePasswordValidatorImpl.class, RetrievePasswordValidator.class);
+
+ CaosDBServer.initServerProperties();
+ Ini config = CaosDBServer.getShiroConfig();
+ final Factory<SecurityManager> factory = new IniSecurityManagerFactory(config);
+
+ final SecurityManager securityManager = factory.getInstance();
+
+ SecurityUtils.setSecurityManager(securityManager);
+ }
+
ScriptingResource resource =
new ScriptingResource() {
@Override
@@ -60,6 +184,12 @@ public class TestScriptingResource {
@Test
public void testUnsupportedMediaType() {
+ Subject user = SecurityUtils.getSubject();
+ if (user.isAuthenticated()) {
+ user.logout();
+ }
+ SessionToken t = SessionToken.generate(new Principal("CaosDB", "user"), null);
+ user.login(t);
Representation entity = new StringRepresentation("asdf");
entity.setMediaType(MediaType.TEXT_ALL);
Request request = new Request(Method.POST, "../test", entity);
@@ -70,6 +200,24 @@ public class TestScriptingResource {
assertEquals(Status.CLIENT_ERROR_UNSUPPORTED_MEDIA_TYPE, resource.getResponse().getStatus());
}
+ @Test
+ public void testAnonymous() {
+ Subject user = SecurityUtils.getSubject();
+ user.login(AuthenticationUtils.ANONYMOUS_USER);
+ assertTrue(resource.isAnonymous());
+ Representation entity = new StringRepresentation("asdf");
+ entity.setMediaType(MediaType.TEXT_ALL);
+ Request request = new Request(Method.POST, "../test", entity);
+ request.setRootRef(new Reference("bla"));
+ request.getAttributes().put("SRID", "asdf1234");
+ request.setDate(new Date());
+ request.setHostRef("bla");
+ resource.init(null, request, new Response(null));
+
+ resource.handle();
+ assertEquals(Status.CLIENT_ERROR_FORBIDDEN, resource.getResponse().getStatus());
+ }
+
@Test
public void testForm2invocation() throws Message {
Form form =
diff --git a/src/test/java/caosdb/server/scripting/TestServerSideScriptingCaller.java b/src/test/java/caosdb/server/scripting/TestServerSideScriptingCaller.java
index e66c646a4e5604f2b7df2dac0af689aa8797160a..f752c29c2f99b33d30a75ff3e09c7043c92fc464 100644
--- a/src/test/java/caosdb/server/scripting/TestServerSideScriptingCaller.java
+++ b/src/test/java/caosdb/server/scripting/TestServerSideScriptingCaller.java
@@ -613,9 +613,7 @@ public class TestServerSideScriptingCaller extends CaosDBTestClass {
caller.cleanup();
}
-/**
- * Does the order of directory creation matter?
- */
+ /** Does the order of directory creation matter? */
@Test
public void testDirectoriesInWrongOrder() throws Message {
final String[] cmd = {testExecutable.getAbsolutePath()};
@@ -645,9 +643,7 @@ public class TestServerSideScriptingCaller extends CaosDBTestClass {
caller.cleanup();
}
- /**
- * Is the new home directory created correctly?
- */
+ /** Is the new home directory created correctly? */
@Test
public void testWorkingDirCreation() throws Exception {
final String[] cmd = {testExecutable.getAbsolutePath()};
@@ -660,9 +656,7 @@ public class TestServerSideScriptingCaller extends CaosDBTestClass {
caller.cleanup();
}
- /**
- * Does copying files over to the new home directory work?
- */
+ /** Does copying files over to the new home directory work? */
@Test
public void testWorkingDirCopying() throws Exception {
final String[] cmd = {testExecutable.getAbsolutePath()};
diff --git a/src/test/java/caosdb/server/utils/FileUtilsTest.java b/src/test/java/caosdb/server/utils/FileUtilsTest.java
index 183f5cf834783f7ad9c6e63bcdb25df1c7870822..e4b2411dc0e77ce0225c0637422c84bbab2cc4aa 100644
--- a/src/test/java/caosdb/server/utils/FileUtilsTest.java
+++ b/src/test/java/caosdb/server/utils/FileUtilsTest.java
@@ -31,7 +31,7 @@ import caosdb.server.CaosDBException;
import caosdb.server.CaosDBServer;
import caosdb.server.FileSystem;
import caosdb.server.ServerProperties;
-import caosdb.server.database.Database;
+import caosdb.server.database.BackendTransaction;
import caosdb.server.database.access.Access;
import caosdb.server.database.backend.implementation.UnixFileSystem.UnixFileSystemGetFileIterator.FileNameIterator;
import caosdb.server.database.backend.implementation.UnixFileSystem.UnixFileSystemHelper;
@@ -69,9 +69,11 @@ public class FileUtilsTest {
public static File tmpFolderCaosDB;
@BeforeClass
- public static void setup() throws Message, IOException {
+ public static void setup() throws Message, IOException, CaosDBException, InterruptedException {
CaosDBServer.initServerProperties();
testRoot = tempFolder.newFolder("fileutils_testfolder");
+ File basePath = tempFolder.newFolder("caosdbRoot");
+ File dropOffBox = tempFolder.newFolder("dropOffBox");
someDir = testRoot.toPath().resolve("some_dir").toFile();
linkToSomeDir = testRoot.toPath().resolve("link_to_some_dir").toFile();
someFile = testRoot.toPath().resolve("some_file").toFile();
@@ -79,12 +81,15 @@ public class FileUtilsTest {
linkToSomeFile = testRoot.toPath().resolve("link_to_some_file").toFile();
tmpFolderCaosDB = tempFolder.newFolder();
+ CaosDBServer.setProperty(ServerProperties.KEY_FILE_SYSTEM_ROOT, basePath.toString());
+ CaosDBServer.setProperty(ServerProperties.KEY_DROP_OFF_BOX, dropOffBox.toString());
CaosDBServer.setProperty(ServerProperties.KEY_TMP_FILES, tmpFolderCaosDB.toString());
FileSystem.init();
- Assert.assertTrue(new File(FileSystem.getBasepath()).canWrite());
- Assert.assertTrue(new File(FileSystem.getBasepath()).canRead());
- Assert.assertTrue(new File(FileSystem.getBasepath()).canExecute());
+ basePath = new File(FileSystem.getBasepath());
+ Assert.assertTrue(basePath.canWrite());
+ Assert.assertTrue(basePath.canRead());
+ Assert.assertTrue(basePath.canExecute());
Assert.assertTrue(new File(FileSystem.getTmp()).canWrite());
Assert.assertTrue(new File(FileSystem.getTmp()).canRead());
Assert.assertTrue(new File(FileSystem.getTmp()).canExecute());
@@ -102,7 +107,7 @@ public class FileUtilsTest {
}
@AfterClass
- public static void teardown() throws IOException {
+ public static void teardown() throws IOException, CaosDBException, InterruptedException {
System.err.println("teardown");
FileUtils.delete(testRoot, true);
deleteTmp();
@@ -110,9 +115,13 @@ public class FileUtilsTest {
// assertEquals(0, new File(FileSystem.getTmp()).listFiles().length);
}
- /** @fixme Currently still fails due to https://github.com/junit-team/junit4/issues/1223 */
+ /**
+ * @throws InterruptedException
+ * @throws CaosDBException
+ * @fixme Currently still fails due to https://github.com/junit-team/junit4/issues/1223
+ */
@Before
- public void testTmpEmpty() throws IOException {
+ public void testTmpEmpty() throws IOException, CaosDBException, InterruptedException {
if (new File(FileSystem.getTmp()).exists()) {
if (0 != new File(FileSystem.getTmp()).list().length) {
System.err.println("TMP not empty, aborting test!");
@@ -128,7 +137,7 @@ public class FileUtilsTest {
}
}
- public static void deleteTmp() throws IOException {
+ public static void deleteTmp() throws IOException, CaosDBException, InterruptedException {
File tmpDir = new File(FileSystem.getTmp());
if (tmpDir.exists())
for (File f : tmpDir.listFiles()) {
@@ -138,7 +147,7 @@ public class FileUtilsTest {
}
@After
- public void callDeleteTmp() throws IOException {
+ public void callDeleteTmp() throws IOException, CaosDBException, InterruptedException {
deleteTmp();
}
@@ -548,7 +557,7 @@ public class FileUtilsTest {
access.setHelper("UnixFileSystemHelper", h);
FileConsistencyCheck check =
- Database.execute(new FileConsistencyCheck("test1.dat", 0, null, 0L, null), access);
+ execute(new FileConsistencyCheck("test1.dat", 0, null, 0L, null), access);
assertEquals(FileConsistencyCheck.FILE_DOES_NOT_EXIST, check.getResult());
final File f = new File(rootPath + "test1.dat");
@@ -564,9 +573,7 @@ public class FileUtilsTest {
final long size = f.length();
final long timestamp = f.lastModified();
- check =
- Database.execute(
- new FileConsistencyCheck("test1.dat", size, hash, timestamp, hasher), access);
+ check = execute(new FileConsistencyCheck("test1.dat", size, hash, timestamp, hasher), access);
assertEquals(FileConsistencyCheck.OK, check.getResult());
// timestamp has 1 second accuracy
@@ -583,9 +590,7 @@ public class FileUtilsTest {
assertTrue(timestamp2 > timestamp);
assertTrue(f.exists());
- check =
- Database.execute(
- new FileConsistencyCheck("test1.dat", size2, hash, timestamp, hasher), access);
+ check = execute(new FileConsistencyCheck("test1.dat", size2, hash, timestamp, hasher), access);
assertEquals(FileConsistencyCheck.OK, check.getResult());
out = new PrintStream(f);
@@ -593,12 +598,16 @@ public class FileUtilsTest {
out.flush();
out.close();
- check =
- Database.execute(
- new FileConsistencyCheck("test1.dat", size, hash, timestamp, hasher), access);
+ check = execute(new FileConsistencyCheck("test1.dat", size, hash, timestamp, hasher), access);
assertEquals(FileConsistencyCheck.FILE_MODIFIED, check.getResult());
}
+ private <K extends BackendTransaction> K execute(K t, Access access2) {
+ t.setAccess(access2);
+ t.executeTransaction();
+ return t;
+ }
+
@Test
public void testFileIterator() throws IOException {
final String rootPath = "./testFileIterator/";
@@ -665,8 +674,7 @@ public class FileUtilsTest {
dat3.deleteOnExit();
dat3.createNewFile();
- final Iterator<String> iterator =
- Database.execute(new GetFileIterator(rootPath), access).getIterator();
+ final Iterator<String> iterator = execute(new GetFileIterator(rootPath), access).getIterator();
assertEquals(rootPath + "dat1", iterator.next());
assertEquals(rootPath + "dat2", iterator.next());
diff --git a/src/test/java/caosdb/server/utils/WebinterfaceUtilsTest.java b/src/test/java/caosdb/server/utils/WebinterfaceUtilsTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..4ff5d7185d43704e4bc97ef435a5c853e58e3eb8
--- /dev/null
+++ b/src/test/java/caosdb/server/utils/WebinterfaceUtilsTest.java
@@ -0,0 +1,45 @@
+package caosdb.server.utils;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+
+import caosdb.server.CaosDBServer;
+import java.io.IOException;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.restlet.data.Reference;
+
+public class WebinterfaceUtilsTest {
+
+ @Rule public ExpectedException exceptionRule = ExpectedException.none();
+
+ @BeforeClass
+ public static void setup() throws IOException {
+ CaosDBServer.initServerProperties();
+ }
+
+ @Test
+ public void testGetWebinterfaceReference() {
+ WebinterfaceUtils utils = new WebinterfaceUtils(new Reference("https://host:2345/some_path"));
+ String buildNumber = utils.getBuildNumber();
+ String ref = utils.getWebinterfaceURI("sub");
+ assertEquals("https://host:2345/webinterface/" + buildNumber + "/sub", ref);
+ }
+
+ @Test
+ public void testGetPublicFile() {
+ WebinterfaceUtils utils = new WebinterfaceUtils(new Reference("https://host:2345/some_path"));
+ assertNull(utils.getPublicFile("../"));
+ }
+
+ @Test
+ public void testGetPublicFilePath() {
+ WebinterfaceUtils utils = new WebinterfaceUtils(new Reference("https://host:2345/some_path"));
+ assertNull(utils.getPublicFilePath("../"));
+ assertNotNull(utils.getPublicFilePath("./"));
+ assertNotNull(utils.getPublicFilePath("bla"));
+ }
+}