diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index aff33252ae055e573eeeabf34e0f5bf19b197206..3e81c4d4378585c937ece6a10dacf83e9b7f97fc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -31,20 +31,38 @@ stages: - test - deploy -# run unit tests of the server +# Setup: Build a docker image in which tests for this repository can run +build-testenv: + tags: [ cached-dind ] + image: docker:19.03 + stage: setup + only: + - schedules + script: + - cd src/test/docker + - docker login -u indiscale -p $CI_REGISTRY_PASSWORD $CI_REGISTRY + # use here general latest or specific branch latest... + - docker pull $CI_REGISTRY_IMAGE || true + - docker build + --pull + --cache-from $CI_REGISTRY_IMAGE + -t $CI_REGISTRY_IMAGE . + - docker push $CI_REGISTRY_IMAGE + +# Test: run unit tests of the server test: tags: [ docker ] stage: test script: + - mvn -v - make test_misc - make easy-units - - mvn dependency:purge-local-repository - mvn antlr4:antlr4 - mvn compile - echo "defaultRealm = CaosDB" > conf/ext/usersources.ini - mvn test -# Trigger building of server image and integration tests +# Deploy: Trigger building of server image and integration tests trigger_build: tags: [ docker ] stage: deploy @@ -55,21 +73,3 @@ trigger_build: -F "variables[TriggerdBy]=SERVER" -F "variables[TriggerdByHash]=$CI_COMMIT_SHORT_SHA" -F ref=dev https://gitlab.indiscale.com/api/v4/projects/14/trigger/pipeline - -# Build a docker image in which tests for this repository can run -build-testenv: - tags: [ cached-dind ] - image: docker:19.03 - stage: setup - only: - - schedules - script: - - cd src/test/docker - - docker login -u indiscale -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - # use here general latest or specific branch latest... - - docker pull $CI_REGISTRY_IMAGE || true - - docker build - --pull - --cache-from $CI_REGISTRY_IMAGE - -t $CI_REGISTRY_IMAGE . - - docker push $CI_REGISTRY_IMAGE diff --git a/conf/core/server.conf b/conf/core/server.conf index c3cf62bafe6951f1b1412e648ea2eb10f469a2a4..462a16905d23da4e5fce8a411b7ecc2823ad8a03 100644 --- a/conf/core/server.conf +++ b/conf/core/server.conf @@ -1,66 +1,172 @@ +# Set the timezone of the server +# e.g. TIMEZONE=Europe/Berlin or TIMEZONE=UTC. +# Leaving this empty means that the server assumes the timezone of the host. TIMEZONE= +# Set the name of the server owner +# e.g: SERVER_OWNER=XY Department SERVER_OWNER= +# Name of this CaosDB Server SERVER_NAME=CaosDB Server + +# -------------------------------------------------- +# The following paths are relative to the working directory of the server. +# -------------------------------------------------- + +# The location of the server side scripting binaries. +# Put your executable python scripts here, if they need to be called from the scripting API. SERVER_SIDE_SCRIPTING_BIN_DIR=./scripting/bin/ + +# Working directory of the server side scripting API. +# On execution of binaries and scripts the server will create a corresponding working directory in this folder. SERVER_SIDE_SCRIPTING_WORKING_DIR=./scripting/working/ + +# Home directories of the server side scripting API. +# Specific config files, pip packages or other prerequisites for running a script or binary +# can go into a specific home directory for the respective script within this folder. SERVER_SIDE_SCRIPTING_HOME_DIR=./scripting/home/ + +# The CaosDB file system root. +# The file hierarchy of CaosDB's internal file system starts at this folder. +# An absolute file path of File objects within CaosDB is relative to this folder. FILE_SYSTEM_ROOT=./CaosDBFileSystem/FileSystemRoot/ + +# Path to the drop off box. +# This is were users can place files that should be picked up by the CaosDB drop off box program. DROP_OFF_BOX=./CaosDBFileSystem/DropOffBox/ + +# Location of temporary files +# All temporary files with the exception of files created by the scripting API will go into this folder. TMP_FILES=./CaosDBFileSystem/TMP/ + +# Shared folder +# Additional folder for longer term storage of scripting API output. +# In contrast to the script's working directory, these subdirectories are publicly accessible. SHARED_FOLDER=./CaosDBFileSystem/Shared/ + +# Path to the chown script which is needed by the drop off box in order to change permissions of files. CHOWN_SCRIPT=./misc/chown_script/caosdb_chown_dropoffbox + +# This file is responsible for setting individual user and group permissions. USER_SOURCES_INI_FILE=./conf/ext/usersources.ini +# The default state of users which are added to the internal user source. NEW_USER_DEFAULT_ACTIVITY=INACTIVE +# If set to true, unauthenticated access to the database is possible with an anonymous user. AUTH_OPTIONAL=FALSE +# -------------------------------------------------- +# MySQL settings +# -------------------------------------------------- +# Hostname of the mysql instance used by CaosDB MYSQL_HOST=localhost +# Port of the mysql instance MYSQL_PORT=3306 +# Database name of the mysql database MYSQL_DATABASE_NAME=caosdb +# User name for connecting to mysql MYSQL_USER_NAME=caosdb +# Password for the user MYSQL_USER_PASSWORD=caosdb - +# Schema of mysql procedures and tables which is required by this CaosDB instance MYSQL_SCHEMA_VERSION=v2.1.1 -CONTEXT_ROOT= +# -------------------------------------------------- +# Server options +# -------------------------------------------------- +# The context root is a prefix which allows running multiple instances of CaosDB using the same +# hostname and port. +CONTEXT_ROOT= +# HTTPS port of this server instance. SERVER_PORT_HTTPS=443 +# HTTP port of this server instance. SERVER_PORT_HTTP=80 +# Initial number of HTTPConnection objects in the pool. +INITIAL_CONNECTIONS=1 +# Maximum number of parallel HTTPConnections of the server +MAX_CONNECTIONS=10 + + +# -------------------------------------------------- +# HTTPS options +# -------------------------------------------------- +# Allowed TLS versions HTTPS_ENABLED_PROTOCOLS=TLSv1.3 TLSv1.2 +# Forbidden TLS versions HTTPS_DISABLED_PROTOCOLS=SSLv3 SSLv2Hello TLSv1.1 TLSv1.0 +# Allowed cipher suites which are used for the encryption of the HTTP payload. HTTPS_ENABLED_CIPHER_SUITES=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 TLS_AES_128_GCM_SHA256 TLS_AES_256_GCM_SHA384 TLS_CHACHA20_POLY1305_SHA256 TLS_AES_128_CCM_SHA256 TLS_AES_128_CCM_8_SHA256 TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 TLS_DHE_RSA_WITH_AES_256_GCM_SHA384 TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 +# Forbidden cipher suites which are used for the encryption of the HTTP payload. HTTPS_DISABLED_CIPHER_SUITES=TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA TLS_RSA_WITH_AES_256_CBC_SHA TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA TLS_ECDH_RSA_WITH_AES_256_CBC_SHA TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA TLS_RSA_WITH_AES_128_CBC_SHA TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA TLS_ECDH_RSA_WITH_AES_128_CBC_SHA TLS_ECDHE_ECDSA_WITH_RC4_128_SHA TLS_ECDHE_RSA_WITH_RC4_128_SHA TLS_ECDH_ECDSA_WITH_RC4_128_SHA TLS_ECDH_RSA_WITH_RC4_128_SHA TLS_ECDHE_ECDSA_WITH_3DES_EDE_CBC_SHA TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA TLS_ECDH_ECDSA_WITH_3DES_EDE_CBC_SHA TLS_ECDH_RSA_WITH_3DES_EDE_CBC_SHASSL_RSA_WITH_RC4_128_MD5 SSL_RSA_WITH_3DES_EDE_CBC_SHA SSL_RSA_WITH_RC4_128_SHA TLS_DHE_RSA_WITH_AES_256_CBC_SHA TLS_DHE_DSS_WITH_AES_256_CBC_SHA TLS_DHE_RSA_WITH_AES_128_CBC_SHA TLS_DHE_DSS_WITH_AES_128_CBC_SHA SSL_DHE_RSA_WITH_3DES_EDE_CBC_SHA SSL_DHE_DSS_WITH_3DES_EDE_CBC_SHA -INITIAL_CONNECTIONS=1 +# Password for the private key for the HTTPS server. +CERTIFICATES_KEY_PASSWORD= +# Path to the keystore which typically ends in jks. +CERTIFICATES_KEY_STORE_PATH= +# Password of the keystore which stores the private key. +CERTIFICATES_KEY_STORE_PASSWORD= -MAX_CONNECTIONS=10 +# -------------------------------------------------- +# Timeout settings +# -------------------------------------------------- +# The session timeout after which the cookie expires. # 10 min SESSION_TIMEOUT_MS=600000 + +# Time after which activation tokens for the activation of new users (internal +# user sources) expire. # 7days ACTIVATION_TIMEOUT_MS=604800000 +# The value for the HTTP cache directive "max-age" +WEBUI_HTTP_HEADER_CACHE_MAX_AGE=28800 + +# -------------------------------------------------- +# Mail settings +# -------------------------------------------------- +# The handler that treats sent mails. +# The default handler pipes mails to a file. MAIL_HANDLER_CLASS=caosdb.server.utils.mail.ToFileHandler +# The file were the ToFileHanlder pipes messages to. MAIL_TO_FILE_HANDLER_LOC=./ +# -------------------------------------------------- +# Admin settings +# # -------------------------------------------------- +# Name of the administrator of this instance ADMIN_NAME=CaosDB Admin +# Email of the administrator of this instance ADMIN_EMAIL= +# An URL to the bugtracker for managing instance related bugs. BUGTRACKER_URI= +# If set to true MySQL stores transaction benchmarks for all SQL queries. Used for benchmarking and debugging. TRANSACTION_BENCHMARK_ENABLED=true +# Location of the configuration file for the CaosDB cache. CACHE_CONF_LOC=./conf/core/cache.ccf +# Set this option to true to lobally disable caching. Used for debugging. +CACHE_DISABLE=false +# The server is allowed to create symlinks to files and folders within this whitelist of directories. INSERT_FILES_IN_DIR_ALLOWED_DIRS= +# Sudo password of the system. +# Needed by the drop off box to set file permissions. SUDO_PASSWORD= + +# If set to false ACL checks are circumvented during querying. This may leak information but is a lot faster. QUERY_FILTER_ENTITIES_WITHOUT_RETRIEVE_PERMISSIONS=TRUE +# When checking the ACL of an entity roles which are unknown to the server +# raise an error (when set to MUST) or a warning (when set to SHOULD). +# Unknown roles occur when a user or group is removed or when entities are +# loaded from other instances of the CaosDB Server where different users are +# present. +# CHECK_ENTITY_ACL_ROLES_MODE=[MUST,SHOULD] CHECK_ENTITY_ACL_ROLES_MODE=MUST +# Location of the global ACL file for entities. The global ACL is implicitly +# part of any Entity ACL. GLOBAL_ENTITY_PERMISSIONS_FILE=./conf/core/global_entity_permissions.xml -CERTIFICATES_KEY_PASSWORD= -CERTIFICATES_KEY_STORE_PATH= -CERTIFICATES_KEY_STORE_PASSWORD= - -WEBUI_HTTP_HEADER_CACHE_MAX_AGE=28800 diff --git a/doc/devel/Benchmarking.md b/doc/devel/Benchmarking.md index a244a3a64b771060fbf025fc0ee47054b9b95b48..3e48dc4f08e023be625bfaeaa2d645e73b2a9345 100644 --- a/doc/devel/Benchmarking.md +++ b/doc/devel/Benchmarking.md @@ -1,4 +1,4 @@ -# Manual Java-Side Benchmarking # +# Manual Java-side benchmarking # Benchmarking can be done using the `TransactionBenchmark` class (in package `caosdb.server.database.misc`). @@ -11,3 +11,60 @@ Benchmarking can be done using the `TransactionBenchmark` class (in package - To work with the benchmarks of often used objects, use these methods: - `Container.getTransactionBenchmark().addBenchmark()` - `Query.addBenchmark()` + +# Miscellaneous notes # + +Notes to self, details, etc. + +## On method calling order and benchmarked events ## + +- `Transaction.execute()` :: Logs benchmarks for events like: + - `INIT` :: The transaction's `init()` method. + - `PRE_CHECK` + - `CHECK` + - `POST_CHECK` + - `PRE_TRANSACTION` + - `TRANSACTION` -> typically calls + `database.backend.transaction.[BackendTransaction].execute()`, which in turn + calls, some levels deeper, `backend.transaction.....execute(<k extends + BackendTransaction> t)` -> see next point + - ... +- `backend.transaction.[...].execute(transaction)` :: This method is benchmarked + again (via parent class `BackendTransaction`), this is probably the deepest + level of benchmarking currently (Benchmark is logged as + e.g. `<RetrieveFullEntity>...</>`). It finally calls + `[MySQLTransaction].execute()`. +- `[MySQLTransaction].execute()` :: This is the deepest backend implementation + part, it typically creates a prepared statement and executes it. +- Currently not benchmarked separately: + - Getting the actual implementation (probably fast?) + - Preparing the SQL statement + - Executing the SQL statement + - Java-side caching + +## Server settings ## + +- To enable the SQL general logs, log into the SQL server and do: + ```sql +SET GLOBAL log_output = 'TABLE'; +SET GLOBAL general_log = 'ON'; +``` +- To enable transaction benchmarks and disable caching in the server, set these + server settings: +```conf +TRANSACTION_BENCHMARK_ENABLED=true +CACHE_DISABLE=true +``` +- Additionally, the server should be started via `make run-debug` (instead of + `make run-single`), otherwise the benchmarking will not be active. + +## Notable benchmarks and where to find them ## + +| Name | Where measured | What measured | +|--------------------------------------|----------------------------------------------|-------------------------------| +| `Retrieve.init` | transaction/Transaction.java#135 | transaction/Retrieve.java#48 | +| `Retrieve.transaction` | transaction/Transaction.java#174 | transaction/Retrieve.java#133 | +| `Retrieve.post_transaction` | transaction/Transaction.java#182 | transaction/Retrieve.java#77 | +| `EntityResource.httpGetInChildClass` | resource/transaction/EntityResource.java#118 | all except XML generation | +| `ExecuteQuery` | ? | ? | +| | | | diff --git a/pom.xml b/pom.xml index ada3403287a657f7684ba8984c57cc80a51820a9..be9279b6e1e927339d1c044b4cbc4313f992fba5 100644 --- a/pom.xml +++ b/pom.xml @@ -227,10 +227,10 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> - <version>3.6.1</version> + <version>3.8.1</version> <configuration> - <source>1.8</source> - <target>1.8</target> + <source>11</source> + <target>11</target> </configuration> </plugin> <plugin> diff --git a/src/main/java/caosdb/server/CaosDBServer.java b/src/main/java/caosdb/server/CaosDBServer.java index 9da7f9afa1642be553629d8115f693862011ce46..34bfe3e2911955fa724ea5e6ac7258d80f9cc120 100644 --- a/src/main/java/caosdb/server/CaosDBServer.java +++ b/src/main/java/caosdb/server/CaosDBServer.java @@ -425,7 +425,7 @@ public class CaosDBServer extends Application { * @param port_https Listen on this port for https connections. * @param port_http Listen on this port for http connections and send http-to-https redirect with * different port. - * @parem port_redirect_https Redirect any http connections to this port. + * @param port_redirect_https Redirect any http connections to this port. * @throws Exception if problems occur starting up this server. */ private static void runHTTPSServer( diff --git a/src/main/java/caosdb/server/ServerProperties.java b/src/main/java/caosdb/server/ServerProperties.java index e68016894d65a3b77e38aa2f84f48ece7db05683..292d99e7ff6eedf39d0aee71b5f2df0f879b6a64 100644 --- a/src/main/java/caosdb/server/ServerProperties.java +++ b/src/main/java/caosdb/server/ServerProperties.java @@ -91,6 +91,7 @@ public class ServerProperties extends Properties { public static final String KEY_ACTIVATION_TIMEOUT_MS = "ACTIVATION_TIMEOUT_MS"; public static final String KEY_CACHE_CONF_LOC = "CACHE_CONF_LOC"; + public static final String KEY_CACHE_DISABLE = "CACHE_DISABLE"; public static final String KEY_TRANSACTION_BENCHMARK_ENABLED = "TRANSACTION_BENCHMARK_ENABLED"; diff --git a/src/main/java/caosdb/server/caching/JCSCacheHelper.java b/src/main/java/caosdb/server/caching/JCSCacheHelper.java index 55efaa191bcb777bd63881dfbd72cd292808064c..1ce455a949bcfb76a36815b3b14bcea89e676a4a 100644 --- a/src/main/java/caosdb/server/caching/JCSCacheHelper.java +++ b/src/main/java/caosdb/server/caching/JCSCacheHelper.java @@ -62,25 +62,32 @@ public class JCSCacheHelper implements CacheHelper { } public static void init() { - init(CaosDBServer.getServerProperty(ServerProperties.KEY_CACHE_CONF_LOC)); + final boolean disabled = + Boolean.parseBoolean(CaosDBServer.getServerProperty(ServerProperties.KEY_CACHE_DISABLE)); + init(CaosDBServer.getServerProperty(ServerProperties.KEY_CACHE_CONF_LOC), disabled); } - public static void init(String configFileLocation) { + public static void init(String configFileLocation, boolean disabled) { Properties config = null; - try { - Properties p = new Properties(); - final InputStream is = new FileInputStream(configFileLocation); - p.load(is); - is.close(); - config = p; - } catch (final FileNotFoundException e) { - logger.error(e); - config = getNOPCachingProperties(); - } catch (final IOException e) { - logger.error(e); + if (disabled) { config = getNOPCachingProperties(); + logger.info("Configuring JCS Caching: disabled"); + } else { + try { + Properties p = new Properties(); + final InputStream is = new FileInputStream(configFileLocation); + p.load(is); + is.close(); + config = p; + } catch (final FileNotFoundException e) { + logger.error(e); + config = getNOPCachingProperties(); + } catch (final IOException e) { + logger.error(e); + config = getNOPCachingProperties(); + } + logger.info("Configuring JCS Caching with {}", config); } - logger.info("Configuring JCS Caching with {}", config); JCS.setConfigProperties(config); } diff --git a/src/main/java/caosdb/server/database/BackendTransaction.java b/src/main/java/caosdb/server/database/BackendTransaction.java index e4b6c617e6b8991e9c5ed4efb79f6bcc53abbbbd..14e65e86bd9fe05b074c731e27b601fa4ed51f68 100644 --- a/src/main/java/caosdb/server/database/BackendTransaction.java +++ b/src/main/java/caosdb/server/database/BackendTransaction.java @@ -207,6 +207,15 @@ public abstract class BackendTransaction implements Undoable { } } + /** + * Execute this BackendTransaction, using the implementation given as an argument. + * + * <p>The implementation's benchmark is set to the corresponding sub-benchmark of this object's + * benchmark. + * + * @param t This BackendTransaction's execute() method will be called. + * @return The BackendTransaction which was passed as an argument. + */ protected <K extends BackendTransaction> K execute(final K t) { assert t != this; this.undoHandler.append(t); @@ -271,6 +280,7 @@ public abstract class BackendTransaction implements Undoable { return this.getClass().getSimpleName(); } + /** Set the benchmark object for this AbstractTransaction. */ public void setTransactionBenchmark(TransactionBenchmark b) { this.benchmark = b; } diff --git a/src/main/java/caosdb/server/database/backend/implementation/MySQL/MySQLRetrieveSparseEntity.java b/src/main/java/caosdb/server/database/backend/implementation/MySQL/MySQLRetrieveSparseEntity.java index 907c7471d3a804d0c4e56380ae1bd649a7de04a7..749b0b91455c9f429821bbabce3aa99375a417cd 100644 --- a/src/main/java/caosdb/server/database/backend/implementation/MySQL/MySQLRetrieveSparseEntity.java +++ b/src/main/java/caosdb/server/database/backend/implementation/MySQL/MySQLRetrieveSparseEntity.java @@ -48,10 +48,10 @@ public class MySQLRetrieveSparseEntity extends MySQLTransaction @Override public SparseEntity execute(final int id) throws TransactionException { try { - final PreparedStatement prepareStatement = prepareStatement(stmtStr); + final PreparedStatement preparedStatement = prepareStatement(stmtStr); - prepareStatement.setInt(1, id); - final ResultSet rs = prepareStatement.executeQuery(); + preparedStatement.setInt(1, id); + final ResultSet rs = preparedStatement.executeQuery(); try { if (rs.next()) { return DatabaseUtils.parseEntityResultSet(rs); diff --git a/src/main/java/caosdb/server/database/misc/TransactionBenchmark.java b/src/main/java/caosdb/server/database/misc/TransactionBenchmark.java index f7c9e80f811418392b6567fcda5bfb8a9948bbff..91bf0acbcd8d8c2affdf0ba48aa7bdb756f34d81 100644 --- a/src/main/java/caosdb/server/database/misc/TransactionBenchmark.java +++ b/src/main/java/caosdb/server/database/misc/TransactionBenchmark.java @@ -28,8 +28,11 @@ import caosdb.server.utils.CronJob; import caosdb.server.utils.Info; import caosdb.server.utils.ServerStat; import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collection; import java.util.Date; import java.util.HashMap; +import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Map.Entry; @@ -71,6 +74,11 @@ class Timer implements Serializable { } } +/** + * The measurement of e.g. a certain action, possibly over multiple runs. A Measurement keeps track + * of how often the action was measured, and what the total and averaged spent time is. One + * Measurement object knows nothing about other Measurement objects. + */ class Measurement implements Serializable { private static final long serialVersionUID = -2429348657382168470L; private final Timer timer; @@ -227,6 +235,8 @@ class JdomConverter { public Element convert(TransactionBenchmark b) { Element ret = new Element(b.getName()); ret.setAttribute("since", new Date(b.since).toString()); + ret.setAttribute("since_ms", Long.toString(b.since)); + ret.setAttribute("called_from", b.called_from()); Iterable<Measurement> measurements = b.getMeasurements(); synchronized (measurements) { @@ -246,6 +256,15 @@ class JdomConverter { } } +/** + * The TransactionBenchmark object allows to store the time which some actions take. At its core, it + * consists of: + * <li>A list of named Measurements. + * <li>A list of named SubBenchmarks. + * + * <p>Each measurement may be recorded on multiple occasions, the stored Measurement objects + * keep a statistic about the total and average recorded times. + */ public abstract class TransactionBenchmark implements Serializable { private static final long serialVersionUID = -8916163825450491067L; @@ -255,13 +274,16 @@ public abstract class TransactionBenchmark implements Serializable { Boolean.valueOf( CaosDBServer.getServerProperty(ServerProperties.KEY_TRANSACTION_BENCHMARK_ENABLED) .toLowerCase()); + protected StackTraceElement[] stackTraceElements; long since = System.currentTimeMillis(); protected final Map<String, Measurement> measurements = new HashMap<>(); protected final Map<String, SubBenchmark> subBenchmarks = new HashMap<>(); - protected TransactionBenchmark() {}; + protected TransactionBenchmark() { + stackTraceElements = Thread.currentThread().getStackTrace(); + }; public Iterable<SubBenchmark> getSubBenchmarks() { return this.subBenchmarks.values(); @@ -335,9 +357,10 @@ public abstract class TransactionBenchmark implements Serializable { } } - public Element toElememt() { + public Element toElement() { if (isActive) { - return new JdomConverter().convert(this).setName("TransactionBenchmark"); + final Element el = new JdomConverter().convert(this); + return el.setName(el.getName() + ".TransactionBenchmark"); } else { final Element ret = new Element("TransactionBenchmark"); ret.setAttribute("info", "TransactionBenchmark is disabled."); @@ -348,7 +371,8 @@ public abstract class TransactionBenchmark implements Serializable { public abstract String getName(); /** - * Create a new independent {@link TransactionBenchmark}. + * Create a new independent {@link TransactionBenchmark}. Existing benchmarks with the same name + * are overwritten by this method. * * <p>This is for starting a series of measurements from a clean slate. * @@ -366,6 +390,7 @@ public abstract class TransactionBenchmark implements Serializable { } } + /** Return (and create if necessary) the sub-benchmark with the given name. */ public TransactionBenchmark getBenchmark(String name) { synchronized (subBenchmarks) { SubBenchmark existing = subBenchmarks.get(name); @@ -379,11 +404,62 @@ public abstract class TransactionBenchmark implements Serializable { } } + /** Return (and create if necessary) the sub-benchmark with the name of the given class. */ public TransactionBenchmark getBenchmark(Class<?> class1) { return this.getBenchmark(class1.getSimpleName()); } + /** + * Create a sub-benchmark with the name of the given class, overwriting existing ones with the + * same name. + */ public TransactionBenchmark createBenchmark(Class<?> class1) { return this.createBenchmark(class1.getSimpleName()); } + + /** Return a String denoting where this TransactionBenchmark was created from. */ + public String called_from() { + final int stackSize = stackTraceElements.length; + if (stackSize <= 5) { + return null; + } + final String[] classComponents = stackTraceElements[0].getClassName().split("\\."); + if (classComponents.length == 0) { + return "unsplittable: " + stackTraceElements[0].getClassName(); + } + final Collection<String> blacklist = List.of("Thread", "Benchmark"); + for (int i = 0; i < stackSize; ++i) { + StackTraceElement ste = stackTraceElements[i]; + final String className = ste.getClassName(); + if (blacklist.stream().anyMatch((blacklistItem) -> className.contains(blacklistItem))) { + continue; + } + + Collection<String> elementStrings = new ArrayList<String>(); + List.of(i, i + 1, i + 2) + .forEach( + (j) -> { + if (j < stackTraceElements.length) { + elementStrings.add(prettifyStackTraceElement(stackTraceElements[j])); + } + }); + return String.join(" / ", elementStrings); + } + return null; + } + + /** + * Return a nice String for the StackTraceElement. + * + * <p>The String is composed like this: CLASS::METHOD [LINE_NUMBER], with "caosdb.server." removed + * from the class name. + */ + private static String prettifyStackTraceElement(final StackTraceElement ste) { + return ste.getClassName().replaceFirst("caosdb\\.server\\.", "") + + "::" + + ste.getMethodName().replaceAll("[<>]", "") + + " [" + + ste.getLineNumber() + + "]"; + } } diff --git a/src/main/java/caosdb/server/entity/container/TransactionContainer.java b/src/main/java/caosdb/server/entity/container/TransactionContainer.java index 79b6f844f78dfe1247b34d0cd5900bd39df70b32..f0ac55fae7b57d24456728508baa7768b83a5711 100644 --- a/src/main/java/caosdb/server/entity/container/TransactionContainer.java +++ b/src/main/java/caosdb/server/entity/container/TransactionContainer.java @@ -104,7 +104,7 @@ public class TransactionContainer extends Container<EntityInterface> public void addToElement(final Element element) { element.setAttribute("count", Integer.toString(size())); if (this.benchmark != null && CaosDBServer.isDebugMode()) { - element.addContent(this.benchmark.toElememt()); + element.addContent(this.benchmark.toElement()); } for (final ToElementable m : this.messages) { m.addToElement(element); diff --git a/src/main/java/caosdb/server/query/POV.java b/src/main/java/caosdb/server/query/POV.java index 6a57caa780d76c5279a37eaba09d44c021215d7c..a50be918da30d6cb69b457ef4b7801ea3019d58d 100644 --- a/src/main/java/caosdb/server/query/POV.java +++ b/src/main/java/caosdb/server/query/POV.java @@ -43,6 +43,7 @@ import java.sql.SQLException; import java.sql.Types; import java.util.HashMap; import java.util.Map.Entry; +import java.util.Stack; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.jdom2.Element; @@ -70,6 +71,7 @@ public class POV implements EntityFilterInterface { private String refIdsTable = null; private final HashMap<String, String> statistics = new HashMap<>(); private Logger logger = LoggerFactory.getLogger(getClass()); + private Stack<String> prefix = new Stack<>(); private Unit getUnit(final String s) throws ParserException { return CaosDBSystemOfUnits.getUnit(s); @@ -88,6 +90,7 @@ public class POV implements EntityFilterInterface { final String operator, final String value, final String aggregate) { + prefix.add("POV"); if (property != null && property.type != Query.Pattern.TYPE_NORMAL) { throw new UnsupportedOperationException( "Regular Expression and Like Patterns are not implemented for properties yet."); @@ -213,7 +216,9 @@ public class POV implements EntityFilterInterface { this.connection = query.getConnection(); this.targetSet = query.getTargetSet(); + prefix.add("#initPOV"); initPOV(query); + prefix.pop(); // applyPOV(sourceSet, targetSet, propertiesTable, refIdsTable, o, // vText, vInt, @@ -315,15 +320,16 @@ public class POV implements EntityFilterInterface { } else { callPOV.setNull(15, VARCHAR); } - + prefix.add("#executeStmt"); executeStmt(callPOV, query); + prefix.pop(); callPOV.close(); } catch (final SQLException e) { logger.error("This POV filter caused an error: " + this.toString()); throw new QueryException(e); } - query.addBenchmark(this.getClass().getSimpleName(), System.currentTimeMillis() - t1); + query.addBenchmark(measurement(""), System.currentTimeMillis() - t1); } private void initPOV(final QueryInterface query) throws SQLException { @@ -350,7 +356,7 @@ public class POV implements EntityFilterInterface { } } final long t2 = System.currentTimeMillis(); - query.addBenchmark(getClass().getSimpleName() + ".initPOVRefidsTable()", t2 - t1); + query.addBenchmark(measurement(".initPOVRefidsTable()"), t2 - t1); try (PreparedStatement stmt = query.getConnection().prepareCall("call initPOVPropertiesTable(?,?,?)")) { // initPOVPropertiesTable(in pid INT UNSIGNED, in pname @@ -385,36 +391,28 @@ public class POV implements EntityFilterInterface { final long st5 = rs.getLong("t5"); final long st6 = rs.getLong("t6"); if (st2 - st1 > 0) { - query.addBenchmark( - getClass().getSimpleName() + ".initPOVPropertiesTable()#initPropertiesTableByName", - st2 - st1); + query.addBenchmark(measurement("#initPropertiesTableByName"), st2 - st1); } if (st3 - st2 > 0) { - query.addBenchmark( - getClass().getSimpleName() + ".initPOVPropertiesTable()#initPropertiesTableById", - st3 - st2); + query.addBenchmark(measurement("#initPropertiesTableById"), st3 - st2); } if (st4 - st3 > 0) { - query.addBenchmark( - getClass().getSimpleName() + ".initPOVPropertiesTable()#getChildren", st4 - st3); + query.addBenchmark(measurement("#getChildren"), st4 - st3); } if (st5 - st4 > 0) { - query.addBenchmark( - getClass().getSimpleName() + ".initPOVPropertiesTable()#findReplacements", st5 - st4); + query.addBenchmark(measurement("#findReplacements"), st5 - st4); } if (st6 - st5 > 0) { - query.addBenchmark( - getClass().getSimpleName() + ".initPOVPropertiesTable()#addReplacements", st6 - st5); + query.addBenchmark(measurement("#addReplacements"), st6 - st5); } } } final long t3 = System.currentTimeMillis(); - query.addBenchmark(getClass().getSimpleName() + ".initPOVPropertiesTable()", t3 - t2); + query.addBenchmark(measurement(""), t3 - t2); if (this.refIdsTable != null) { query.getQuery().applyQueryTemplates(query, this.refIdsTable); - query.addBenchmark( - getClass().getSimpleName() + ".applyQueryTemplates()", System.currentTimeMillis() - t3); + query.addBenchmark(measurement(".applyQueryTemplates()"), System.currentTimeMillis() - t3); } if (hasSubProperty() && this.targetSet != null) { @@ -434,8 +432,7 @@ public class POV implements EntityFilterInterface { try { final long t1 = System.currentTimeMillis(); final ResultSet rs = callPOV.executeQuery(); - query.addBenchmark( - getClass().getSimpleName() + ".executeStmt()", System.currentTimeMillis() - t1); + query.addBenchmark(measurement(".callPOV"), System.currentTimeMillis() - t1); if (rs.next()) { final int c = rs.getMetaData().getColumnCount(); for (int i = 0; i < c; i++) { @@ -452,8 +449,7 @@ public class POV implements EntityFilterInterface { if (hasSubProperty()) { final long t2 = System.currentTimeMillis(); getSubProperty().apply(query, this.targetSet, this.propertiesTable, this.refIdsTable); - query.addBenchmark( - getClass().getSimpleName() + ".applySubProperty()", System.currentTimeMillis() - t2); + query.addBenchmark(measurement(".applySubProperty()"), System.currentTimeMillis() - t2); } } catch (final SQLException e) { if (e.getMessage().trim().startsWith("Can't reopen table:") && retry > this.retry_count++) { @@ -498,4 +494,8 @@ public class POV implements EntityFilterInterface { public String getAggregate() { return this.aggregate; } + + private String measurement(String m) { + return String.join("", prefix) + m; + } } diff --git a/src/main/java/caosdb/server/resource/FileSystemResource.java b/src/main/java/caosdb/server/resource/FileSystemResource.java index 37f0e9ffda3f81b84c6001c5521ba5c8451b473e..ecaa97fe93cb59fe6a4c58fb60d99a5dca373080 100644 --- a/src/main/java/caosdb/server/resource/FileSystemResource.java +++ b/src/main/java/caosdb/server/resource/FileSystemResource.java @@ -126,7 +126,7 @@ public class FileSystemResource extends AbstractCaosDBServerResource { final long t2 = System.currentTimeMillis(); getBenchmark().addMeasurement(this.getClass().getSimpleName(), t2 - t1); - rootElem.addContent(getBenchmark().toElememt()); + rootElem.addContent(getBenchmark().toElement()); final Document doc = new Document(rootElem); return ok(doc); diff --git a/src/main/java/caosdb/server/resource/InfoResource.java b/src/main/java/caosdb/server/resource/InfoResource.java index 40c071f4ae90bde9600102613a4e94ab35a119cf..4beb9757cff83696a546eb174fbb0f21453029b7 100644 --- a/src/main/java/caosdb/server/resource/InfoResource.java +++ b/src/main/java/caosdb/server/resource/InfoResource.java @@ -48,7 +48,7 @@ public class InfoResource extends AbstractCaosDBServerResource { root.addContent(getTimeZone()); root.addContent(Info.toElement()); root.addContent(FlagInfo.toElement()); - root.addContent(TransactionBenchmark.getRootInstance().toElememt()); + root.addContent(TransactionBenchmark.getRootInstance().toElement()); doc.setRootElement(root); return ok(doc); } diff --git a/src/main/java/caosdb/server/resource/transaction/EntityResource.java b/src/main/java/caosdb/server/resource/transaction/EntityResource.java index 0c853096c1af71dcacf2ef775c46f4dceecb1673..8dec9c00986fd3b27c050d39caf5705eb3dceaba 100644 --- a/src/main/java/caosdb/server/resource/transaction/EntityResource.java +++ b/src/main/java/caosdb/server/resource/transaction/EntityResource.java @@ -109,16 +109,30 @@ public class EntityResource extends AbstractCaosDBServerResource { getGetRequestHandler().handle(this, entityContainer); + final long t2 = System.currentTimeMillis(); + entityContainer + .getTransactionBenchmark() + .addMeasurement(getClass().getSimpleName() + ".httpGetInChildClass.handle", t2 - t1); + final Retrieve retrieve = new Retrieve(entityContainer); retrieve.execute(); - final long t2 = System.currentTimeMillis(); + final long t3 = System.currentTimeMillis(); entityContainer .getTransactionBenchmark() - .addMeasurement(getClass().getSimpleName() + ".httpGetInChildClass", t2 - t1); + .addMeasurement( + getClass().getSimpleName() + ".httpGetInChildClass.retrieve.execute", t3 - t2); + final Element rootElem = generateRootElement(); entityContainer.addToElement(rootElem); doc.setRootElement(rootElem); + + final long t4 = System.currentTimeMillis(); + entityContainer + .getTransactionBenchmark() + .addMeasurement( + getClass().getSimpleName() + ".httpGetInChildClass.element_handling", t4 - t3); + return ok(doc); } diff --git a/src/test/docker/Dockerfile b/src/test/docker/Dockerfile index d8b831eebbda33a3143f89a8da1edb770788bce5..ecc6b332b2d88e0d587fdb3df0fc13cdfd0c159a 100644 --- a/src/test/docker/Dockerfile +++ b/src/test/docker/Dockerfile @@ -1,4 +1,5 @@ -FROM debian:stretch +FROM debian:buster RUN apt-get update && \ - apt-get install git make mariadb-server maven openjdk-8-jdk-headless \ + apt-get install \ + git make mariadb-server maven openjdk-11-jdk-headless \ python3-pip screen libpam0g-dev unzip curl shunit2 -y diff --git a/src/test/java/caosdb/server/caching/TestNoCaching.java b/src/test/java/caosdb/server/caching/TestNoCaching.java new file mode 100644 index 0000000000000000000000000000000000000000..dfab9de67b8c79871b6d2fd70fc6e25a1c231230 --- /dev/null +++ b/src/test/java/caosdb/server/caching/TestNoCaching.java @@ -0,0 +1,29 @@ +package caosdb.server.caching; + +import static org.junit.Assert.assertEquals; + +import caosdb.server.CaosDBServer; +import caosdb.server.ServerProperties; +import caosdb.server.database.backend.transaction.RetrieveProperties; +import java.io.IOException; +import org.apache.commons.jcs.JCS; +import org.apache.commons.jcs.access.CacheAccess; +import org.junit.BeforeClass; +import org.junit.Test; + +public class TestNoCaching { + + @BeforeClass + public static void init() throws IOException { + CaosDBServer.initServerProperties(); + CaosDBServer.setProperty(ServerProperties.KEY_CACHE_DISABLE, "TRUE"); + JCSCacheHelper.init(); + } + + @Test + public void testCacheConfig() { + CacheAccess<String, String> retrieve_properties_cache = + JCS.getInstance(RetrieveProperties.CACHE_REGION); + assertEquals(0, retrieve_properties_cache.getCacheAttributes().getMaxObjects()); + } +}