Merge branch 'f-map-bundle' into 'dev'

F map bundle

See merge request !116

.gitlab-ci.yml

@@ -30,7 +30,6 @@ image: $CI_REGISTRY_IMAGE:latest
 
 stages:
   - setup
-  - linting
   - test
   - deploy
 
@@ -38,7 +37,7 @@ stages:
 linting:
   timeout: 15 minutes
   tags: [ docker ]
-  stage: linting
+  stage: test
   script:
     - make pylint
   allow_failure: true
@@ -55,7 +54,7 @@ test:
       - screenlog.*
       - screenshot.*
   script:
-    - make run-qunit
+    - make -d run-qunit
 
 test-server-side-scripting:
   timeout: 10 minutes

CHANGELOG.md

@@ -8,6 +8,25 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added ###
 
+* `BUILD_MODULE_LEGACY_MAP={DISABLED,ENABLED}`, which is enabled by default.
+  This options prepares the web for the upcomming new map implementation (see
+  below).
+* The new map from the [CaosDB WebUI Legacy
+  Adapter](https://gitlab.com/caosdb/caosdb-webui-legacy-adapter) has been
+  added. It is disabled by default because it depends not only on the server's
+  GRPC API but also on a proxy translating WebGRPC/HTTP1.1 (browser) requests
+  to GRPC/HTTP2 (server) and vice versa.  
+  The new map should work as a drop-in for the legacy map with the exact
+  functionality of the old map and the same config files. However, the new map
+  also implements a new behavior for the query button: The query will be
+  executed immediately, see
+  [caosdb-webui#205](https://gitlab.com/caosdb/caosdb-webui/-/issues/205).  
+  You can enable the new map by disabling the old one AND placing a valid
+  configuration at `conf/ext/json/ext_map.json`. A suitable proxy is envoy and
+  a suitable envoy.yaml can be found in `misc/envoy.yaml`. Please have a look
+  at that file for more information.
+
+
 ### Changed ###
 
 ### Deprecated ###
@@ -16,6 +35,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed ###
 
+* The new query panel can't be deactivated.
+  [caosdb-webui#226](https://gitlab.com/caosdb/caosdb-webui/-/issues/226)
+
 ### Security ###
 
 ### Documentation ###

Makefile

@@ -39,12 +39,16 @@ CONF_EXT_DIR = $(abspath conf/ext)
 SRC_CORE_DIR = $(abspath src/core)
 SRC_EXT_DIR = $(abspath src/ext)
 SRC_SSS_DIR = $(abspath src/server_side_scripting)
+SRC_LEGACY_DIR = $(abspath src/legacy)
 LIBS_DIR = $(abspath libs)
 TEST_CORE_DIR = $(abspath test/core/)
 TEST_EXT_DIR = $(abspath test/ext)
 TEST_SSS_DIR =$(abspath test/server_side_scripting)
-LIBS = fonts css/fonts css/bootstrap.css css/bootstrap-icons.css js/state-machine.js js/jquery.js js/showdown.js js/dropzone.js css/dropzone.css js/loglevel.js js/leaflet.js css/leaflet.css css/images js/leaflet-latlng-graticule.js js/proj4.js js/proj4leaflet.js js/leaflet-coordinates.js css/leaflet-coordinates.css js/leaflet-graticule.js js/bootstrap-select.js css/bootstrap-select.css js/bootstrap-autocomplete.min.js js/plotly.js js/pako.js js/utif.js js/bootstrap.js js/qrcode.js js/ckeditor.js
+LIBS = fonts css/fonts css/bootstrap.css css/bootstrap-icons.css js/state-machine.js js/jquery.js js/showdown.js js/dropzone.js css/dropzone.css js/loglevel.js js/bootstrap-select.js css/bootstrap-select.css js/bootstrap-autocomplete.min.js js/plotly.js js/pako.js js/utif.js js/bootstrap.js js/qrcode.js js/ckeditor.js
 
+LEGACY_MAP_JS = leaflet.js leaflet-latlng-graticule.js proj4.js proj4leaflet.js leaflet-coordinates.js leaflet-graticule.js ext_map.js
+LEGACY_MAP_LIBS = $(addprefix js/, $(LEGACY_MAP_JS)) css/leaflet.css css/leaflet-coordinates.css css/images
+LEGACY_QUERY_FORM_LIBS = js/query_form.js
 
 TEST_LIBS = $(LIBS) js/qunit.js css/qunit.css $(subst $(TEST_CORE_DIR)/,,$(shell find $(TEST_CORE_DIR)/))
 
@@ -53,9 +57,14 @@ LIBS_SUBDIRS = $(addprefix $(LIBS_DIR)/, js css fonts)
 
 ALL: install
 
-install: clean-ignore-zips install-sss cp-src cp-ext cp-conf $(addprefix $(PUBLIC_DIR)/, $(LIBS)) merge_js build_properties merge_xsl
+install: clean-ignore-zips install-sss cp-src cp-legacy cp-ext cp-conf $(addprefix $(PUBLIC_DIR)/, $(LIBS)) merge_js build_properties merge_xsl
 
-test: clean-ignore-zips install-sss cp-src cp-ext cp-ext-test cp-conf $(addprefix $(PUBLIC_DIR)/, $(TEST_LIBS)) merge_js build_properties merge_xsl
+.PHONY: test
+test:
+	$(MAKE) EXTRA_BUILD_FILE="$(TEST_CORE_DIR)/test.properties" build_test
+
+.PHONY: build_test
+build_test: clean-ignore-zips install-sss cp-src cp-legacy cp-ext cp-ext-test cp-conf $(addprefix $(PUBLIC_DIR)/, $(TEST_LIBS)) merge_js build_properties merge_xsl
 	@for f in $(shell find $(TEST_EXT_DIR) -type f -iname *.js) ; do \
 		sed -i "/EXTENSIONS/a \<script src=\"$${f#$(TEST_EXT_DIR)/}\" ></script>" $(PUBLIC_DIR)/index.html ; \
 		echo include $$f; \
@@ -70,15 +79,17 @@ merge_xsl:
 
 merge_js:
 	for f in ${BUILDFILELIST} ; do source "$$f" ; done ; \
+	if [ "$${BUILD_MODULE_LEGACY_MAP}" = "ENABLED" ] ; then \
+	MODULE_DEPENDENCIES+=($(LEGACY_MAP_JS)); \
+	fi ; \
 	JS_DIST_BUNDLE=$${JS_DIST_BUNDLE} AUTO_DISCOVER_MODULES=$$AUTO_DISCOVER_MODULES misc/merge_js.sh $${MODULE_DEPENDENCIES[*]}
 
 EXCLUDE_EXPR = %~ %.backup
-BUILDFILELIST = $(sort $(filter-out $(EXCLUDE_EXPR),$(wildcard build.properties.d/*)))
+EXTRA_BUILD_FILE?=
+BUILDFILELIST = $(sort $(filter-out $(EXCLUDE_EXPR),$(wildcard build.properties.d/*))) $(EXTRA_BUILD_FILE)
 build_properties:
 	@set -a -e ; \
-	pushd build.properties.files ; \
-	for f in ${BUILDFILELIST} ; do echo "processing ../$$f" && source "../$$f" ; done ; \
-	popd ; \
+	for f in ${BUILDFILELIST} ; do echo "processing $$f" && source "$$f" ; done ; \
 	BUILD_NUMBER=$(BUILD_NUMBER) ; \
 	PROPS=$$(printenv | grep -e "^BUILD_") ; \
 	echo "SET BUILD PROPERTIES:" ; \
@@ -206,6 +217,25 @@ convert-yaml:
 	  misc/yaml_to_json.py $$f > $${f%.yaml}.json ; \
 	done
 
+cp-legacy:
+	for f in ${BUILDFILELIST} ; do source "$$f" ; done ; \
+	if [ "$${BUILD_MODULE_LEGACY_MAP}" = "ENABLED" ] ; then \
+	$(MAKE) cp-legacy-map ; \
+	fi
+	for f in ${BUILDFILELIST} ; do source "$$f" ; done ; \
+	if [ "$${BUILD_MODULE_LEGACY_QUERY_FORM}" = "ENABLED" ] ; then \
+	$(MAKE) cp-legacy-query-form ; \
+	fi
+
+cp-legacy-map: $(addprefix $(PUBLIC_DIR)/, $(LEGACY_MAP_LIBS))
+	@echo "/*disabled in favor of legacy ext_map.js*/" > "$(PUBLIC_DIR)/js/map.bundle.js"
+
+cp-legacy-query-form: $(addprefix $(PUBLIC_DIR)/, $(LEGACY_QUERY_FORM_LIBS))
+	@echo "/*disabled in favor of legacy query_form.js*/" > "$(PUBLIC_DIR)/js/query-form.bundle.js"
+
+$(PUBLIC_DIR)/%: $(SRC_LEGACY_DIR)/%
+	cp $< $@
+
 cp-src:
 	cp -r $(SRC_CORE_DIR) $(PUBLIC_DIR)
 
@@ -267,7 +297,7 @@ $(LIBS_DIR)/css/leaflet.css: $(LIBS_DIR)/css/images
 	ln -s $(LIBS_DIR)/leaflet-1.5.1/leaflet.css $@
 
 $(LIBS_DIR)/css/images: unzip $(LIBS_DIR)/css
-	ln -s $(LIBS_DIR)/leaflet-1.5.1/images $@
+	ln -s $(LIBS_DIR)/leaflet-1.5.1/images $@ || true
 
 $(LIBS_DIR)/js/leaflet-graticule.js: unzip $(LIBS_DIR)/js
 	ln -s $(LIBS_DIR)/L.Graticule.js $@
@@ -319,7 +349,7 @@ $(LIBS_DIR)/css/fonts: $(LIBS_DIR)/css
 clean-ignore-zips:
 	$(RM) -r $(SSS_BIN_DIR)
 	$(RM) -r $(PUBLIC_DIR)
-	for f in $(LIBS_SUBDIRS); do unlink $$f || $(RM) -r $$f || true; done
+	for f in $(LIBS_SUBDIRS); do $(RM) -r $$f || true; done
 	$(RM) .server_done
 
 .PHONY: clean

build.properties.d/00_default.properties

@@ -56,6 +56,7 @@ BUILD_MODULE_EXT_COSMETICS_CUSTOMDATETIME=DISABLED
 BUILD_MODULE_EXT_QRCODE=ENABLED
 BUILD_MODULE_SHOW_ID_IN_LABEL=DISABLED
 BUILD_MODULE_LEGACY_QUERY_FORM=DISABLED
+BUILD_MODULE_LEGACY_MAP=ENABLED
 
 BUILD_MODULE_USER_MANAGEMENT=ENABLED
 BUILD_MODULE_USER_MANAGEMENT_CHANGE_OWN_PASSWORD_REALM=CaosDB
@@ -174,7 +175,6 @@ MODULE_DEPENDENCIES=(
     loglevel.js
     plotly.js
     webcaosdb.js
-    query_form.js
     pako.js
     utif.js
     ext_version_history.js
@@ -192,13 +192,6 @@ MODULE_DEPENDENCIES=(
     edit_mode.js
     ext_entity_state.js
     ext_file_download.js
-    leaflet.js
-    leaflet-graticule.js
-    leaflet-latlng-graticule.js
-    leaflet-coordinates.js
-    proj4.js
-    proj4leaflet.js
-    ext_map.js
     tour.js
     ext_bottom_line.js
     ext_sss_markdown.js
@@ -213,4 +206,5 @@ MODULE_DEPENDENCIES=(
     reference_resolver/caosdb_default_person.js
     ext_prop_display.js
     query-form.bundle.js
+    map.bundle.js
 )

misc/envoy.yaml

+# Sample envoy config suitable for the new map implementation which depends on
+# WebGRPC.
+#
+# Start with:
+#
+#     docker run -p 8081:8081 -p 9901:9901 -e ENVOY_UID=$(id -u) -v $(pwd)/envoy.yaml:/etc/envoy/envoy.yaml --network host envoyproxy/envoy:v1.27-latest`
+#
+# Browse to:
+#
+#     http://localhost:8081/
+#
+# We assume that the server's REST API is served locally on port 10443 (TLS)
+# and the server's GRPC API is served locally on port 8080 (no TLS).
+#
+# Envoy will expose both the REST API and the (translated) WebGRPC API under
+# port 8081 (no TLS). The /api/ path is routed to the GRPC API while all other
+# paths are routed to the REST API.
+#
+# WARNING: This is for testing and development purposes!
+static_resources:
+  listeners:
+  - name: listener_0
+    address:
+      socket_address: { address: 0.0.0.0, port_value: 8081 }
+    filter_chains:
+    - filters:
+      - name: envoy.filters.network.http_connection_manager
+        typed_config:
+          "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
+          access_log:
+            - name: stderr_logger
+              typed_config:
+                "@type": type.googleapis.com/envoy.extensions.access_loggers.stream.v3.StderrAccessLog
+          codec_type: auto
+          stat_prefix: ingress_http
+          upgrade_configs:
+          - upgrade_type: websocket
+          route_config:
+            name: local_route
+            virtual_hosts:
+            - name: local_service
+              domains: ["*"]
+              routes:
+              - match: { prefix: "/api/" }
+                route:
+                  prefix_rewrite: "/"
+                  cluster: caosdb_grpc_api
+                  max_stream_duration:
+                    grpc_timeout_header_max: 0s
+              - match: { prefix: "/" }
+                route:
+                  cluster: caosdb_http
+              cors:
+                allow_origin_string_match:
+                - prefix: "*"
+                allow_methods: GET, PUT, DELETE, POST, OPTIONS
+                allow_headers: keep-alive,user-agent,cache-control,content-type,content-transfer-encoding,custom-header-1,x-accept-content-transfer-encoding,x-accept-response-streaming,x-user-agent,x-grpc-web,grpc-timeout
+                max_age: "1728000"
+                expose_headers: custom-header-1,grpc-status,grpc-message
+          http_filters:
+          - name: envoy.filters.http.grpc_web
+            typed_config:
+              "@type": type.googleapis.com/envoy.extensions.filters.http.grpc_web.v3.GrpcWeb
+          - name: envoy.filters.http.cors
+            typed_config:
+              "@type": type.googleapis.com/envoy.extensions.filters.http.cors.v3.Cors
+          - name: envoy.filters.http.router
+            typed_config:
+              "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
+  clusters:
+
+  # this is the caosdb server's GRPC endpoint
+  - name: caosdb_grpc_api
+    connect_timeout: 0.25s
+    type: logical_dns
+    http2_protocol_options: {}
+    lb_policy: round_robin
+    load_assignment:
+      cluster_name: caosdb_grpc_api_cluster_0
+      endpoints:
+        - lb_endpoints:
+            - endpoint:
+                address:
+                  socket_address:
+                    address: 127.0.0.1
+                    port_value: 8080
+
+  # this is the caosdb server's XML/HTTP endpoint
+  - name: caosdb_http
+    connect_timeout: 0.25s
+    type: logical_dns
+    lb_policy: round_robin
+    transport_socket:
+      name: envoy.transport_sockets.tls
+      typed_config:
+        "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext
+
+    load_assignment:
+      cluster_name: caosdb_http_cluster_0
+      endpoints:
+        - lb_endpoints:
+            - endpoint:
+                address:
+                  socket_address:
+                    address: 127.0.0.1
+                    port_value: 10443

test/core/test.properties

+BUILD_MODULE_LEGACY_QUERY_FORM=ENABLED
+BUILD_MODULE_LEGACY_MAP=ENABLED