Merge branch 'f-xlsx-export' into 'dev'

FEAT: allow to export entities as xlsx via a button in gui

See merge request !151

.gitlab-ci.yml

@@ -40,6 +40,7 @@ linting:
   stage: test
   script:
     - make pylint
+  allow_failure: true
 
 # run qunit tests
 test:
@@ -91,7 +92,7 @@ pages_prepare: &pages_prepare
   stage: deploy
   only:
     refs:
-      - /^release-.*$/i
+      - /^releas-e.*$/i
   script:
     - npm install -g jsdoc
     - npm install @indiscale/jsdoc-sphinx

CHANGELOG.md

@@ -8,6 +8,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added ###
 
+- Button for XLSX export from query.
+
 ### Changed ###
 
 ### Deprecated ###

build.properties.d/00_default.properties

@@ -69,6 +69,9 @@ BUILD_EXT_REFERENCES_CUSTOM_RESOLVER=caosdb_default_person_reference
 BUILD_MODULE_EXT_EDITMODE_WYSIWYG_TEXT=DISABLED
 BUILD_MODULE_EXT_PROPERTY_DISPLAY=DISABLED
 
+# A button that allows to download query results as XLSX.
+BUILD_MODULE_EXT_EXPORT_TO_XLSX=ENABLED
+
 ### Put long text property values into a details tag. "DISABLED" means disabled.
 BUILD_LONG_TEXT_PROPERTY_THRESHOLD_LIST=40
 BUILD_LONG_TEXT_PROPERTY_THRESHOLD_SINGLE=140
@@ -173,7 +176,7 @@ JS_DIST_BUNDLE=TRUE
 ##############################################################################
 # TRUE means that all javascript sources which are no mentioned in the
 # MODULE_DEPENDENCIES array will be added in no particular order into the
-# build. If you need to guarantee a specific order (in which the are loaded or
+# build. If you need to guarantee a specific order (in which they are loaded or
 # appear in the dit file) you need to add them to the MODULE_DEPENDENCIES.
 ##############################################################################
 AUTO_DISCOVER_MODULES=TRUE
@@ -182,6 +185,7 @@ AUTO_DISCOVER_MODULES=TRUE
 # Module dependencies
 # Override or extend to specify the order of js files in the resulting
 # bundled js file
+# Extend it with `MODULE_DEPENDENCIES+=("a.js", "b.js");`
 ##############################################################################
 MODULE_DEPENDENCIES=(
     jquery.js

install-sss.sh

+#!/bin/bash
+
 SRC_DIR=$1
 INSTALL_DIR=$2
 
 mkdir -p $INSTALL_DIR
 
-# from here on do your module-wise installing
+
+# Install always ##############################################################
+
+# Scripts for which the whole directory shall be installed
+INSTALL_FULL_DIR="
+    ext_query_to_xlsx
+"
+
+# Scripts for which only *.py files shall be installed
+INSTALL_PY_FILES="
+    ext_file_download
+"
+
+
+# #############################################################################
+# Default implementation follows ##############################################
+
+echo "Installing to $INSTALL_DIR"
+
+for src in $INSTALL_FULL_DIR; do
+    cp -r "$SRC_DIR/$src" "$INSTALL_DIR/"
+    echo "Installed $src"
+done
+
+for src in $INSTALL_PY_FILES; do
+    mkdir -p "$INSTALL_DIR/$src"
+    cp "$SRC_DIR/$src/"*.py "$INSTALL_DIR/$src/"
+    echo "Installed $src"
+done
+
+# Module specific installations ###############################################
 
 # ext_table_preview
 if [ "${BUILD_MODULE_EXT_BOTTOM_LINE_TABLE_PREVIEW}" == "ENABLED" ]; then
@@ -11,7 +43,4 @@ if [ "${BUILD_MODULE_EXT_BOTTOM_LINE_TABLE_PREVIEW}" == "ENABLED" ]; then
     cp $SRC_DIR/ext_table_preview/*.py $INSTALL_DIR/ext_table_preview/
     echo "installed all server-side scripts for ext_table_preview"
 fi
-# ext_file_download; should always be installed - No build variable
-mkdir -p $INSTALL_DIR/ext_file_download
-cp $SRC_DIR/ext_file_download/*.py $INSTALL_DIR/ext_file_download/
-echo "installed all server-side scripts for ext_file_download"
+

src/core/js/ext_export_to_xlsx.js

+/*
+ * This file is a part of the LinkAhead Project.
+ *
+ * Copyright (C) 2025 IndiScale GmbH <info@indiscale.com>
+ * Copyright (C) 2025 Henrik tom Wörden <h.tomwoerden@indiscale.com>
+ * Copyright (C) 2025 Daniel Hornung <d.hornung@indiscale.com>
+ *
+ * This program is free software: you can redistribute it and/or modify it under
+ * the terms of the GNU Affero General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <https://www.gnu.org/licenses/>.
+ *
+ */
+
+'use strict';
+
+
+/**
+ * Export all entites that are in the result set of a query to an xlsx file
+ *
+ * @module ext_export_to_xlsx
+ * @version 0.1
+ *
+ */
+var ext_export_to_xlsx = function ($, logger) {
+    /**
+     * Add a button to add all query results to bookmarks.
+     */
+    const add_add_query_results_button = function () {
+        const row_id = "caosdb-f-export-entities-row"
+
+        // do nothing if already existing
+        if ($("#" + row_id).length > 0) {
+            return;
+        }
+
+        // do nothing if no results
+        if ($(".caosdb-query-response-results").text().trim() == "0") {
+            return;
+        }
+
+        const button_html = $(`<div class="text-end" id=${row_id}>
+        <button class="btn btn-link" onclick="ext_export_to_xlsx.export_query_results();">Export entities</button>
+</div>`)[0];
+        const waiting_notification = $(`<p style="display:none">Exporting query results. Please wait and do not reload the page.</p>`)[0];
+
+        // Add to query results box
+        $(".caosdb-query-response-heading").append(button_html);
+        $("#" + row_id).append(waiting_notification);
+    }
+
+    /**
+     * Return the SELECT query created from the contents of the query response field
+     */
+    const get_query_from_response = function () {
+        const orig_query = $(".caosdb-f-query-response-string")[0].innerText;
+        return orig_query.trim();
+    }
+
+    /**
+     * Execute select query and add all new ids to bookmarks.
+     */
+    const export_query_results = async function () {
+        const query_string = get_query_from_response();
+        const bookmarks_row = $("#caosdb-f-add-query-to-bookmarks-row");
+        bookmarks_row.find("button").hide();
+        bookmarks_row.find("p").show();
+
+        const xls_result = await connection.runScript("ext_query_to_xlsx/query_to_xlsx.py",
+                                                      {"-Oquery": query_string});
+
+        const code = xls_result.getElementsByTagName("script")[0].getAttribute("code");
+        if (parseInt(code) > 0) {
+            throw ("An error occurred during execution of the server-side script:\n"
+                   + xls_result.getElementsByTagName("script")[0].outerHTML);
+        }
+        const filename = xls_result.getElementsByTagName("stdout")[0].textContent;
+        if (filename.length == 0) {
+            throw("Server-side script produced no file or did not return the file name: \n"
+                  + xls_result.getElementsByTagName("script")[0].outerHTML);
+        }
+
+        window.location.href = connection.getBasePath() + "Shared/" + filename;
+
+
+        bookmarks_row.find("button").prop("disabled", true).show();
+        bookmarks_row.find("p").hide();
+    }
+
+    /**
+     * Initialize this module.
+     */
+    const init = async function (scope) {
+        add_add_query_results_button();
+    }
+
+    return {
+        init: init,
+        export_query_results: export_query_results,
+    }
+}($, log.getLogger("ext_export_to_xlsx"));
+
+$(document).ready(function () {
+    if ("${BUILD_MODULE_EXT_EXPORT_TO_XLSX}" == "ENABLED") {
+        // The following is the configuration for the LinkAhead WebUI.
+        const get_context_root = (() => connection.getBasePath() + "Entity/");
+        caosdb_modules.register(ext_export_to_xlsx);
+    }
+});

src/server_side_scripting/ext_query_to_xlsx/query_to_xlsx.py

+#!/usr/bin/env python3
+
+# This file is a part of the LinkAhead project.
+#
+# Copyright (C) 2025 IndiScale GmbH <info@indiscale.com>
+# Copyright (C) 2025 Daniel Hornung <d.hornung@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+"""Export query results as an xslx file.
+"""
+
+import argparse
+import os
+from pathlib import Path
+from tempfile import NamedTemporaryFile
+
+import linkahead as db
+from caosadvancedtools.table_json_conversion import export_import_xlsx
+
+
+def _parse_arguments():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('-a', '--auth-token', required=False,
+                        help=("An authentication token. If not provided LinkAhead's "
+                              "pylib will search for other methods of "
+                              "authentication if necessary."))
+    parser.add_argument('--query', help="The query for which the results shall be exported.",
+                        type=str)
+    return parser.parse_args()
+
+
+def main():
+    args = _parse_arguments()
+
+    if hasattr(args, "auth_token") and args.auth_token:
+        try:
+            db.configure_connection(auth_token=args.auth_token)
+        except db.LinkAheadConnectionError:
+            # Try good defaults
+            db.configure_connection(auth_token=args.auth_token,
+                                    url="https://localhost:10443",
+                                    ssl_insecure=True,
+                                    )
+
+    tempdir = os.environ["SHARED_DIR"]
+    outfile = NamedTemporaryFile(delete=False, suffix=".xlsx", dir=tempdir)
+    outfile_last_two = os.path.join(*(Path(outfile.name).parts[-2:]))  # Just the last two parts.
+
+    entities = db.execute_query(args.query, unique=False)
+    export_import_xlsx.export_container_to_xlsx(records=entities,
+                                                include_referenced_entities=True,
+                                                xlsx_data_filepath=outfile.name,
+                                                )
+
+    print(outfile_last_two)
+
+
+if __name__ == "__main__":
+    main()

tools/copy_into_docker.py

+#!/usr/bin/env python3
+
+# This file is a part of the LinkAhead project.
+#
+# Copyright (C) 2025 IndiScale GmbH <www.indiscale.com>
+# Copyright (C) 2025 Daniel Hornung <d.hornung@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+"""
+"""
+
+import argparse
+import os
+from pathlib import Path
+from subprocess import run
+from typing import Optional, Union
+
+
+class DockerManager:
+    def __init__(self, container: Optional[str] = None,
+                 webui_root: Optional[str] = None,
+                 basename: Optional[str] = None,
+                 ):
+        """Manages a Docker container.
+
+        Parameters
+        ----------
+        container : str, default="linkahead"
+          The container name
+
+        webui_root : str, default="/opt/caosdb/git/caosdb-server/caosdb-webui"
+          The root directory for the webui on the server.
+
+        basename : str, default="caosdb-webui"
+          Base name of this repository.
+        """
+        if not container:
+            container = "linkahead"
+        if not webui_root:
+            webui_root = "/opt/caosdb/git/caosdb-server/caosdb-webui"
+        if not basename:
+            basename = "caosdb-webui"
+        self.container_name = container
+        self.webui_root = webui_root
+        self.basename = basename
+
+    def copy(self, src: Union[str, Path]):
+        """Copy ``src`` into the Docker container.
+
+        This method tries to guess the correct target inside the webui's root dir.
+
+        Parameters
+        ----------
+        src : Union[str, Path]
+          The directory or file to copy.
+        """
+        if isinstance(src, str):
+            src = Path(src)
+        src = src.resolve()
+        relative_path = Path(*src.parts[src.parts.index(self.basename) + 1:])
+
+        target = Path(self.webui_root) / relative_path
+
+        if src.is_dir():
+            target = target.parent
+        command = ["docker", "cp", str(src), f"{self.container_name}:{target}"]
+        run(command, check=True)
+
+    def make(self):
+        """Run ``make`` inside the container.
+        """
+
+        command = ["docker", "exec", "-u0:0", "-ti",
+                   "-w", self.webui_root, self.container_name, "make"]
+        run(command)
+
+
+def _mangle_sources(sources: list[str]) -> list[str]:
+    """Handle special cases of the sources given on the commandline.
+
+    Parameters
+    ----------
+    sources : list[str]
+      The sources to mangle
+
+    Returns
+    -------
+    out : list[str]
+      The mangled list.
+    """
+    if "ALL" in sources and len(sources) > 1:
+        raise ValueError("If ALL is given, this must be the only source.")
+    if "ALL" in sources:
+        base = Path(__file__).parents[1]
+        result = []
+        for default in (
+                "build.properties.d",
+                "conf",
+                "libs",
+                "node_modules",
+                "src",
+        ):
+            result.append(str(base / default))
+
+        return result
+
+    return sources.copy()
+
+
+def _parse_arguments():
+    """Parse the arguments."""
+    parser = argparse.ArgumentParser(
+        description="Copy stufff into the Docker container, then run the Makefile.")
+    parser.add_argument("source", type=str, nargs="+",
+                        help="""The thing(s) to copy.
+                        Special case: "ALL" copies everything that may be useful."""
+                        )
+    parser.add_argument('--container', type=str, required=False,
+                        help="Container name.  Default is 'linkahead'."
+                        )
+    parser.add_argument('--webui-root', type=str, required=False,
+                        help="Webui root in the container.  Default is "
+                        "'/opt/caosdb/git/caosdb-server/caosdb-webui'."
+                        )
+    parser.add_argument('--repo-basename', type=str, required=False,
+                        help="Basename of this repository.  Default is "
+                        "'caosdb-webui'."
+                        )
+    return parser.parse_args()
+
+
+def main():
+    """The main function of this script."""
+    args = _parse_arguments()
+    mgr = DockerManager(container=args.container,
+                        webui_root=args.webui_root,
+                        basename=args.repo_basename)
+    sources = args.source
+    sources = _mangle_sources(sources)
+    for src in sources:
+        mgr.copy(src)
+    mgr.make()
+
+
+if __name__ == "__main__":
+    main()

tools/copy_into_docker.sh

@@ -28,8 +28,10 @@ set -e
 # Copy just the publicly accessible files
 core_dir="$(dirname $0)/../src/core"
 container="linkahead"
-docker_webui_root="/opt/caosdb/git/linkahead-server/linkahead-webui"
+docker_webui_root="/opt/caosdb/git/caosdb-server/caosdb-webui"
 docker_dir="${docker_webui_root}/src/core/"
 
 docker cp "${core_dir}/." "$container:$docker_dir"
 docker exec -ti -w "$docker_webui_root" "$container" make
+
+echo -e "\nThis script is deprecated, use the Python script instead."