diff --git a/models_and_helper_scripts/sample-management-datamodel.yml b/models_and_helper_scripts/sample-management-datamodel.yml
index aef3b68183e47b862b3189622822247dcb8c3ee3..4d12dc6999feb0e3f5272912349392f7b699ed84 100644
--- a/models_and_helper_scripts/sample-management-datamodel.yml
+++ b/models_and_helper_scripts/sample-management-datamodel.yml
@@ -83,7 +83,7 @@ Sample:
recommended_properties:
Container:
SampleType:
- Parent_sample:
+ Parent_Sample:
datatype: Sample
Main User:
datatype: Person
diff --git a/sample-management-custom/caosdb-server/caosdb-webui/src/ext/js/ext_samplemanagement.js b/sample-management-custom/caosdb-server/caosdb-webui/src/ext/js/ext_samplemanagement.js
index e8e46e0d88ac2cc40b4fe89fe5032ecbbdd3cb82..acf9a189737a9ba0d8b07fa1d6bd5226e99b2c1d 100644
--- a/sample-management-custom/caosdb-server/caosdb-webui/src/ext/js/ext_samplemanagement.js
+++ b/sample-management-custom/caosdb-server/caosdb-webui/src/ext/js/ext_samplemanagement.js
@@ -19,11 +19,11 @@
const ext_samplemanagement = function($, navbar, log, form_elements, form_panel, sample_management_reference_resolver) {
const logger = log.getLogger("samplemanagement_form");
const tool_box = "Sample Management"
- const upload_sample_template_title = "Upload sample template"; // title of the form and text in the toolbox
+ const upload_sample_template_title = "Update samples"; // title of the form and text in the toolbox
const upload_sample_template_panel_id = "upload_sample_template_form_panel";
- const register_new_samples_title = "Register new samples"; // title of the form and text in the toolbox
+ const register_new_samples_title = "Register samples"; // title of the form and text in the toolbox
const register_new_samples_panel_id = "register_new_samples_form_panel";
- const registerNewChildSamplesTitle = "Register new child samples"; // title of the form and text in the toolbox
+ const registerNewChildSamplesTitle = "Register child samples"; // title of the form and text in the toolbox
const registerNewChildSamplesPanelId = "register_new_children_form_panel";
const upload_pdf_id = "upload_pdf_form_panel";
const upload_pdf_title = "Upload PDF file";
@@ -39,36 +39,35 @@ const ext_samplemanagement = function($, navbar, log, form_elements, form_panel,
const required_column_names = [
"LinkAhead ID",
"Main User",
- "Latitude start",
- "Longitude start",
- "Elevation start",
- "Start date",
- "Storage ID",
- "Device"
];
const requiredColumnNamesChildren = [
"LinkAhead ID",
"Main User",
"Parent LinkAhead ID",
- "SampleType",
- "Storage ID",
];
const non_sample_rt_column_names = [
+ "Latitude start",
+ "Storage ID",
"Biome",
"Campaign",
- "Start date",
+ "Device",
+ "Elevation start",
+ "Elevation stop",
"End date",
+ "Event responsible",
"IGSN DOI",
+ "Latitude stop",
"Level",
"Locality description",
"Locality name",
- "Sphere",
- "Event responsible",
- "Latitude stop",
+ "Longitude start",
"Longitude stop",
- "Elevation stop",
"PDFReport",
"Parent LinkAhead ID",
+ "Sample name",
+ "Sphere",
+ "Start date",
+ "Start date",
];
const allColumnNames = non_sample_rt_column_names.concat(requiredColumnNamesChildren).concat(required_column_names);
@@ -77,7 +76,7 @@ const ext_samplemanagement = function($, navbar, log, form_elements, form_panel,
"Event",
"Container",
'NagoyaCase',
- "Parent_sample",
+ "Parent_Sample",
]
const upload_sample_template_form_config = {
@@ -191,7 +190,7 @@ const ext_samplemanagement = function($, navbar, log, form_elements, form_panel,
// refresh the options of the column names select field once the list
// is available (The sample RT needs to be retrieved
var column_name_select = $(form).find("select[name='column_names']");
- collect_column_names(requiredColumnNames).then(function(names) {
+ collect_column_names(requiredColumnNames, true).then(function(names) {
for (let option of names) {
column_name_select.append(
form_elements._make_option(option.value, option.label));
@@ -260,11 +259,14 @@ const ext_samplemanagement = function($, navbar, log, form_elements, form_panel,
*
* @return {Array} Options
*/
- const collect_column_names = async function(requiredColumnNames) {
+ const collect_column_names = async function(requiredColumnNames, sorted) {
const a = await query('find recordtype with name=sample');
const column_names = $(a[0]).find(".caosdb-property-name").toArray().map(e => e.textContent);
var options = [...new Set(allColumnNames.concat(column_names))];
options = options.filter(n => !(unused_property_names.includes(n) || requiredColumnNames.includes(n))).sort();
+ if (sorted == true) {
+ options.sort();
+ }
return options.map(e => ({
value: e,
label: e
@@ -305,12 +307,6 @@ const ext_samplemanagement = function($, navbar, log, form_elements, form_panel,
*/
const init_show_samplemanagement_panel_button = async function() {
//var form_wrapper = form_elements.make_form(config);
- navbar.add_tool(upload_sample_template_title, tool_box, {
- callback: form_panel.create_show_form_callback(
- upload_sample_template_panel_id,
- upload_sample_template_title,
- upload_sample_template_form_config)
- });
navbar.add_tool(register_new_samples_title, tool_box, {
callback: form_panel.create_show_form_callback(
register_new_samples_panel_id,
@@ -325,12 +321,11 @@ const ext_samplemanagement = function($, navbar, log, form_elements, form_panel,
undefined,
initRegisterChildSamplesForm)
});
- navbar.add_tool("Create sample template", tool_box, {
+ navbar.add_tool(upload_sample_template_title, tool_box, {
callback: form_panel.create_show_form_callback(
- "create-sample-template",
- "Create sample template",
- undefined,
- initRegisterTemplateForm)
+ upload_sample_template_panel_id,
+ upload_sample_template_title,
+ upload_sample_template_form_config)
});
// Set auto_focus=false because of WebUI bug: https://gitlab.com/linkahead/linkahead-webui/-/issues/258
navbar.add_tool(upload_pdf_title, tool_box, {
@@ -359,6 +354,14 @@ const ext_samplemanagement = function($, navbar, log, form_elements, form_panel,
input_field.addEventListener("blur", check_pattern);
}
});
+
+ navbar.add_tool("Create sample template", tool_box, {
+ callback: form_panel.create_show_form_callback(
+ "create-sample-template",
+ "Create sample template",
+ undefined,
+ initRegisterTemplateForm)
+ });
};
diff --git a/sample-management-custom/caosdb-server/caosdb-webui/src/ext/js/ext_stockmanagement.js b/sample-management-custom/caosdb-server/caosdb-webui/src/ext/js/ext_stockmanagement.js
index 4775b9dff5327cf026106a530da05023f0858f8d..4463cfbb539bcfaf9898b280520e9e584638945f 100644
--- a/sample-management-custom/caosdb-server/caosdb-webui/src/ext/js/ext_stockmanagement.js
+++ b/sample-management-custom/caosdb-server/caosdb-webui/src/ext/js/ext_stockmanagement.js
@@ -16,7 +16,7 @@ const ext_stockmanagement = function($, navbar, log, form_elements, form_panel,
const tool_box = "Storage Management" // The item that is shown in the top-navbar
///////////////// REGISTER NEW CONTAINERS START
- const register_new_containers_title = "Register new Containers";
+ const register_new_containers_title = "Register containers";
const register_new_containers_panel_id = "register_new_containers_form_panel";
const container_id_label = ("${BUILD_MODULE_EXT_STOCKMANAGEMENT_ID_LABEL}" != "") ? "${BUILD_MODULE_EXT_STOCKMANAGEMENT_ID_LABEL}" : "Container ID";
const register_new_containers_form_config = {
@@ -109,7 +109,7 @@ const ext_stockmanagement = function($, navbar, log, form_elements, form_panel,
///////////////// EXPORT CONTAINER CSV START
const export_csv_id = "export_container_csv";
- const export_csv_title = "Export existing containers to CSV";
+ const export_csv_title = "Export containers to CSV";
const export_csv_form_config = {
script: "export_container_csv.py",
diff --git a/sample-management-custom/caosdb-server/scripting/bin/crawl_sample_data_async.py b/sample-management-custom/caosdb-server/scripting/bin/crawl_sample_data_async.py
index ace23eef4c481fdc20057fce83ef44d4c45dbca5..bf8192c8d97c08c29abc9f06c8189f2e8580cade 100755
--- a/sample-management-custom/caosdb-server/scripting/bin/crawl_sample_data_async.py
+++ b/sample-management-custom/caosdb-server/scripting/bin/crawl_sample_data_async.py
@@ -144,6 +144,12 @@ def update_sample_records(data, htmluserlog_public):
row[get_column_header_name("entity_id")])
raise DataInconsistencyError(msg)
+ if get_column_header_name("sample_name") in data:
+ # We want to allow to overwrite the name with empty
+ # values. Note that this is cureently broken in the
+ # crawler and needs to be fixed there.
+ sample.name = return_value_if_not_none(row[get_column_header_name("sample_name")])
+
# All special properties are added here
sample = add_special_properties(sample, row)
diff --git a/sample-management-custom/caosdb-server/scripting/bin/export_sample_csv.py b/sample-management-custom/caosdb-server/scripting/bin/export_sample_csv.py
index 085c8398fbea5b4fa5ce12a5625b7ea8d0c24e29..e97cdf53659682d849ea7f92ee21e3765741b164 100755
--- a/sample-management-custom/caosdb-server/scripting/bin/export_sample_csv.py
+++ b/sample-management-custom/caosdb-server/scripting/bin/export_sample_csv.py
@@ -335,6 +335,11 @@ def extract_event_url(record, key):
return None
+def extract_sample_name(record, key):
+
+ return record.name
+
+
# must include all keys from SPECIAL_TREATMENT
EXTRACTORS = use_custom_names({
"entity_id": lambda record, key: record.id,
@@ -356,6 +361,7 @@ EXTRACTORS = use_custom_names({
"Longitude stop": extract_lng_stop,
"PDFReport": extract_pdf_id,
"PI": extract_person,
+ "sample_name": extract_sample_name,
"Sampling method": default_find,
"Sphere": extract_sphere,
"Start date": extract_start_date,
@@ -391,6 +397,7 @@ ADDITIONAL_EXPORTS = use_custom_names([
"LinkAhead URL",
"parent_sample_prop",
"Storage chain",
+ "sample_name"
])
diff --git a/sample-management-custom/caosdb-server/scripting/bin/register_new_samples.py b/sample-management-custom/caosdb-server/scripting/bin/register_new_samples.py
index 15e22bd9c17a67098519e38f6fc8f9c976f09718..a320c0d9aabb7af53d780a0e35c9065773ff7085 100755
--- a/sample-management-custom/caosdb-server/scripting/bin/register_new_samples.py
+++ b/sample-management-custom/caosdb-server/scripting/bin/register_new_samples.py
@@ -13,6 +13,7 @@ from bis_utils import (create_email_with_link_text,
get_options_row, send_mail_with_defaults)
from sample_helpers.sample_registration_get_person_identifier import get_person_identifier
from sample_helpers.sample_registration_post_processing import post_process_samples
+from sample_helpers.sample_registration_check_column_names import check_column_names
from sample_helpers.utils import CONSTANTS, get_column_header_name, get_entity_name
ERROR_PREFIX = CONSTANTS["error_prefix"]
@@ -34,8 +35,9 @@ def get_column_names(data):
other_names = data["required_column_names"].split(',') + data["column_names"]
starting_names.extend([name.strip()
for name in other_names if name.strip() not in starting_names])
- return [get_column_header_name(name) for name in starting_names]
-
+ cnames = [get_column_header_name(name) for name in starting_names]
+ check_column_names(cnames)
+ return cnames
def create_sample_entities(data):
responsible_person_id = int(data["responsible_person"])
diff --git a/sample-management-custom/caosdb-server/scripting/bin/sample_helpers/default_constants.yml b/sample-management-custom/caosdb-server/scripting/bin/sample_helpers/default_constants.yml
index c575cb61f4cbd50676d8b944f7af7631963907bf..8761cae6a01228c593292b78ff32830eaa1dd1f5 100644
--- a/sample-management-custom/caosdb-server/scripting/bin/sample_helpers/default_constants.yml
+++ b/sample-management-custom/caosdb-server/scripting/bin/sample_helpers/default_constants.yml
@@ -25,6 +25,7 @@ csv_column_names:
locality_name_prop: "Locality name"
responsible_person_event: "Event responsible"
parent_sample_prop: "Parent LinkAhead ID"
+ sample_name: "Sample name"
csv_column_descriptions:
LinkAhead ID: "An ID generated by LinkAhead (either integer or URL to this entity). Do not change this column!"
@@ -52,6 +53,7 @@ csv_column_descriptions:
Timezone: "Timezone: Either UTC, CET, .... or in the form +hh[:mm], -hh:[mm]."
Water depth start: "The bottom depth in meters where the sampling started as a positive value"
Water depth stop: "The bottom depth in meters where the sampling ended as a positive value"
+ Sample name: "Name which will be given to the sample entity"
entity_names:
abbreviation_prop: abbreviation
@@ -68,7 +70,7 @@ entity_names:
labelcounter_prop: counter
labelcounter_rt: LabelCounter
last_name_prop: last_name
- parent_sample_prop: Parent_sample
+ parent_sample_prop: Parent_Sample
responsible_rt: Responsible
start_date_prop: start_date
locality_description_prop: locality_description
diff --git a/sample-management-custom/caosdb-server/scripting/bin/sample_helpers/sample_registration_check_column_names.py b/sample-management-custom/caosdb-server/scripting/bin/sample_helpers/sample_registration_check_column_names.py
new file mode 100644
index 0000000000000000000000000000000000000000..79f62ddf697619f8300b5d680e5092926974f74e
--- /dev/null
+++ b/sample-management-custom/caosdb-server/scripting/bin/sample_helpers/sample_registration_check_column_names.py
@@ -0,0 +1,68 @@
+#
+# Copyright (C) 2025 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2025 Henrik tom Wörden <h.tomwoerden@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+import linkahead as db
+import logging
+
+from caosadvancedtools.datainconsistency import DataInconsistencyError
+
+from .utils import CONSTANTS, get_column_header_name, get_entity_name
+from .sample_upload_column_definitions import use_custom_names
+
+logger = logging.getLogger("caosadvancedtools")
+
+def check_column_names(names: list[str]) -> None:
+ event_and_position_properties = use_custom_names([
+ "Campaign",
+ "Biome",
+ "Device",
+ "Elevation start",
+ "Elevation stop",
+ "End date",
+ "responsible_person_event",
+ "igsn_doi_prop",
+ "Latitude start",
+ "Latitude stop",
+ "Level",
+ "locality_description_prop",
+ "locality_name_prop",
+ "Longitude start",
+ "Longitude stop",
+ "Sphere",
+ "Start date",
+ ])
+ needs_events = False
+ for name in names:
+ if name in event_and_position_properties:
+ needs_events = True
+ appended = []
+ if needs_events:
+ for name in use_custom_names([
+ "Longitude start",
+ "Latitude start",
+ "Elevation start",
+ "Start date",
+ ]):
+ if name not in names:
+ names.append(name)
+ appended.append(name)
+
+ if appended:
+ logger.warning(
+ "The following columns were added in order to allow consitent data insertion:\n"
+ f"{', '.join(appended)}"
+ )
diff --git a/sample-management-custom/caosdb-server/scripting/bin/sample_helpers/sample_upload_column_definitions.py b/sample-management-custom/caosdb-server/scripting/bin/sample_helpers/sample_upload_column_definitions.py
index 058452266722257db5023f076031d2cc570a54dd..251f5f1dc1fb43e9d8f7d162a94cc02bf3ddfc7a 100644
--- a/sample-management-custom/caosdb-server/scripting/bin/sample_helpers/sample_upload_column_definitions.py
+++ b/sample-management-custom/caosdb-server/scripting/bin/sample_helpers/sample_upload_column_definitions.py
@@ -69,6 +69,7 @@ DATATYPE_DEFINITIONS = use_custom_names({
"Longitude stop": float,
"PI": str,
"Parent LinkAhead ID": str,
+ "sample_name": str,
"SampleMethod": str,
"SampleType": str,
"Start date": str,
@@ -79,7 +80,6 @@ DATATYPE_DEFINITIONS = use_custom_names({
# Must exist
OBLIGATORY_COLUMNS = use_custom_names([
"entity_id",
- "Start date",
])
OBLIGATORY_COLUMNS_CHILD = use_custom_names([
@@ -91,7 +91,7 @@ COLUMN_CONVERTER = use_custom_names({
"Collector": semicolon_separated_list,
"Curator": semicolon_separated_list,
"Embargo": _embargo_converter,
- "Event responsible": semicolon_separated_list,
+ "responsible_person_event": semicolon_separated_list,
"Sphere": semicolon_separated_list,
})
@@ -118,6 +118,7 @@ SPECIAL_TREATMENT_SAMPLE = use_custom_names([
"PI",
"PDFReport",
"parent_sample_prop",
+ "sample_name",
"Sphere",
"Start date",
"Storage ID",
diff --git a/sample-management-custom/caosdb-server/scripting/bin/sample_helpers/sample_upload_get_event.py b/sample-management-custom/caosdb-server/scripting/bin/sample_helpers/sample_upload_get_event.py
index c15df79805965ec982012775a113704979ecdf1b..b726a66977be066b7e5371e03c3dfac147f28eb4 100644
--- a/sample-management-custom/caosdb-server/scripting/bin/sample_helpers/sample_upload_get_event.py
+++ b/sample-management-custom/caosdb-server/scripting/bin/sample_helpers/sample_upload_get_event.py
@@ -15,6 +15,7 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#
+
import linkahead as db
import pandas as pd
@@ -85,6 +86,20 @@ def add_event_to_sample(sample: db.Record, data: pd.Series) -> db.Record:
# only add if there was any event data at all:
if len(event.properties) > 0:
+ # Start date and position are identifying event properties
+ if event.get_property(get_entity_name("start_date_prop")) is None:
+ raise DataInconsistencyError(
+ f"Sample with {get_column_header_name('entity_id')} {sample.id} has an "
+ f"{get_entity_name('event_rt')} but is missing a "
+ f"{get_column_header_name('start_date_prop')} which is required."
+ )
+ if event.get_property(get_entity_name(position_prop.name)) is None:
+ raise DataInconsistencyError(
+ f"Sample with {get_column_header_name('entity_id')} {sample.id} has an "
+ f"{get_entity_name('event_rt')} but is missing start position information "
+ "which is required."
+ )
+
sample = update_property(sample, event_rt.id, event, property_name=event_rt.name)
return sample
@@ -108,39 +123,39 @@ def _create_position(mode: str, lat: float, lng: float, ele: float):
def _perform_sanity_checks(sample, data):
- if (get_column_header_name("end_date") in data and
- return_value_if_not_none(data[get_column_header_name("end_date")]) is not None):
- if (get_column_header_name("start_date") not in data or
- return_value_if_not_none(data[get_column_header_name("start_date")]) is None):
+ if (get_column_header_name("end_date_prop") in data and
+ return_value_if_not_none(data[get_column_header_name("end_date_prop")]) is not None):
+ if (get_column_header_name("start_date_prop") not in data or
+ return_value_if_not_none(data[get_column_header_name("start_date_prop")]) is None):
raise DataInconsistencyError(
f"Sample with {get_entity_name('entity_id')} {sample.id} has a "
- f"{get_column_header_name('end_date')} but no valid "
- f"{get_column_header_name('start_date')}."
+ f"{get_column_header_name('end_date_prop')} but no valid "
+ f"{get_column_header_name('start_date_prop')}."
)
for name in ["start", "stop"]:
- bool_list = [get_column_header_name(f"{val}_{name}") in data for val in [
- "latitude", "longitude", "elevation"]]
+ bool_list = [get_column_header_name(f"{val} {name}") in data for val in [
+ "Latitude", "Longitude", "Elevation"]]
raise_error = False
if any(bool_list):
if not all(bool_list):
raise_error = True
- elif any([return_value_if_not_none(data[get_column_header_name(f"{val}_{name}")]) is None for val in ["latitude", "longitude", "elevation"]]):
+ elif any([return_value_if_not_none(data[get_column_header_name(f"{val} {name}")]) is None for val in ["Latitude", "Longitude", "Elevation"]]):
raise_error = True
if raise_error:
raise DataInconsistencyError(
- f"Sample with {get_entity_name('entity_id')} {sample.id} has an "
+ f"Sample with {get_column_header_name('entity_id')} {sample.id} has an "
f"invalid {name} position. Please make sure that latitude, longitude, "
"and elevation are provided."
)
# only need to check lat since we already checked that if lat is
# present, lng and ele are present, too
- if (get_column_header_name("latitude_stop") in data and
- return_value_if_not_none(data[get_column_header_name("latitude_stop")]) is not None):
- if (get_column_header_name("latitude_start") not in data or
- return_value_if_not_none(data[get_column_header_name("latitude_start")]) is not None):
+ if (get_column_header_name("Latitude stop") in data and
+ return_value_if_not_none(data[get_column_header_name("Latitude stop")]) is not None):
+ if (get_column_header_name("Latitude start") not in data or
+ return_value_if_not_none(data[get_column_header_name("Latitude start")]) is not None):
raise DataInconsistencyError(
f"Sample with {get_entity_name('entity_id')} {sample.id} has a "
f"{get_entity_name('StopPosition')} but no valid "
diff --git a/sample-management-custom/caosdb-server/scripting/bin/upload_sample_template.py b/sample-management-custom/caosdb-server/scripting/bin/upload_sample_template.py
index 716000720b5a8595dd0add4a31e870c9dac7b275..1e25408f5da7d3d073cef81f31364e468f930bf5 100755
--- a/sample-management-custom/caosdb-server/scripting/bin/upload_sample_template.py
+++ b/sample-management-custom/caosdb-server/scripting/bin/upload_sample_template.py
@@ -40,8 +40,7 @@ from caosadvancedtools.serverside import helper
from caosadvancedtools.table_importer import CSVImporter
from caoscrawler.logging import configure_server_side_logging
-from bis_utils import (replace_entity_urls_by_ids,
- SPECIAL_TREATMENT_SAMPLE, whitespace_cleanup_in_df)
+from bis_utils import (replace_entity_urls_by_ids)
from sample_helpers.sample_upload_column_definitions import (
COLUMN_CONVERTER, DATATYPE_DEFINITIONS,
OBLIGATORY_COLUMNS, OBLIGATORY_COLUMNS_CHILD, SPECIAL_TREATMENT_SAMPLE)
@@ -96,8 +95,8 @@ def read_data_from_file(filename):
)
raise DataInconsistencyError("There was a problem with the CSV upload.")
- # strip leading and trailing whitespaces
- return whitespace_cleanup_in_df(df)
+ # TODO: strip leading and trailing whitespaces
+ return df
def _get_converter_from_property_datatype(dt):