From 8045be1d864b950816e99404cc384fd884bb64a5 Mon Sep 17 00:00:00 2001
From: Florian Spreckelsen <f.spreckelsen@indiscale.com>
Date: Fri, 21 Mar 2025 10:47:20 +0100
Subject: [PATCH 01/12] FEAT(ext_samplemanagement): Sort sample column names
and add sample name column
---
.../src/ext/js/ext_samplemanagement.js | 34 ++++++++++---------
1 file changed, 18 insertions(+), 16 deletions(-)
diff --git a/sample-management-custom/caosdb-server/caosdb-webui/src/ext/js/ext_samplemanagement.js b/sample-management-custom/caosdb-server/caosdb-webui/src/ext/js/ext_samplemanagement.js
index e8e46e0..d4956df 100644
--- a/sample-management-custom/caosdb-server/caosdb-webui/src/ext/js/ext_samplemanagement.js
+++ b/sample-management-custom/caosdb-server/caosdb-webui/src/ext/js/ext_samplemanagement.js
@@ -39,36 +39,35 @@ const ext_samplemanagement = function($, navbar, log, form_elements, form_panel,
const required_column_names = [
"LinkAhead ID",
"Main User",
- "Latitude start",
- "Longitude start",
- "Elevation start",
- "Start date",
- "Storage ID",
- "Device"
];
const requiredColumnNamesChildren = [
"LinkAhead ID",
"Main User",
"Parent LinkAhead ID",
- "SampleType",
- "Storage ID",
];
const non_sample_rt_column_names = [
+ "Latitude start",
+ "Storage ID",
"Biome",
"Campaign",
- "Start date",
+ "Device",
+ "Elevation start",
+ "Elevation stop",
"End date",
+ "Event responsible",
"IGSN DOI",
+ "Latitude stop",
"Level",
"Locality description",
"Locality name",
- "Sphere",
- "Event responsible",
- "Latitude stop",
+ "Longitude start",
"Longitude stop",
- "Elevation stop",
"PDFReport",
"Parent LinkAhead ID",
+ "Sample name",
+ "Sphere",
+ "Start date",
+ "Start date",
];
const allColumnNames = non_sample_rt_column_names.concat(requiredColumnNamesChildren).concat(required_column_names);
@@ -77,7 +76,7 @@ const ext_samplemanagement = function($, navbar, log, form_elements, form_panel,
"Event",
"Container",
'NagoyaCase',
- "Parent_sample",
+ "Parent_Sample",
]
const upload_sample_template_form_config = {
@@ -191,7 +190,7 @@ const ext_samplemanagement = function($, navbar, log, form_elements, form_panel,
// refresh the options of the column names select field once the list
// is available (The sample RT needs to be retrieved
var column_name_select = $(form).find("select[name='column_names']");
- collect_column_names(requiredColumnNames).then(function(names) {
+ collect_column_names(requiredColumnNames, true).then(function(names) {
for (let option of names) {
column_name_select.append(
form_elements._make_option(option.value, option.label));
@@ -260,11 +259,14 @@ const ext_samplemanagement = function($, navbar, log, form_elements, form_panel,
*
* @return {Array} Options
*/
- const collect_column_names = async function(requiredColumnNames) {
+ const collect_column_names = async function(requiredColumnNames, sorted) {
const a = await query('find recordtype with name=sample');
const column_names = $(a[0]).find(".caosdb-property-name").toArray().map(e => e.textContent);
var options = [...new Set(allColumnNames.concat(column_names))];
options = options.filter(n => !(unused_property_names.includes(n) || requiredColumnNames.includes(n))).sort();
+ if (sorted == true) {
+ options.sort();
+ }
return options.map(e => ({
value: e,
label: e
--
GitLab
From b8403ed3c34c77692a6151d2ca75feb63800d255 Mon Sep 17 00:00:00 2001
From: Florian Spreckelsen <f.spreckelsen@indiscale.com>
Date: Fri, 21 Mar 2025 10:53:40 +0100
Subject: [PATCH 02/12] FEAT(export_sample_csv): Export sample names
---
.../caosdb-server/scripting/bin/export_sample_csv.py | 6 ++++++
.../scripting/bin/sample_helpers/default_constants.yml | 2 ++
.../bin/sample_helpers/sample_upload_column_definitions.py | 2 +-
3 files changed, 9 insertions(+), 1 deletion(-)
diff --git a/sample-management-custom/caosdb-server/scripting/bin/export_sample_csv.py b/sample-management-custom/caosdb-server/scripting/bin/export_sample_csv.py
index 085c839..885fb55 100755
--- a/sample-management-custom/caosdb-server/scripting/bin/export_sample_csv.py
+++ b/sample-management-custom/caosdb-server/scripting/bin/export_sample_csv.py
@@ -335,6 +335,11 @@ def extract_event_url(record, key):
return None
+def extract_sample_name(record, key):
+
+ return record.name
+
+
# must include all keys from SPECIAL_TREATMENT
EXTRACTORS = use_custom_names({
"entity_id": lambda record, key: record.id,
@@ -356,6 +361,7 @@ EXTRACTORS = use_custom_names({
"Longitude stop": extract_lng_stop,
"PDFReport": extract_pdf_id,
"PI": extract_person,
+ "sample_name": extract_sample_name,
"Sampling method": default_find,
"Sphere": extract_sphere,
"Start date": extract_start_date,
diff --git a/sample-management-custom/caosdb-server/scripting/bin/sample_helpers/default_constants.yml b/sample-management-custom/caosdb-server/scripting/bin/sample_helpers/default_constants.yml
index c575cb6..136d9ee 100644
--- a/sample-management-custom/caosdb-server/scripting/bin/sample_helpers/default_constants.yml
+++ b/sample-management-custom/caosdb-server/scripting/bin/sample_helpers/default_constants.yml
@@ -25,6 +25,7 @@ csv_column_names:
locality_name_prop: "Locality name"
responsible_person_event: "Event responsible"
parent_sample_prop: "Parent LinkAhead ID"
+ sample_name: "Sample name"
csv_column_descriptions:
LinkAhead ID: "An ID generated by LinkAhead (either integer or URL to this entity). Do not change this column!"
@@ -52,6 +53,7 @@ csv_column_descriptions:
Timezone: "Timezone: Either UTC, CET, .... or in the form +hh[:mm], -hh:[mm]."
Water depth start: "The bottom depth in meters where the sampling started as a positive value"
Water depth stop: "The bottom depth in meters where the sampling ended as a positive value"
+ Sample name: "Name which will be given to the sample entity"
entity_names:
abbreviation_prop: abbreviation
diff --git a/sample-management-custom/caosdb-server/scripting/bin/sample_helpers/sample_upload_column_definitions.py b/sample-management-custom/caosdb-server/scripting/bin/sample_helpers/sample_upload_column_definitions.py
index 0584522..1ad1ade 100644
--- a/sample-management-custom/caosdb-server/scripting/bin/sample_helpers/sample_upload_column_definitions.py
+++ b/sample-management-custom/caosdb-server/scripting/bin/sample_helpers/sample_upload_column_definitions.py
@@ -79,7 +79,6 @@ DATATYPE_DEFINITIONS = use_custom_names({
# Must exist
OBLIGATORY_COLUMNS = use_custom_names([
"entity_id",
- "Start date",
])
OBLIGATORY_COLUMNS_CHILD = use_custom_names([
@@ -118,6 +117,7 @@ SPECIAL_TREATMENT_SAMPLE = use_custom_names([
"PI",
"PDFReport",
"parent_sample_prop",
+ "sample_name",
"Sphere",
"Start date",
"Storage ID",
--
GitLab
From ea2c2b7217bbba1cb48589780a6dcee9eb3261dc Mon Sep 17 00:00:00 2001
From: Florian Spreckelsen <f.spreckelsen@indiscale.com>
Date: Fri, 21 Mar 2025 10:58:56 +0100
Subject: [PATCH 03/12] FEAT(export_sample_csv): Always export sample names
---
.../caosdb-server/scripting/bin/export_sample_csv.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/sample-management-custom/caosdb-server/scripting/bin/export_sample_csv.py b/sample-management-custom/caosdb-server/scripting/bin/export_sample_csv.py
index 885fb55..e97cdf5 100755
--- a/sample-management-custom/caosdb-server/scripting/bin/export_sample_csv.py
+++ b/sample-management-custom/caosdb-server/scripting/bin/export_sample_csv.py
@@ -397,6 +397,7 @@ ADDITIONAL_EXPORTS = use_custom_names([
"LinkAhead URL",
"parent_sample_prop",
"Storage chain",
+ "sample_name"
])
--
GitLab
From 9f91ac71dd8b505bb7f35dc49c8fceea5ccba30b Mon Sep 17 00:00:00 2001
From: Florian Spreckelsen <f.spreckelsen@indiscale.com>
Date: Fri, 21 Mar 2025 11:16:55 +0100
Subject: [PATCH 04/12] FEAT(sample_upload): Add special treatment for sample
names
---
.../caosdb-server/scripting/bin/crawl_sample_data_async.py | 6 ++++++
.../bin/sample_helpers/sample_upload_column_definitions.py | 1 +
.../caosdb-server/scripting/bin/upload_sample_template.py | 2 +-
3 files changed, 8 insertions(+), 1 deletion(-)
diff --git a/sample-management-custom/caosdb-server/scripting/bin/crawl_sample_data_async.py b/sample-management-custom/caosdb-server/scripting/bin/crawl_sample_data_async.py
index ace23ee..a786927 100755
--- a/sample-management-custom/caosdb-server/scripting/bin/crawl_sample_data_async.py
+++ b/sample-management-custom/caosdb-server/scripting/bin/crawl_sample_data_async.py
@@ -145,6 +145,12 @@ def update_sample_records(data, htmluserlog_public):
raise DataInconsistencyError(msg)
# All special properties are added here
+ if get_column_header_name("sample_name") in data:
+ # We want to allow to overwrite the name with empty
+ # values. Note that this is cureently broken in the
+ # crawler and needs to be fixed there.
+ sample.name = return_value_if_not_none(row[get_column_header_name("sample_name")])
+
sample = add_special_properties(sample, row)
# Add additional properties
diff --git a/sample-management-custom/caosdb-server/scripting/bin/sample_helpers/sample_upload_column_definitions.py b/sample-management-custom/caosdb-server/scripting/bin/sample_helpers/sample_upload_column_definitions.py
index 1ad1ade..3bd2b85 100644
--- a/sample-management-custom/caosdb-server/scripting/bin/sample_helpers/sample_upload_column_definitions.py
+++ b/sample-management-custom/caosdb-server/scripting/bin/sample_helpers/sample_upload_column_definitions.py
@@ -69,6 +69,7 @@ DATATYPE_DEFINITIONS = use_custom_names({
"Longitude stop": float,
"PI": str,
"Parent LinkAhead ID": str,
+ "sample_name": str,
"SampleMethod": str,
"SampleType": str,
"Start date": str,
diff --git a/sample-management-custom/caosdb-server/scripting/bin/upload_sample_template.py b/sample-management-custom/caosdb-server/scripting/bin/upload_sample_template.py
index 7160007..db9aa64 100755
--- a/sample-management-custom/caosdb-server/scripting/bin/upload_sample_template.py
+++ b/sample-management-custom/caosdb-server/scripting/bin/upload_sample_template.py
@@ -41,7 +41,7 @@ from caosadvancedtools.table_importer import CSVImporter
from caoscrawler.logging import configure_server_side_logging
from bis_utils import (replace_entity_urls_by_ids,
- SPECIAL_TREATMENT_SAMPLE, whitespace_cleanup_in_df)
+ whitespace_cleanup_in_df)
from sample_helpers.sample_upload_column_definitions import (
COLUMN_CONVERTER, DATATYPE_DEFINITIONS,
OBLIGATORY_COLUMNS, OBLIGATORY_COLUMNS_CHILD, SPECIAL_TREATMENT_SAMPLE)
--
GitLab
From e4a851ee07b88a5e4e03f1d12a95dfea5e4c6660 Mon Sep 17 00:00:00 2001
From: Florian Spreckelsen <f.spreckelsen@indiscale.com>
Date: Fri, 21 Mar 2025 11:25:50 +0100
Subject: [PATCH 05/12] DOCS(sample_export): Move inline comment
---
.../caosdb-server/scripting/bin/crawl_sample_data_async.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/sample-management-custom/caosdb-server/scripting/bin/crawl_sample_data_async.py b/sample-management-custom/caosdb-server/scripting/bin/crawl_sample_data_async.py
index a786927..bf8192c 100755
--- a/sample-management-custom/caosdb-server/scripting/bin/crawl_sample_data_async.py
+++ b/sample-management-custom/caosdb-server/scripting/bin/crawl_sample_data_async.py
@@ -144,13 +144,13 @@ def update_sample_records(data, htmluserlog_public):
row[get_column_header_name("entity_id")])
raise DataInconsistencyError(msg)
- # All special properties are added here
if get_column_header_name("sample_name") in data:
# We want to allow to overwrite the name with empty
# values. Note that this is cureently broken in the
# crawler and needs to be fixed there.
sample.name = return_value_if_not_none(row[get_column_header_name("sample_name")])
+ # All special properties are added here
sample = add_special_properties(sample, row)
# Add additional properties
--
GitLab
From 1e4188c75904b0232fa412eedffee399355f474b Mon Sep 17 00:00:00 2001
From: Florian Spreckelsen <f.spreckelsen@indiscale.com>
Date: Fri, 21 Mar 2025 11:34:02 +0100
Subject: [PATCH 06/12] REFACTOR: Sort and use consistent button names
---
.../src/ext/js/ext_samplemanagement.js | 41 ++++++++++---------
.../src/ext/js/ext_stockmanagement.js | 4 +-
2 files changed, 23 insertions(+), 22 deletions(-)
diff --git a/sample-management-custom/caosdb-server/caosdb-webui/src/ext/js/ext_samplemanagement.js b/sample-management-custom/caosdb-server/caosdb-webui/src/ext/js/ext_samplemanagement.js
index d4956df..322e9d3 100644
--- a/sample-management-custom/caosdb-server/caosdb-webui/src/ext/js/ext_samplemanagement.js
+++ b/sample-management-custom/caosdb-server/caosdb-webui/src/ext/js/ext_samplemanagement.js
@@ -19,11 +19,11 @@
const ext_samplemanagement = function($, navbar, log, form_elements, form_panel, sample_management_reference_resolver) {
const logger = log.getLogger("samplemanagement_form");
const tool_box = "Sample Management"
- const upload_sample_template_title = "Upload sample template"; // title of the form and text in the toolbox
+ const upload_sample_template_title = "Update sample"; // title of the form and text in the toolbox
const upload_sample_template_panel_id = "upload_sample_template_form_panel";
- const register_new_samples_title = "Register new samples"; // title of the form and text in the toolbox
+ const register_new_samples_title = "Register samples"; // title of the form and text in the toolbox
const register_new_samples_panel_id = "register_new_samples_form_panel";
- const registerNewChildSamplesTitle = "Register new child samples"; // title of the form and text in the toolbox
+ const registerNewChildSamplesTitle = "Register child samples"; // title of the form and text in the toolbox
const registerNewChildSamplesPanelId = "register_new_children_form_panel";
const upload_pdf_id = "upload_pdf_form_panel";
const upload_pdf_title = "Upload PDF file";
@@ -46,8 +46,8 @@ const ext_samplemanagement = function($, navbar, log, form_elements, form_panel,
"Parent LinkAhead ID",
];
const non_sample_rt_column_names = [
- "Latitude start",
- "Storage ID",
+ "Latitude start",
+ "Storage ID",
"Biome",
"Campaign",
"Device",
@@ -64,7 +64,7 @@ const ext_samplemanagement = function($, navbar, log, form_elements, form_panel,
"Longitude stop",
"PDFReport",
"Parent LinkAhead ID",
- "Sample name",
+ "Sample name",
"Sphere",
"Start date",
"Start date",
@@ -264,9 +264,9 @@ const ext_samplemanagement = function($, navbar, log, form_elements, form_panel,
const column_names = $(a[0]).find(".caosdb-property-name").toArray().map(e => e.textContent);
var options = [...new Set(allColumnNames.concat(column_names))];
options = options.filter(n => !(unused_property_names.includes(n) || requiredColumnNames.includes(n))).sort();
- if (sorted == true) {
- options.sort();
- }
+ if (sorted == true) {
+ options.sort();
+ }
return options.map(e => ({
value: e,
label: e
@@ -307,12 +307,6 @@ const ext_samplemanagement = function($, navbar, log, form_elements, form_panel,
*/
const init_show_samplemanagement_panel_button = async function() {
//var form_wrapper = form_elements.make_form(config);
- navbar.add_tool(upload_sample_template_title, tool_box, {
- callback: form_panel.create_show_form_callback(
- upload_sample_template_panel_id,
- upload_sample_template_title,
- upload_sample_template_form_config)
- });
navbar.add_tool(register_new_samples_title, tool_box, {
callback: form_panel.create_show_form_callback(
register_new_samples_panel_id,
@@ -327,12 +321,11 @@ const ext_samplemanagement = function($, navbar, log, form_elements, form_panel,
undefined,
initRegisterChildSamplesForm)
});
- navbar.add_tool("Create sample template", tool_box, {
+ navbar.add_tool(upload_sample_template_title, tool_box, {
callback: form_panel.create_show_form_callback(
- "create-sample-template",
- "Create sample template",
- undefined,
- initRegisterTemplateForm)
+ upload_sample_template_panel_id,
+ upload_sample_template_title,
+ upload_sample_template_form_config)
});
// Set auto_focus=false because of WebUI bug: https://gitlab.com/linkahead/linkahead-webui/-/issues/258
navbar.add_tool(upload_pdf_title, tool_box, {
@@ -361,6 +354,14 @@ const ext_samplemanagement = function($, navbar, log, form_elements, form_panel,
input_field.addEventListener("blur", check_pattern);
}
});
+
+ navbar.add_tool("Create sample template", tool_box, {
+ callback: form_panel.create_show_form_callback(
+ "create-sample-template",
+ "Create sample template",
+ undefined,
+ initRegisterTemplateForm)
+ });
};
diff --git a/sample-management-custom/caosdb-server/caosdb-webui/src/ext/js/ext_stockmanagement.js b/sample-management-custom/caosdb-server/caosdb-webui/src/ext/js/ext_stockmanagement.js
index 4775b9d..4463cfb 100644
--- a/sample-management-custom/caosdb-server/caosdb-webui/src/ext/js/ext_stockmanagement.js
+++ b/sample-management-custom/caosdb-server/caosdb-webui/src/ext/js/ext_stockmanagement.js
@@ -16,7 +16,7 @@ const ext_stockmanagement = function($, navbar, log, form_elements, form_panel,
const tool_box = "Storage Management" // The item that is shown in the top-navbar
///////////////// REGISTER NEW CONTAINERS START
- const register_new_containers_title = "Register new Containers";
+ const register_new_containers_title = "Register containers";
const register_new_containers_panel_id = "register_new_containers_form_panel";
const container_id_label = ("${BUILD_MODULE_EXT_STOCKMANAGEMENT_ID_LABEL}" != "") ? "${BUILD_MODULE_EXT_STOCKMANAGEMENT_ID_LABEL}" : "Container ID";
const register_new_containers_form_config = {
@@ -109,7 +109,7 @@ const ext_stockmanagement = function($, navbar, log, form_elements, form_panel,
///////////////// EXPORT CONTAINER CSV START
const export_csv_id = "export_container_csv";
- const export_csv_title = "Export existing containers to CSV";
+ const export_csv_title = "Export containers to CSV";
const export_csv_form_config = {
script: "export_container_csv.py",
--
GitLab
From a44c56446f47aeee2d4a19b9260076c7651b6159 Mon Sep 17 00:00:00 2001
From: Florian Spreckelsen <f.spreckelsen@indiscale.com>
Date: Fri, 21 Mar 2025 12:49:29 +0100
Subject: [PATCH 07/12] FIX(datamodel): Fix capitalization of Parent_Sample
---
models_and_helper_scripts/sample-management-datamodel.yml | 2 +-
.../scripting/bin/sample_helpers/default_constants.yml | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/models_and_helper_scripts/sample-management-datamodel.yml b/models_and_helper_scripts/sample-management-datamodel.yml
index aef3b68..4d12dc6 100644
--- a/models_and_helper_scripts/sample-management-datamodel.yml
+++ b/models_and_helper_scripts/sample-management-datamodel.yml
@@ -83,7 +83,7 @@ Sample:
recommended_properties:
Container:
SampleType:
- Parent_sample:
+ Parent_Sample:
datatype: Sample
Main User:
datatype: Person
diff --git a/sample-management-custom/caosdb-server/scripting/bin/sample_helpers/default_constants.yml b/sample-management-custom/caosdb-server/scripting/bin/sample_helpers/default_constants.yml
index 136d9ee..8761cae 100644
--- a/sample-management-custom/caosdb-server/scripting/bin/sample_helpers/default_constants.yml
+++ b/sample-management-custom/caosdb-server/scripting/bin/sample_helpers/default_constants.yml
@@ -70,7 +70,7 @@ entity_names:
labelcounter_prop: counter
labelcounter_rt: LabelCounter
last_name_prop: last_name
- parent_sample_prop: Parent_sample
+ parent_sample_prop: Parent_Sample
responsible_rt: Responsible
start_date_prop: start_date
locality_description_prop: locality_description
--
GitLab
From cb62744a594ccbe6472d716c9aefcaa6d5415af8 Mon Sep 17 00:00:00 2001
From: Florian Spreckelsen <f.spreckelsen@indiscale.com>
Date: Fri, 21 Mar 2025 12:59:24 +0100
Subject: [PATCH 08/12] FIX: typo
---
.../caosdb-webui/src/ext/js/ext_samplemanagement.js | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/sample-management-custom/caosdb-server/caosdb-webui/src/ext/js/ext_samplemanagement.js b/sample-management-custom/caosdb-server/caosdb-webui/src/ext/js/ext_samplemanagement.js
index 322e9d3..acf9a18 100644
--- a/sample-management-custom/caosdb-server/caosdb-webui/src/ext/js/ext_samplemanagement.js
+++ b/sample-management-custom/caosdb-server/caosdb-webui/src/ext/js/ext_samplemanagement.js
@@ -19,7 +19,7 @@
const ext_samplemanagement = function($, navbar, log, form_elements, form_panel, sample_management_reference_resolver) {
const logger = log.getLogger("samplemanagement_form");
const tool_box = "Sample Management"
- const upload_sample_template_title = "Update sample"; // title of the form and text in the toolbox
+ const upload_sample_template_title = "Update samples"; // title of the form and text in the toolbox
const upload_sample_template_panel_id = "upload_sample_template_form_panel";
const register_new_samples_title = "Register samples"; // title of the form and text in the toolbox
const register_new_samples_panel_id = "register_new_samples_form_panel";
--
GitLab
From 29b2b317eef79c24dbffe4d6d2049225bf837f53 Mon Sep 17 00:00:00 2001
From: Florian Spreckelsen <f.spreckelsen@indiscale.com>
Date: Fri, 21 Mar 2025 13:47:28 +0100
Subject: [PATCH 09/12] FIX(sample_upload): Use correct column names in event
sanity checks
---
.../sample_helpers/sample_upload_get_event.py | 29 ++++++++++---------
1 file changed, 15 insertions(+), 14 deletions(-)
diff --git a/sample-management-custom/caosdb-server/scripting/bin/sample_helpers/sample_upload_get_event.py b/sample-management-custom/caosdb-server/scripting/bin/sample_helpers/sample_upload_get_event.py
index c15df79..1949541 100644
--- a/sample-management-custom/caosdb-server/scripting/bin/sample_helpers/sample_upload_get_event.py
+++ b/sample-management-custom/caosdb-server/scripting/bin/sample_helpers/sample_upload_get_event.py
@@ -85,6 +85,7 @@ def add_event_to_sample(sample: db.Record, data: pd.Series) -> db.Record:
# only add if there was any event data at all:
if len(event.properties) > 0:
+
sample = update_property(sample, event_rt.id, event, property_name=event_rt.name)
return sample
@@ -108,39 +109,39 @@ def _create_position(mode: str, lat: float, lng: float, ele: float):
def _perform_sanity_checks(sample, data):
- if (get_column_header_name("end_date") in data and
- return_value_if_not_none(data[get_column_header_name("end_date")]) is not None):
- if (get_column_header_name("start_date") not in data or
- return_value_if_not_none(data[get_column_header_name("start_date")]) is None):
+ if (get_column_header_name("end_date_prop") in data and
+ return_value_if_not_none(data[get_column_header_name("end_date_prop")]) is not None):
+ if (get_column_header_name("start_date_prop") not in data or
+ return_value_if_not_none(data[get_column_header_name("start_date_prop")]) is None):
raise DataInconsistencyError(
f"Sample with {get_entity_name('entity_id')} {sample.id} has a "
- f"{get_column_header_name('end_date')} but no valid "
- f"{get_column_header_name('start_date')}."
+ f"{get_column_header_name('end_date_prop')} but no valid "
+ f"{get_column_header_name('start_date_prop')}."
)
for name in ["start", "stop"]:
- bool_list = [get_column_header_name(f"{val}_{name}") in data for val in [
- "latitude", "longitude", "elevation"]]
+ bool_list = [get_column_header_name(f"{val} {name}") in data for val in [
+ "Latitude", "Longitude", "Elevation"]]
raise_error = False
if any(bool_list):
if not all(bool_list):
raise_error = True
- elif any([return_value_if_not_none(data[get_column_header_name(f"{val}_{name}")]) is None for val in ["latitude", "longitude", "elevation"]]):
+ elif any([return_value_if_not_none(data[get_column_header_name(f"{val} {name}")]) is None for val in ["Latitude", "Longitude", "Elevation"]]):
raise_error = True
if raise_error:
raise DataInconsistencyError(
- f"Sample with {get_entity_name('entity_id')} {sample.id} has an "
+ f"Sample with {get_column_header_name('entity_id')} {sample.id} has an "
f"invalid {name} position. Please make sure that latitude, longitude, "
"and elevation are provided."
)
# only need to check lat since we already checked that if lat is
# present, lng and ele are present, too
- if (get_column_header_name("latitude_stop") in data and
- return_value_if_not_none(data[get_column_header_name("latitude_stop")]) is not None):
- if (get_column_header_name("latitude_start") not in data or
- return_value_if_not_none(data[get_column_header_name("latitude_start")]) is not None):
+ if (get_column_header_name("Latitude stop") in data and
+ return_value_if_not_none(data[get_column_header_name("Latitude stop")]) is not None):
+ if (get_column_header_name("Latitude start") not in data or
+ return_value_if_not_none(data[get_column_header_name("Latitude start")]) is not None):
raise DataInconsistencyError(
f"Sample with {get_entity_name('entity_id')} {sample.id} has a "
f"{get_entity_name('StopPosition')} but no valid "
--
GitLab
From aaf057af48008bde561a18f0d7a6f23339382886 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Henrik=20tom=20W=C3=B6rden?= <h.tomwoerden@indiscale.com>
Date: Fri, 21 Mar 2025 14:23:26 +0100
Subject: [PATCH 10/12] FEAT: add columns required for event insertion
automatically
---
.../scripting/bin/register_new_samples.py | 6 +-
.../sample_registration_check_column_names.py | 68 +++++++++++++++++++
2 files changed, 72 insertions(+), 2 deletions(-)
create mode 100644 sample-management-custom/caosdb-server/scripting/bin/sample_helpers/sample_registration_check_column_names.py
diff --git a/sample-management-custom/caosdb-server/scripting/bin/register_new_samples.py b/sample-management-custom/caosdb-server/scripting/bin/register_new_samples.py
index 15e22bd..a320c0d 100755
--- a/sample-management-custom/caosdb-server/scripting/bin/register_new_samples.py
+++ b/sample-management-custom/caosdb-server/scripting/bin/register_new_samples.py
@@ -13,6 +13,7 @@ from bis_utils import (create_email_with_link_text,
get_options_row, send_mail_with_defaults)
from sample_helpers.sample_registration_get_person_identifier import get_person_identifier
from sample_helpers.sample_registration_post_processing import post_process_samples
+from sample_helpers.sample_registration_check_column_names import check_column_names
from sample_helpers.utils import CONSTANTS, get_column_header_name, get_entity_name
ERROR_PREFIX = CONSTANTS["error_prefix"]
@@ -34,8 +35,9 @@ def get_column_names(data):
other_names = data["required_column_names"].split(',') + data["column_names"]
starting_names.extend([name.strip()
for name in other_names if name.strip() not in starting_names])
- return [get_column_header_name(name) for name in starting_names]
-
+ cnames = [get_column_header_name(name) for name in starting_names]
+ check_column_names(cnames)
+ return cnames
def create_sample_entities(data):
responsible_person_id = int(data["responsible_person"])
diff --git a/sample-management-custom/caosdb-server/scripting/bin/sample_helpers/sample_registration_check_column_names.py b/sample-management-custom/caosdb-server/scripting/bin/sample_helpers/sample_registration_check_column_names.py
new file mode 100644
index 0000000..79f62dd
--- /dev/null
+++ b/sample-management-custom/caosdb-server/scripting/bin/sample_helpers/sample_registration_check_column_names.py
@@ -0,0 +1,68 @@
+#
+# Copyright (C) 2025 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2025 Henrik tom Wörden <h.tomwoerden@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+import linkahead as db
+import logging
+
+from caosadvancedtools.datainconsistency import DataInconsistencyError
+
+from .utils import CONSTANTS, get_column_header_name, get_entity_name
+from .sample_upload_column_definitions import use_custom_names
+
+logger = logging.getLogger("caosadvancedtools")
+
+def check_column_names(names: list[str]) -> None:
+ event_and_position_properties = use_custom_names([
+ "Campaign",
+ "Biome",
+ "Device",
+ "Elevation start",
+ "Elevation stop",
+ "End date",
+ "responsible_person_event",
+ "igsn_doi_prop",
+ "Latitude start",
+ "Latitude stop",
+ "Level",
+ "locality_description_prop",
+ "locality_name_prop",
+ "Longitude start",
+ "Longitude stop",
+ "Sphere",
+ "Start date",
+ ])
+ needs_events = False
+ for name in names:
+ if name in event_and_position_properties:
+ needs_events = True
+ appended = []
+ if needs_events:
+ for name in use_custom_names([
+ "Longitude start",
+ "Latitude start",
+ "Elevation start",
+ "Start date",
+ ]):
+ if name not in names:
+ names.append(name)
+ appended.append(name)
+
+ if appended:
+ logger.warning(
+ "The following columns were added in order to allow consitent data insertion:\n"
+ f"{', '.join(appended)}"
+ )
--
GitLab
From 1d3df5795cc432e606aff3839e0929a0dee8aa5a Mon Sep 17 00:00:00 2001
From: Florian Spreckelsen <f.spreckelsen@indiscale.com>
Date: Fri, 21 Mar 2025 15:21:27 +0100
Subject: [PATCH 11/12] FEAT: Raise error when identifying event properties are
missing
---
.../sample_upload_column_definitions.py | 2 +-
.../bin/sample_helpers/sample_upload_get_event.py | 15 +++++++++++++++
.../scripting/bin/upload_sample_template.py | 7 +++----
3 files changed, 19 insertions(+), 5 deletions(-)
diff --git a/sample-management-custom/caosdb-server/scripting/bin/sample_helpers/sample_upload_column_definitions.py b/sample-management-custom/caosdb-server/scripting/bin/sample_helpers/sample_upload_column_definitions.py
index 3bd2b85..251f5f1 100644
--- a/sample-management-custom/caosdb-server/scripting/bin/sample_helpers/sample_upload_column_definitions.py
+++ b/sample-management-custom/caosdb-server/scripting/bin/sample_helpers/sample_upload_column_definitions.py
@@ -91,7 +91,7 @@ COLUMN_CONVERTER = use_custom_names({
"Collector": semicolon_separated_list,
"Curator": semicolon_separated_list,
"Embargo": _embargo_converter,
- "Event responsible": semicolon_separated_list,
+ "responsible_person_event": semicolon_separated_list,
"Sphere": semicolon_separated_list,
})
diff --git a/sample-management-custom/caosdb-server/scripting/bin/sample_helpers/sample_upload_get_event.py b/sample-management-custom/caosdb-server/scripting/bin/sample_helpers/sample_upload_get_event.py
index 1949541..3bc4947 100644
--- a/sample-management-custom/caosdb-server/scripting/bin/sample_helpers/sample_upload_get_event.py
+++ b/sample-management-custom/caosdb-server/scripting/bin/sample_helpers/sample_upload_get_event.py
@@ -15,6 +15,7 @@
# You should have received a copy of the GNU Affero General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#
+
import linkahead as db
import pandas as pd
@@ -85,6 +86,20 @@ def add_event_to_sample(sample: db.Record, data: pd.Series) -> db.Record:
# only add if there was any event data at all:
if len(event.properties) > 0:
+ # Start date and position are identifying event properties
+ if event.get_property(get_entity_name("start_date_prop")) is None:
+ raise DataInconsistencyError(
+ f"Sample with {get_column_header_name('entity_id')} {sample.id} has an "
+ f"{get_entity_name('event_rt')} but is missing a {
+ get_column_header_name('start_date_prop')} "
+ "which is required."
+ )
+ if event.get_property(get_entity_name(position_prop.name)) is None:
+ raise DataInconsistencyError(
+ f"Sample with {get_column_header_name('entity_id')} {sample.id} has an "
+ f"{get_entity_name('event_rt')} but is missing start position information "
+ "which is required."
+ )
sample = update_property(sample, event_rt.id, event, property_name=event_rt.name)
diff --git a/sample-management-custom/caosdb-server/scripting/bin/upload_sample_template.py b/sample-management-custom/caosdb-server/scripting/bin/upload_sample_template.py
index db9aa64..1e25408 100755
--- a/sample-management-custom/caosdb-server/scripting/bin/upload_sample_template.py
+++ b/sample-management-custom/caosdb-server/scripting/bin/upload_sample_template.py
@@ -40,8 +40,7 @@ from caosadvancedtools.serverside import helper
from caosadvancedtools.table_importer import CSVImporter
from caoscrawler.logging import configure_server_side_logging
-from bis_utils import (replace_entity_urls_by_ids,
- whitespace_cleanup_in_df)
+from bis_utils import (replace_entity_urls_by_ids)
from sample_helpers.sample_upload_column_definitions import (
COLUMN_CONVERTER, DATATYPE_DEFINITIONS,
OBLIGATORY_COLUMNS, OBLIGATORY_COLUMNS_CHILD, SPECIAL_TREATMENT_SAMPLE)
@@ -96,8 +95,8 @@ def read_data_from_file(filename):
)
raise DataInconsistencyError("There was a problem with the CSV upload.")
- # strip leading and trailing whitespaces
- return whitespace_cleanup_in_df(df)
+ # TODO: strip leading and trailing whitespaces
+ return df
def _get_converter_from_property_datatype(dt):
--
GitLab
From b152e29deb70b262e4f86648c6bf3268b004b3b4 Mon Sep 17 00:00:00 2001
From: Florian Spreckelsen <f.spreckelsen@indiscale.com>
Date: Fri, 21 Mar 2025 15:27:57 +0100
Subject: [PATCH 12/12] FIX: Faulty line break
---
.../scripting/bin/sample_helpers/sample_upload_get_event.py | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/sample-management-custom/caosdb-server/scripting/bin/sample_helpers/sample_upload_get_event.py b/sample-management-custom/caosdb-server/scripting/bin/sample_helpers/sample_upload_get_event.py
index 3bc4947..b726a66 100644
--- a/sample-management-custom/caosdb-server/scripting/bin/sample_helpers/sample_upload_get_event.py
+++ b/sample-management-custom/caosdb-server/scripting/bin/sample_helpers/sample_upload_get_event.py
@@ -90,9 +90,8 @@ def add_event_to_sample(sample: db.Record, data: pd.Series) -> db.Record:
if event.get_property(get_entity_name("start_date_prop")) is None:
raise DataInconsistencyError(
f"Sample with {get_column_header_name('entity_id')} {sample.id} has an "
- f"{get_entity_name('event_rt')} but is missing a {
- get_column_header_name('start_date_prop')} "
- "which is required."
+ f"{get_entity_name('event_rt')} but is missing a "
+ f"{get_column_header_name('start_date_prop')} which is required."
)
if event.get_property(get_entity_name(position_prop.name)) is None:
raise DataInconsistencyError(
--
GitLab