Skip to content
Snippets Groups Projects
Commit 51662cc7 authored by Florian Spreckelsen's avatar Florian Spreckelsen
Browse files

REL: Release 0.12.0 

Release 0.12.0

See merge request !112
parents c75928ec 602a643b
No related branches found
No related tags found
1 merge request!112Release 0.12.0
Pipeline #53557 passed
Stage: setup
Stage: cert
Stage: style
Stage: unittest
Stage: integrationtest
Stage: deploy
Show whitespace changes
Inline Side-by-side
CHANGELOG.md
+ 24
− 1
View file @ 51662cc7
......@@ -4,7 +4,30 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
## [0.11.0] 2024-07-09 ##
## [0.12.0] - 2024-07-31 ##
### Added ###
- Possibility to configure custom CSS to be used in static HTML sites created
by the crawler
### Changed ###
- The static HTML pages that are generated by the crawler now require the
webinterface to be built with `JS_DIST_BUNDLE=TRUE`, i.e. all js source files
bundled into one `webcaosdb.dist.js`. This is the default and highly
recommended for production systems in any case but it will break the display
of these static files in certain corner cases where you may want to use
`JS_DIST_BUNDLE=FALSE` for development or debugging reasons.
### Fixed ###
- Rendering of entities in static HTML sites created by the crawler.
- [#63](https://gitlab.com/linkahead/linkahead-advanced-user-tools/-/issues/63)
The `TableImporter` now gives a detailed output in what row/column
which TypeError occurred.
## [0.11.0] - 2024-07-09 ##
### Added ###
......
CITATION.cff
+ 2
− 2
View file @ 51662cc7
......@@ -20,6 +20,6 @@ authors:
given-names: Stefan
orcid: https://orcid.org/0000-0001-7214-8125
title: CaosDB - Advanced User Tools
version: 0.11.0
version: 0.12.0
doi: 10.3390/data4020083
date-released: 2024-07-09
\ No newline at end of file
date-released: 2024-07-31
\ No newline at end of file
setup.py
+ 1
− 1
View file @ 51662cc7
......@@ -46,7 +46,7 @@ from setuptools import find_packages, setup
########################################################################
MAJOR = 0
MINOR = 11
MINOR = 12
MICRO = 0
PRE = "" # e.g. rc0, alpha.1, 0.beta-23
ISRELEASED = True
......
src/caosadvancedtools/crawler.py
+ 19
− 15
View file @ 51662cc7
......@@ -518,6 +518,12 @@ ____________________\n""".format(i+1, len(pending_changes)) + str(el[3]))
"""
from xml.sax.saxutils import escape
caosdb_config = db.configuration.get_config()
if ("advancedtools" in caosdb_config and "crawler.customcssfile" in
caosdb_config["advancedtools"]):
cssfile = caosdb_config["advancedtools"]["crawler.customcssfile"]
else:
cssfile = None
# TODO move path related stuff to sss_helper
form = """
<html>
......@@ -526,21 +532,17 @@ ____________________\n""".format(i+1, len(pending_changes)) + str(el[3]))
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Crawler</title>
<link rel="stylesheet" href="{url}/webinterface/css/webcaosdb.css"/>
<link rel="stylesheet" href="{url}/webinterface/css/bootstrap.css">
<script src="{url}/webinterface/js/jquery.js"></script>
<script src="{url}/webinterface/js/utif.js"></script>
<script src="{url}/webinterface/js/loglevel.js"></script>
<script src="{url}/webinterface/js/bootstrap.js"></script>
<script src="{url}/webinterface/js/ext_table_preview.js"></script>
<script src="{url}/webinterface/js/webcaosdb.js"></script>
<script src="{url}/webinterface/js/plotly.js"></script>
<script src="{url}/webinterface/js/caosdb.js"></script>
<script src="{url}/webinterface/js/state-machine.js"></script>
<script src="{url}/webinterface/js/showdown.js"></script>
<script src="{url}/webinterface/js/preview.js"></script>
<script src="{url}/webinterface/js/ext_references.js"></script>
<script src="{url}/webinterface/js/ext_bottom_line.js"></script>
<link rel="stylesheet" href="{url}/webinterface/css/webcaosdb.css"/>
<link rel="stylesheet" href="{url}/webinterface/css/linkahead.css"/>
<link rel="stylesheet" href="{url}/webinterface/css/dropzone.css">
<link rel="stylesheet" href="{url}/webinterface/css/tour.css">
<link rel="stylesheet" href="{url}/webinterface/css/leaflet.css">
<link rel="stylesheet" href="{url}/webinterface/css/leaflet-coordinates.css">
<link rel="stylesheet" href="{url}/webinterface/css/bootstrap-select.css">
<link rel="stylesheet" href="{url}/webinterface/css/bootstrap-icons.css">
{customcssfile}
<script src="{url}/webinterface/webcaosdb.dist.js"></script>
</head>
<body>
<form method="post" action="{url}/scripting">
......@@ -582,9 +584,11 @@ ____________________\n""".format(i+1, len(pending_changes)) + str(el[3]))
</script>
</body>
</html>
""".format(url=db.configuration.get_config()["Connection"]["url"],
""".format(url=caosdb_config["Connection"]["url"],
rid=run_id,
changes=escape("\n".join(changes)),
customcssfile='<link rel="stylesheet" href="{url}/webinterface/css/{customcssfile}"/>'.format(
url=caosdb_config["Connection"]["url"], customcssfile=cssfile) if cssfile else "",
path=path)
if "SHARED_DIR" in os.environ:
......
src/caosadvancedtools/table_importer.py
+ 41
− 1
View file @ 51662cc7
......@@ -31,7 +31,7 @@ import logging
import pathlib
from datetime import datetime
import caosdb as db
import linkahead as db
import numpy as np
import pandas as pd
from xlrd import XLRDError
......@@ -537,6 +537,46 @@ class CSVImporter(TableImporter):
extra={'identifier': str(filename),
'category': "inconsistency"})
raise DataInconsistencyError(*ve.args)
except TypeError as te:
# Iterate through the columns and rows to identify
# problematic cells with wrong types.
df = pd.read_csv(filename, sep=sep,
converters=applicable_converters, dtype=None,
**kwargs)
error_dict = {}
columns_with_errors = []
for key, dtype in self.datatypes.items():
if key not in df.columns:
continue
try:
df[key].astype(dtype)
except (TypeError, ValueError):
columns_with_errors.append(key)
if not columns_with_errors:
# We may have run into any other TypeError not caused
# by wrong datatypes within the table.
raise te
for ii, row in df.iterrows():
for name in columns_with_errors:
try:
# we need to check with astype to provoke the
# same errors, but that only works on
# Dataframes, so cast value to list to
# DataFrame.
pd.DataFrame([row[name]]).astype(self.datatypes[name])
except (TypeError, ValueError):
if ii not in error_dict:
error_dict[ii] = []
error_dict[ii].append(
(name, str(self.datatypes[name]).strip("<>"), str(type(row[name])).strip("<>"))
)
msg = "Elements with wrong datatypes encountered:\n"
for ii, error_list in error_dict.items():
msg += f"* row {ii}:\n"
for err in error_list:
msg += f" * column \"{err[0]}\": Expected \"{err[1]}\" but found \"{err[2]}\".\n"
msg += '\n'
raise DataInconsistencyError(msg)
df = self.check_dataframe(df, filename)
......
src/doc/conf.py
+ 2
− 2
View file @ 51662cc7
......@@ -27,9 +27,9 @@ copyright = '2023, IndiScale GmbH'
author = 'Daniel Hornung'
# The short X.Y version
version = '0.11.0'
version = '0.12.0'
# The full version, including alpha/beta/rc tags
release = '0.11.0'
release = '0.12.0'
# -- General configuration ---------------------------------------------------
......
unittests/test_table_importer.py
+ 44
− 0
View file @ 51662cc7
......@@ -379,6 +379,50 @@ class CSVImporterTest(TableImporterTest):
assert df["int_with_gaps"].dtype == "Int64"
assert df["float"].dtype == float
def test_wrong_datatype_type_errors(self):
"""Test for
https://gitlab.com/linkahead/linkahead-advanced-user-tools/-/issues/63:
Highlight rows and columns in which type errors occur.
"""
tmpfile = NamedTemporaryFile(delete=False, suffix=".csv")
with open(tmpfile.name, 'w') as tmp:
# Wrong types in row 2, columns 1 and 2, and row 4, column 2.
tmp.write(
"int,float\n"
"1,2.3\n"
"4.5,word\n"
"0,1.2\n"
"-12,12+3j\n"
)
kwargs = {
"datatypes": {
"int": int,
"float": float,
"not-in-table": str # An unused datatype definition must not cause problems.
},
"obligatory_columns": ["int"],
"converters": {}
}
importer = CSVImporter(**kwargs)
with pytest.raises(DataInconsistencyError) as die:
df = importer.read_file(tmpfile.name)
msg = str(die.value)
print("\n" + msg)
assert "Elements with wrong datatypes encountered:\n" in msg
# Errors in rows 1 and 3, no errors in 2 and 4
assert "* row 1:\n" in msg
assert "* row 2:\n" not in msg
assert "* row 3:\n" in msg
assert "* row 4:\n" not in msg
row_1_msgs, row_3_msgs = msg.split("* row 1:\n")[1].split("* row 3:\n")
# exactly 2 errors in row 1, exactly 1 in row 3
assert len(row_1_msgs.strip().split('\n')) == 2
assert len(row_3_msgs.strip().split('\n')) == 1
assert " * column \"int\"" in row_1_msgs
assert " * column \"float\"" in row_1_msgs
assert " * column \"float\"" in row_3_msgs
class TSVImporterTest(TableImporterTest):
def test_full(self):
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment