Merge branch 'dev' into f-table-json-data-schema

.gitlab-ci.yml

@@ -163,18 +163,17 @@ unittest_py312:
   script: *python_test_script
 
 unittest_py313:
-  allow_failure: true
   tags: [docker]
   stage: unittest
   image: python:3.13-rc
   script:
     # TODO: Replace by '*python_test_script' as soon as 3.13 has been officially released.
-    - apt update && apt install -y cargo
-    - pip install meson[ninja] meson-python
-    - pip install pynose pandas pytest pytest-cov gitignore-parser openpyxl>=3.0.7 xlrd==1.2 h5py
-    - pip install git+https://gitlab.indiscale.com/caosdb/src/caosdb-pylib.git@dev
-    - pip install .
-    - pytest --cov=caosadvancedtools unittests
+    - apt update && apt install -y cargo || true
+    - pip install meson[ninja] meson-python || true
+    - pip install pynose pandas pytest pytest-cov gitignore-parser openpyxl>=3.0.7 xlrd==1.2 h5py || true
+    - pip install git+https://gitlab.indiscale.com/caosdb/src/caosdb-pylib.git@dev || true
+    - pip install . || true
+    - pytest --cov=caosadvancedtools unittests || true
 
 # Build the sphinx documentation and make it ready for deployment by Gitlab Pages
 # Special job for serving a static website. See https://docs.gitlab.com/ee/ci/yaml/README.html#pages

CHANGELOG.md

@@ -8,6 +8,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added ###
 
+### Changed ###
+
+### Deprecated ###
+
+### Removed ###
+
+### Fixed ###
+
+### Security ###
+
+### Documentation ###
+
+## [0.10.0] - 2024-04-24 ##
+
+### Added ###
+
 - XLSX handling: template generator and conversion from Json to XLSX
 - Json schema exporter:
   - has new parameter `use_rt_pool`
@@ -16,7 +32,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed ###
 
-### Deprecated ###
+* `table_converter.to_table` now returns an empty DataFrame instead of raising a
+  ValueError when called with an empty container.
 
 ### Removed ###
 
@@ -27,8 +44,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Fixed ###
 
 - Json schema exporter handles reference properties better.
-
-### Security ###
+- [#59](https://gitlab.com/linkahead/linkahead-advanced-user-tools/-/issues/59)
+  `to_table` failed on lists as values.
 
 ### Documentation ###
 

CITATION.cff

@@ -20,6 +20,6 @@ authors:
     given-names: Stefan
     orcid: https://orcid.org/0000-0001-7214-8125
 title: CaosDB - Advanced User Tools
-version: 0.9.0
+version: 0.10.0
 doi: 10.3390/data4020083
-date-released: 2023-11-27
\ No newline at end of file
+date-released: 2024-04-24
\ No newline at end of file

README.md

@@ -52,7 +52,7 @@ Conduct](https://gitlab.com/caosdb/caosdb/-/blob/dev/CODE_OF_CONDUCT.md).
 
 * Copyright (C) 2018 Research Group Biomedical Physics, Max Planck Institute
   for Dynamics and Self-Organization Göttingen.
-* Copyright (C) 2020-2021 Indiscale GmbH <info@indiscale.com>
+* Copyright (C) 2020-2023 Indiscale GmbH <info@indiscale.com>
 
 All files in this repository are licensed under a [GNU Affero General Public
 License](LICENCE.md) (version 3 or later).

setup.py

@@ -46,7 +46,7 @@ from setuptools import find_packages, setup
 ########################################################################
 
 MAJOR = 0
-MINOR = 9
+MINOR = 10
 MICRO = 1
 PRE = ""  # e.g. rc0, alpha.1, 0.beta-23
 ISRELEASED = False
@@ -158,7 +158,7 @@ def setup_package():
         install_requires=["linkahead>=0.13.1",
                           "jsonref",
                           "jsonschema[format]>=4.4.0",
-                          "numpy>=1.17.3",
+                          "numpy>=1.24.0",
                           "openpyxl>=3.0.7",
                           "pandas>=1.2.0",
                           "xlrd>=2.0",

src/caosadvancedtools/table_converter.py

@@ -25,6 +25,7 @@ import re
 import sys
 
 import caosdb as db
+import numpy as np
 import pandas as pd
 
 
@@ -48,27 +49,25 @@ def generate_property_name(prop):
 
 
 def to_table(container):
-    """ creates a table from the records in a container """
+    """Create a table from the records in a container."""
 
     if len(container) == 0:
-        raise ValueError("Container is empty")
-    properties = set()
+        return pd.DataFrame()
+    rts = {p.name for p in container[0].parents}
 
+    data = []
     for rec in container:
-        properties.update([generate_property_name(p)
-                           for p in container[0].get_properties()])
-    df = pd.DataFrame(columns=list(properties))
-    rts = set([p.name for p in container[0].parents])
-
-    for ii, rec in enumerate(container):
-        if set([p.name for p in rec.parents]) != rts:
+        if {p.name for p in rec.parents} != rts:
             raise ValueError("Parents differ")
 
-        for p in rec.get_properties():
-
-            df.loc[ii, generate_property_name(p)] = p.value
+        row_dict = {}
+        for prop in rec.get_properties():
+            propname = generate_property_name(prop)
+            row_dict[propname] = prop.value
+        data.append(row_dict)
+    result = pd.DataFrame(data=data)
 
-    return df
+    return result
 
 
 def from_table(spreadsheet, recordtype):

src/doc/conf.py

@@ -27,9 +27,9 @@ copyright = '2023, IndiScale GmbH'
 author = 'Daniel Hornung'
 
 # The short X.Y version
-version = '0.9.1'
+version = '0.10.1'
 # The full version, including alpha/beta/rc tags
-release = '0.9.1-dev'
+release = '0.10.1-dev'
 
 
 # -- General configuration ---------------------------------------------------

unittests/test_table_converter.py

@@ -27,6 +27,7 @@ from tempfile import NamedTemporaryFile
 import caosdb as db
 import pandas as pd
 from caosdb.apiutils import compare_entities
+from numpy import nan
 
 from caosadvancedtools.table_converter import (from_table, from_tsv, to_table,
                                                to_tsv)
@@ -42,7 +43,8 @@ class TableTest(unittest.TestCase):
 
     def test_empty(self):
         c = db.Container()
-        self.assertRaises(ValueError, to_table, c)
+        df = to_table(c)
+        assert df.shape == (0, 0)
 
     def test_different_props(self):
         r1 = db.Record()
@@ -65,6 +67,36 @@ class TableTest(unittest.TestCase):
         c.extend([r1, r2])
         self.assertRaises(ValueError, to_table, c)
 
+    def test_list(self):
+        r1 = db.Record()
+        r1.add_parent("no1")
+        r1.add_property("p1", value=1)
+        r1.add_property("p3", value=23)
+        r1.add_property("p4", value=[1])
+        r2 = db.Record()
+        r2.add_parent("no1")
+        r2.add_property("p1")
+        r2.add_property("p2", value=[20, 21])
+        r2.add_property("p3", value=[30, 31])
+        r2.add_property("p4", value=[40.0, 41.0])
+        r3 = db.Record()
+        r3.add_parent("no1")
+        r3.add_property("p5", value=[50, 51])
+        c = db.Container()
+        c.extend([r1, r2, r3])
+        result = to_table(c)
+        # NaN is hard to compare, so we replace it by -999
+        # autopep8: off
+        assert result.replace(to_replace=nan, value=-999).to_dict() == {
+            'p1': {0: 1,    1: -999,         2: -999},  # noqa: E202
+            'p3': {0: 23,   1: [30, 31],     2: -999},  # noqa: E202
+            'p4': {0: [1],  1: [40.0, 41.0], 2: -999},  # noqa: E202
+            'p2': {0: -999, 1: [20, 21],     2: -999},  # noqa: E202
+            'p5': {0: -999, 1: -999,         2: [50, 51]}
+        }
+        # autopep8: on
+        assert list(result.dtypes) == [float, object, object, object, object]
+
 
 class FromTsvTest(unittest.TestCase):
     def test_basic(self):