Merge branch 'dev' into f-convert-xlsx-to-json

.gitlab-ci.yml

@@ -163,18 +163,17 @@ unittest_py312:
   script: *python_test_script
 
 unittest_py313:
-  allow_failure: true
   tags: [docker]
   stage: unittest
   image: python:3.13-rc
   script:
     # TODO: Replace by '*python_test_script' as soon as 3.13 has been officially released.
-    - apt update && apt install -y cargo
-    - pip install meson[ninja] meson-python
-    - pip install pynose pandas pytest pytest-cov gitignore-parser openpyxl>=3.0.7 xlrd==1.2 h5py
-    - pip install git+https://gitlab.indiscale.com/caosdb/src/caosdb-pylib.git@dev
-    - pip install .
-    - pytest --cov=caosadvancedtools unittests
+    - apt update && apt install -y cargo || true
+    - pip install meson[ninja] meson-python || true
+    - pip install pynose pandas pytest pytest-cov gitignore-parser openpyxl>=3.0.7 xlrd==1.2 h5py || true
+    - pip install git+https://gitlab.indiscale.com/caosdb/src/caosdb-pylib.git@dev || true
+    - pip install . || true
+    - pytest --cov=caosadvancedtools unittests || true
 
 # Build the sphinx documentation and make it ready for deployment by Gitlab Pages
 # Special job for serving a static website. See https://docs.gitlab.com/ee/ci/yaml/README.html#pages

CHANGELOG.md

@@ -16,6 +16,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed ###
 
+* `table_converter.to_table` now returns an empty DataFrame instead of raising a
+  ValueError when called with an empty container.
+
 ### Deprecated ###
 
 ### Removed ###
@@ -27,6 +30,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Fixed ###
 
 - Json schema exporter handles reference properties better.
+- [#59](https://gitlab.com/linkahead/linkahead-advanced-user-tools/-/issues/59) `to_table` failed on lists as values.
 
 ### Security ###
 

setup.py

@@ -158,7 +158,7 @@ def setup_package():
         install_requires=["linkahead>=0.13.1",
                           "jsonref",
                           "jsonschema[format]>=4.4.0",
-                          "numpy>=1.17.3",
+                          "numpy>=1.24.0",
                           "openpyxl>=3.0.7",
                           "pandas>=1.2.0",
                           "xlrd>=2.0",

src/caosadvancedtools/table_converter.py

@@ -25,6 +25,7 @@ import re
 import sys
 
 import caosdb as db
+import numpy as np
 import pandas as pd
 
 
@@ -48,27 +49,25 @@ def generate_property_name(prop):
 
 
 def to_table(container):
-    """ creates a table from the records in a container """
+    """Create a table from the records in a container."""
 
     if len(container) == 0:
-        raise ValueError("Container is empty")
-    properties = set()
+        return pd.DataFrame()
+    rts = {p.name for p in container[0].parents}
 
+    data = []
     for rec in container:
-        properties.update([generate_property_name(p)
-                           for p in container[0].get_properties()])
-    df = pd.DataFrame(columns=list(properties))
-    rts = set([p.name for p in container[0].parents])
-
-    for ii, rec in enumerate(container):
-        if set([p.name for p in rec.parents]) != rts:
+        if {p.name for p in rec.parents} != rts:
             raise ValueError("Parents differ")
 
-        for p in rec.get_properties():
-
-            df.loc[ii, generate_property_name(p)] = p.value
+        row_dict = {}
+        for prop in rec.get_properties():
+            propname = generate_property_name(prop)
+            row_dict[propname] = prop.value
+        data.append(row_dict)
+    result = pd.DataFrame(data=data)
 
-    return df
+    return result
 
 
 def from_table(spreadsheet, recordtype):

unittests/test_table_converter.py

@@ -27,6 +27,7 @@ from tempfile import NamedTemporaryFile
 import caosdb as db
 import pandas as pd
 from caosdb.apiutils import compare_entities
+from numpy import nan
 
 from caosadvancedtools.table_converter import (from_table, from_tsv, to_table,
                                                to_tsv)
@@ -42,7 +43,8 @@ class TableTest(unittest.TestCase):
 
     def test_empty(self):
         c = db.Container()
-        self.assertRaises(ValueError, to_table, c)
+        df = to_table(c)
+        assert df.shape == (0, 0)
 
     def test_different_props(self):
         r1 = db.Record()
@@ -65,6 +67,36 @@ class TableTest(unittest.TestCase):
         c.extend([r1, r2])
         self.assertRaises(ValueError, to_table, c)
 
+    def test_list(self):
+        r1 = db.Record()
+        r1.add_parent("no1")
+        r1.add_property("p1", value=1)
+        r1.add_property("p3", value=23)
+        r1.add_property("p4", value=[1])
+        r2 = db.Record()
+        r2.add_parent("no1")
+        r2.add_property("p1")
+        r2.add_property("p2", value=[20, 21])
+        r2.add_property("p3", value=[30, 31])
+        r2.add_property("p4", value=[40.0, 41.0])
+        r3 = db.Record()
+        r3.add_parent("no1")
+        r3.add_property("p5", value=[50, 51])
+        c = db.Container()
+        c.extend([r1, r2, r3])
+        result = to_table(c)
+        # NaN is hard to compare, so we replace it by -999
+        # autopep8: off
+        assert result.replace(to_replace=nan, value=-999).to_dict() == {
+            'p1': {0: 1,    1: -999,         2: -999},  # noqa: E202
+            'p3': {0: 23,   1: [30, 31],     2: -999},  # noqa: E202
+            'p4': {0: [1],  1: [40.0, 41.0], 2: -999},  # noqa: E202
+            'p2': {0: -999, 1: [20, 21],     2: -999},  # noqa: E202
+            'p5': {0: -999, 1: -999,         2: [50, 51]}
+        }
+        # autopep8: on
+        assert list(result.dtypes) == [float, object, object, object, object]
+
 
 class FromTsvTest(unittest.TestCase):
     def test_basic(self):