Merge branch 'resulttable' into 'dev'

Resulttable

See merge request !6

CHANGELOG.md

@@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added ###
 
+- CFood that creates a Record for each line in a csv file
 - `generic_analysis.py` allows to easily call scripts to perform analyses in
   server side scripting [EXPERIMENTAL]
 

integrationtests/crawl.py

@@ -34,7 +34,7 @@ from caosadvancedtools.crawler import FileCrawler
 from caosadvancedtools.guard import INSERT, UPDATE
 from caosadvancedtools.scifolder import (AnalysisCFood, ExperimentCFood,
                                          PublicationCFood, SimulationCFood,
-                                         SoftwareCFood)
+                                         SoftwareCFood, ResultTableCFood)
 
 from example_hdf5cfood import ExampleH5CFood
 
@@ -91,6 +91,7 @@ if __name__ == "__main__":
                     interactive=False, hideKnown=False,
                     cfood_types=[ExperimentCFood, AnalysisCFood, SoftwareCFood,
                                  PublicationCFood, SimulationCFood,
+                                 ResultTableCFood,
                                  ExampleH5CFood
                                  ])
 

integrationtests/extroot/ExperimentalData/2010_TestProject/2019-02-03/result_table_DepthTest.csv

+temperature [°C] ,depth 
+234.4,3.0
+344.6,5.1

integrationtests/model.yml

@@ -19,6 +19,14 @@ SoftwareVersion:
     binaries:
     sourceCode:
     Software:
+DepthTest:
+  obligatory_properties:
+    temperature:
+      datatype: DOUBLE 
+      description: 'temp'
+    depth:
+      datatype: DOUBLE 
+      description: 'temp'
 Person:
   obligatory_properties:
     firstName:

integrationtests/test_crawler_with_cfoods.py

@@ -66,6 +66,17 @@ class CrawlerTest(unittest.TestCase):
                          datfile.description)
         assert os.path.basename(datfile.path) == "datafile.dat"
 
+        # There should be two DepthTest Properties
+        depthtests = exp.get_property("DepthTest")
+        assert depthtests is not None
+        assert len(depthtests.value) == 2
+        depthtest = db.Record(id=depthtests.value[0])
+        depthtest.retrieve()
+        assert "DepthTest" in [p.name for p in depthtest.get_parents()]
+        assert 234.4 == depthtest.get_property("temperature").value
+        assert "°C" == depthtest.get_property("temperature").unit
+        assert 3.0 == depthtest.get_property("depth").value
+
         # Should have a responsible person
         self.assertIsNotNone(exp.get_property("responsible"))
         person = db.Record(id=exp.get_property("responsible").value[0])

src/caosadvancedtools/scifolder/__init__.py

@@ -3,3 +3,4 @@ from .experiment_cfood import ExperimentCFood
 from .publication_cfood import PublicationCFood
 from .simulation_cfood import SimulationCFood
 from .software_cfood import SoftwareCFood
+from .result_table_cfood import ResultTableCFood

src/caosadvancedtools/scifolder/result_table_cfood.py

+#!/usr/bin/env python
+# encoding: utf-8
+#
+# Copyright (C) 2019 Henrik tom Wörden
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+
+import re
+
+import caosdb as db
+import pandas as pd
+from caosadvancedtools.cfood import (AbstractFileCFood, assure_has_description,
+                                     assure_has_parent, assure_has_property,
+                                     assure_object_is_in_list, get_entity)
+from caosadvancedtools.read_md_header import get_header
+
+from ..cfood import assure_property_is, fileguide
+from .experiment_cfood import ExperimentCFood
+from .generic_pattern import date_pattern, date_suffix_pattern, project_pattern
+from .utils import parse_responsibles, reference_records_corresponding_to_files
+from .withreadme import DATAMODEL as dm
+from .withreadme import RESULTS, REVISIONOF, SCRIPTS, WithREADME, get_glob
+
+
+# TODO similarities with TableCrawler
+class ResultTableCFood(AbstractFileCFood):
+
+    # win_paths can be used to define fields that will contain windows style
+    # path instead of the default unix ones. Possible fields are:
+    # ["results", "revisionOf"]
+    win_paths = []
+    table_re = r"result_table_(?P<recordtype>.*).csv$"
+    property_name_re = re.compile(r"^(?P<pname>.+?)\s*(\[\s?(?P<unit>.*?)\s?\] *)?$")
+
+    @staticmethod
+    def name_beautifier(x): return x
+
+    def __init__(self,  *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.table = pd.read_csv(fileguide.access(self.crawled_path))
+
+    @staticmethod
+    def get_re():
+        return (".*/ExperimentalData/"+project_pattern + date_pattern +
+                date_suffix_pattern + ResultTableCFood.table_re)
+
+    def create_identifiables(self):
+        self.recs = []
+        self.experiment, self.project = (
+            ExperimentCFood.create_identifiable_experiment(self.match))
+
+        for idx, row in self.table.iterrows():
+            rec = db.Record()
+            rec.add_parent(self.match.group("recordtype"))
+
+            for col in self.table.columns[:2]:
+                match = re.match(ResultTableCFood.property_name_re, col)
+
+                if match.group("unit"):
+                    rec.add_property(match.group("pname"), row.loc[col], unit=match.group("unit"))
+                else:
+                    rec.add_property(match.group("pname"), row.loc[col])
+            self.identifiables.append(rec)
+            self.recs.append(rec)
+
+        self.identifiables.extend([self.project, self.experiment])
+
+    def update_identifiables(self):
+        for ii, (idx, row) in enumerate(self.table.iterrows()):
+            for col in row.index:
+                match = re.match(ResultTableCFood.property_name_re, col)
+                assure_property_is(self.recs[ii], match.group("pname"), row.loc[col], to_be_updated=self.to_be_updated)
+        assure_property_is(self.experiment, self.match.group("recordtype"),
+                           self.recs, to_be_updated=self.to_be_updated,
+                           datatype=db.LIST(self.match.group("recordtype")))

unittests/test.csv

+temperature [°C] ,depth 
+234.4,3.0
+344.6,5.1

unittests/test_result_table_cfood.py

+#!/usr/bin/env python3
+# encoding: utf-8
+#
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2018 Research Group Biomedical Physics,
+# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+
+"""
+test module for ResultTableCFood
+"""
+
+
+import os
+import re
+import unittest
+
+import caosdb as db
+from caosadvancedtools.scifolder.result_table_cfood import ResultTableCFood
+
+
+class CFoodTest(unittest.TestCase):
+    def test_re(self):
+        self.assertIsNotNone(re.match(ResultTableCFood.table_re, "result_table_Hallo.csv"))
+        self.assertEqual(re.match(ResultTableCFood.table_re, "result_table_Hallo.csv").group("recordtype"),
+                         "Hallo")
+        self.assertIsNotNone(re.match(ResultTableCFood.table_re,
+                                      "result_table_Cool RecordType.csv"))
+        self.assertEqual(re.match(ResultTableCFood.table_re, "result_table_Cool RecordType.csv").group("recordtype"),
+                         "Cool RecordType")
+        self.assertIsNone(re.match(ResultTableCFood.table_re, "result_tableCool RecordType.csv"))
+
+        self.assertIsNotNone(re.match(ResultTableCFood.property_name_re,
+                                      "temperature [C]"))
+        self.assertEqual(re.match(ResultTableCFood.property_name_re,
+                                  "temperature [C]").group("pname"),
+                         "temperature")
+        self.assertEqual(re.match(ResultTableCFood.property_name_re,
+                                  "temperature [C]").group("unit"), "C")
+        self.assertEqual(re.match(ResultTableCFood.property_name_re,
+                                  "temperature [ C ]").group("unit"), "C")
+        self.assertEqual(re.match(ResultTableCFood.property_name_re,
+                                  "temperature").group("pname"), "temperature")
+
+    def test_ident(self):
+        rtc = ResultTableCFood(os.path.join(os.path.dirname(__file__), "test.csv"))
+        rtc.match = re.match(ResultTableCFood.get_re(),
+                             "/ExperimentalData/2010_TestProject/2019-02-03_something/result_table_RT.csv")
+        rtc.create_identifiables()
+        rtc.update_identifiables()