Merge branch 'dev' into f-send-mail

src/caosadvancedtools/table_importer.py

@@ -28,9 +28,12 @@ Those converters can also be used to apply checks on the entries.
 """
 
 import logging
+import pathlib
+from datetime import datetime
 
 import numpy as np
 import pandas as pd
+from caosadvancedtools.utils import check_win_path
 from xlrd import XLRDError
 
 from .datainconsistency import DataInconsistencyError
@@ -68,6 +71,30 @@ def yes_no_converter(val):
             "Field should be 'Yes' or 'No', but is '{}'.".format(val))
 
 
+def date_converter(val, fmt="%Y-%m-%d"):
+    """ if the value is already a datetime, it is returned otherwise it
+    converts it using format string
+    """
+
+    if isinstance(val, datetime):
+        return val
+    else:
+        return datetime.strptime(val, fmt)
+
+
+def win_path_converter(val):
+    """
+    checks whether the value looks like a windows path and converts it to posix
+    """
+
+    if not check_win_path(val):
+        raise ValueError(
+            "Field should be a Windows path, but is\n'{}'.".format(val))
+    path = pathlib.PureWindowsPath(val)
+
+    return path.as_posix()
+
+
 class TSVImporter(object):
     def __init__(self, converters, obligatory_columns=[], unique_columns=[]):
         raise NotImplementedError()
@@ -128,15 +155,6 @@ class XLSImporter(object):
                        'category': "inconsistency"})
             raise DataInconsistencyError(*e.args)
 
-        try:
-            df = xls_file.parse(converters=self.converters)
-        except Exception as e:
-            logger.warning(
-                "Cannot parse {}.".format(filename),
-                extra={'identifier': str(filename),
-                       'category': "inconsistency"})
-            raise DataInconsistencyError(*e.args)
-
         self.check_columns(df, filename=filename)
         df = self.check_missing(df, filename=filename)
 

src/caosadvancedtools/utils.py

@@ -140,23 +140,6 @@ def check_win_path(path, filename=None):
     return True
 
 
-def treat_win_path(path, relative_to, filename=None):
-    """
-    check win path and convert it to posix and make it absolute.
-    Parameters:
-    path: windows path as string
-    relative_to: unix path as string
-    filename: if the path is located in a file, this parameter can be used to
-              direct the user to the file where the path is located.
-    """
-    check_win_path(path, filename=filename)
-    path = pathlib.PureWindowsPath(path)
-    path = path.as_posix()
-    path = assure_absolute_path_in_glob(str(path), prefix=relative_to)
-
-    return path
-
-
 def return_field_or_property(value, prop=None):
     """
     returns value itself of a property.

unittests/test_table_importer.py

@@ -17,13 +17,17 @@
 # along with this program. If not, see <https://www.gnu.org/licenses/>.
 
 
+import os
 import unittest
+from functools import partial
 from tempfile import NamedTemporaryFile
 
 import numpy as np
 import pandas as pd
 from caosadvancedtools.datainconsistency import DataInconsistencyError
 from caosadvancedtools.table_importer import (XLSImporter, assure_name_format,
+                                              date_converter,
+                                              win_path_converter,
                                               yes_no_converter)
 
 
@@ -45,6 +49,25 @@ class ConverterTest(unittest.TestCase):
                          "Müstermann, Max")
         self.assertRaises(ValueError, assure_name_format, "Max Mustermann")
 
+    def test_winpath(self):
+        self.assertRaises(ValueError, win_path_converter, "/hallo/python")
+        self.assertEqual(win_path_converter(r"\this\computer"),
+                         "/this/computer")
+
+    def test_date(self):
+        test_file = os.path.join(os.path.dirname(__file__), "date.xlsx")
+        self.importer = XLSImporter(converters={'a': date_converter,
+                                                'b': date_converter,
+                                                'c': partial(date_converter,
+                                                             fmt="%d.%m.%y")
+                                                }, obligatory_columns=['a'])
+
+        xls_file = pd.io.excel.ExcelFile(test_file)
+        df = xls_file.parse()
+        df = self.importer.read_xls(test_file)
+        assert df.shape[0] == 2
+        assert df.a.iloc[0] == df.b.iloc[0] == df.c.iloc[0]
+
 
 class XLSImporterTest(unittest.TestCase):
     def setUp(self):

unittests/test_utils.py

@@ -24,7 +24,7 @@ import unittest
 
 from caosadvancedtools.utils import (assure_absolute_path_in_glob,
                                      check_win_path, string_to_person,
-                                     treat_win_path)
+                                     )
 
 
 class Assure_absoluteTest(unittest.TestCase):
@@ -63,5 +63,3 @@ class PathTest(unittest.TestCase):
         assert check_win_path(r"C:\hallo")
         assert check_win_path(r"\hallo")
         assert not check_win_path("/hallo")
-        self.assertEqual(treat_win_path(r"tag\hallo", "/lol"),
-                         "/lol/tag/hallo")