From 938fb844c3820244a5ba8a0f955a0bea29b76971 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Henrik=20tom=20W=C3=B6rden?= <h.tomwoerden@indiscale.com>
Date: Fri, 28 Aug 2020 12:22:08 +0000
Subject: [PATCH] Introduce date converter

---
 CHANGELOG.md                            |   6 +++++-
 src/caosadvancedtools/table_importer.py |  10 +++++++++-
 unittests/date.xlsx                     | Bin 4895 -> 4957 bytes
 unittests/test_table_importer.py        |  13 +++++++++++++
 4 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index eb435792..0d7f77a9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,7 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Added ###
 
-* `send_mail` function in `caosdbadvanced.serverside.helper` module
+* `send_mail` function in `caosadvancedtools.serverside.helper` module
 - New class to collect possible problems with the data model
 - New class for checking and importing tables
 - Function to get a file path to a shared resource directory
@@ -19,6 +19,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed ###
 
+* The `caosadvancedtools.table_importer.date_converter` now actually returns
+  `datetime.date` instance. A new
+  `caosadvancedtools.table_importer.datetime_converter` replaces the old
+  `date_converter` and returns a `datetime.datetime` instance.
 - The suppression module is now a logging filter.
 - The WebUIHandler is now a python logging formatter.
 - instead of `get_entity`, type-specific functions are used in
diff --git a/src/caosadvancedtools/table_importer.py b/src/caosadvancedtools/table_importer.py
index 03b4ffbc..bc1c2452 100755
--- a/src/caosadvancedtools/table_importer.py
+++ b/src/caosadvancedtools/table_importer.py
@@ -71,7 +71,7 @@ def yes_no_converter(val):
             "Field should be 'Yes' or 'No', but is '{}'.".format(val))
 
 
-def date_converter(val, fmt="%Y-%m-%d"):
+def datetime_converter(val, fmt="%Y-%m-%d %H:%M:%S"):
     """ if the value is already a datetime, it is returned otherwise it
     converts it using format string
     """
@@ -82,6 +82,14 @@ def date_converter(val, fmt="%Y-%m-%d"):
         return datetime.strptime(val, fmt)
 
 
+def date_converter(val, fmt="%Y-%m-%d"):
+    """ if the value is already a datetime, it is returned otherwise it
+    converts it using format string
+    """
+
+    return datetime_converter(val, fmt=fmt).date()
+
+
 def win_path_list_converter(val):
     """
     checks whether the value looks like a list of windows paths and converts
diff --git a/unittests/date.xlsx b/unittests/date.xlsx
index c852cde999b0014aece605328a407481e0de24b5..3402631e9cf48f0a08bd4e6269fcd9a01ae18472 100644
GIT binary patch
delta 3399
zcmZu!cQl;e7M&q8%4nl^qKqD*w?qko5z(TI8X08t7Bze!N|b1WNE4z<f{b3G3qkY|
zF<P{!Aw-XO;<tWpC2zg&pS#vs``q=NefQnx+!u&R1f>Cz_#8cml#~=SVafiMk`YXN
zx?+7O_<$b$HVK-BZHDBHm~9ZsLPuC_IYw8TKCFf%_$e!GCvGzTIQUtv4(vy>o0F6B
zk+&kjEU%i_O+!qI9&&z5754;YfnIW`+$`vlO4rrzL8(Vk*$s#`=|1LpJ-V6e0vRWl
z$dpQnl1R&bSQ%^QItCSXCMrluu=-T@a#YRQ4$%C}BnIld84xE@mTHXja#?yc2<H$}
zU=zceK6(yzV%9BGMcvQgZWVX~v3?v!t+=RXK^AP_X{eZ#6Xek@==ub4J4JuxFmaQx
z^}Uu2&xEsPAWkjl)FK^L74z(8Not&EZb0m9<B1l$n4?4GcXDnsM9v4Pr+f&G1>Sj3
z0!KBYfjyb`;uM(kBXQrNE*<cG@zim6dk5C9u)iIxFPT4XoIWy^r*wz{H7Us_!fLZ~
zRx*w{wZHo^@bi&<_;{dwU(L*-O}_#Vzh3u}*R@`jka*p$iVJh@LRD+}Ktc3oKhWC*
zHb9b)_I?Xzb^?Py1R@aV-vfc1od}kJN&;BUch<uoXlA9!s32aiaP5=fTi0Wh?o4Nb
zhN`6GR8&@-j^3`jUh@jisPN~gEp;}!hqb}@FLec<7?tQG8_4s=`wt{b@!=Ny*TfD%
zaG6{lI;)i~_(fO2Yg?qIxx$1CN;!$9cIn?$kL8`o`fc<X*@yB=EXk$9jkCHv!9a@2
z>|(@8q_8;f4as$^faY|%G-h{>$ms`amosv?67nmFz|_KdMFK_M8?*U*>T1BTwHm?M
z;!SaWF_Y4Ie&D`U^_6;Icde+O?M&LaJwe~4F*)5)gO=<$pC<O@eiJ~HcC&j81OnYX
zCE;HJ&gwTj4PtGG*?~#-rxMKXrj5-TZO>!W$H|xKOfCGAi%>?G;VKvtg}IEZq(0N(
zkMfH&x3R}%I!J!+TSuGE`#VY4l>GYTgdo)B^AS?i2??&G60iVm^R*TN{^t(axmpcT
zjZ~?%7NW146TreEE-T=%ooHBhvo|%u$k8c1jf70!FtlB&7Z|}lh@;oR)4xq~BFd5Y
zl$(mWMeut~4~K?4dNUj->_f9DgXMfCO(W59WPtHO&}8^NQZ_M~x)}v|IWW$mybiUs
zwup<|u{drFUtKNln02k4X!G@x+SBN-8}#taxbd*lpSQ2BZRp;{V&V+gP_w${wO~hn
z$oU5>1`$T;Fn~BghK5gjzT5SRh|<NPAifIfC>zFcggWNdFf(hjzZ4%oqiYFkR&i;p
zUTyD)l{BWi##W<~h#}{Dg{B}3zrn$j{p9FQKd<=}Za%?9ia|o}efUd-C`#XsrgGLF
z*0H=rv@lwMx>V(IUs|ESVqux9{%(?T4ET_HLe?vhKLDZegeg-Uxxww2{_ZH$3smq8
zna9P@Zb?*ctv*i&t64KaGMCuefVP}og{%Q?HLIFF9Y(urZ9)<r0oy7xT4iE3Th#|-
z`N*Wv?_pu>YzN6mG*0l^G^E><+*OZHR8+~de@lbLuW8n2NZC}@;UR@;pl4zm3azfL
z>86PSo-V~0?~WE2+P)Bee*K|yn_IyOX4AVpp5$V1`wtt-`x$Dz+>SI-50-fyz3*x(
zEuei3oDn2>j(4@4^-s!~UxnP2i#`}t2p!*JZi>ki@mM`TE{m`0*&buLT#t+ihy7JR
zwZ2WJOP|E&!!cPl1Yx7b@y1h))s=$J66|B@Kv<*3_6$i#xwhr|$v0$a5s%Y%o_T&4
zrpJHJ!9;sDJOVY2O|zy9pI1=QaXBvmi;a?6^+c%5`5r1RZ#vco$eEv8;gC)prs}X-
zar<cUaQ$P&3Gr{`=eq~-4W3^6DaQY!{8RD&^HG;egL#S0tl0}D0JONbaZ^BNjxvc3
zGC@Tq4h#crxtG-TK=A>TMiN4Z8+*IA2sf*RI&FWg+2RybhIqc2%RAz-f+0jg^Fl-T
zR>DFho0{j2^Z`E|YF{I&JXo^#M#T+#n9rM49bGCZM1P1)^7uUOqj{IO1>r!kC4o%1
z3u-=V#1O$nrqRV_a|8C~R!#t6b3u-)g(acyr^&N`ootzEw)lmFnTBLM%YoQ0!;N~A
zvAle1U6sV*;Vb?F_4{gjK}%P6q8*NkkBEL(UZtp0-+Ede_J8kuxfwarKW*qTR0J^e
zA;?mMPL~cWXJ3En@WE>>7Z>#SQfWPM&gxl-e{haw*K8!ujTGF|pNC<&q9T6otGfm&
zT|6K#*JagvL)6p4oNKBWM2BcRNt*;H8QI<L#~2!F^-GeH*zG}91QJ}n$>O_z{zzyN
zxs!_o(Pa^5JKe<Sg>Yypif8i#aRV+ig+jT-4_b1@wdzb&3xf-14JFxav>0yk{NOCX
z@9igPws@O&H592bZ)ku8pkG!~@QS@gOkZpukU8jxfDYVGtp+-RN5fnsUbqy<C2ny-
zNbaQO86J728%4oIMZ&`E8K>Gm#U)pILIKIK)v1syWUVbvUld)4=huiPKt#*qF({Q4
zU(rfZ`vUn3{#kYQD7H!UgMQXnl}1FhK}KKR`KL0TU6cdAd|tixq`-xi$fo#H3DW0&
zd9U8=X~Z$Fq@fIp%uCBbS`cV_W+w7kptY69#K*Qa5iN14ctrtro!El5se`^tp~98E
zQz=fL<xvJo8gJIq9;RM51<=P-Zpl`R=~GX}8l4Rzz-r4>%B<)H|J_9B;VY2`rCK9t
zSpXpTap(?JdYuE2#eQp{5Zp|eIWZcGc^0pfStiWY;+?R9z87*5j+fU){mK(KWb|tu
zoC2@(NQC!`2SK=z$tLS1={EY#Zm==Mk8EyeJkK`2FiRfy6vZlFZ#qv9ZQnA{s{!0H
ztz(j0HxMx-ncQ4s*%mZca3&u@vYr(su3q%ZPFQ|nw>f&!98K((&Vg`i`~1H3y<XY0
zV|EDA|FgtHKSn#3@2j<%KgaDu-IE)t%md<ToGz}2^7J+!RSsH~1*pclz;v94_<*25
z^|0!?rIsf7?3oF`{;s;(P)-&T*-?mMV;yTE!h=rouq7hCF)nGaiPqcQOSfxDg_d^5
zCLMM0rfH?6xLOP`+up2Kvqxgvu}L^GKlo9ey|@u`>(`oc)t#xkRt-@LUt_yGimz+9
zu|OByArvIoW%R_i$T_$`f#t*;dg!<e#fB+U_Q2>?`zHWY<U-dr*zZ{4ioE1MWsR;u
zoO&3B#x4*kjll9N-p>6hDUzKbi6s<9lMy)UlUbpo>$<gLCf_=4k=aL5*nJX=I#<Jg
z8Rk+H&qHC>5{3P_tTiCcFEyD$Rmk%$GGop@qo+HR@^=03xo%d<9%J8vJo6EX`hJ9W
zVT)l(&yz5~g=N0AywaB)lF9%X)nWd6Lb}XtV|YmTwx575${i^^SmP*K+4J!a*{e)m
zJmw9xTbiL#;p9YXw)az)b(E4Ur1f3UpP0VJ$}wr4$k}_ceOrNS;`p5p;ZQdxG&5kW
zaRc|y5M=*F({zRy1mY$C(+ST!?-DgXka{+cYVIA!c3INbJA~2bjzbdpO<ulEjuD8#
z{7hO;-`!vlU??sgh<;#hvD^F3$#ISEshbJ*bxR<|S0CZ4M>`M+H!@QQv{cTkM$2+I
z#Jz#Muq)YBD7sSSAAj%?+HpySQCdrPj-I;a4nui^`4;)uPQcwZ@*Lu&q$prXE&nrd
z>Ro)qyx09s4yFaZ+Knw9VS$NpEk<R#j1Iw7yu$6D@Gq#i`zu4%4)kem38i^EW=(Xj
zOQ=9b9euNbO|!%5O}Ug=i;&Ls+<WLg>wraay0PH3h{3R<WnMDl-sf*QOFO`qF7NX;
z6zYqyhQVn2cwg=;s2Zo^KtBL;p;Uth_ZhkS&>pOdZB^hCr?P6QnoSv11Il&1>q$-V
zmZ8k$>`yq%ifcv+6iaJ~zq7K3_THzqn?BOun4Ft>@g7kAF$t-F1dm?8hDC4AHNCo(
z`gZrK!ZGx>8{<+E!ceD9YzFzwKA4Ce^xs#3p<{7)HFk>TBH8KF!0>EV#oEy_5V3%<
z{<K`&zk`3yZ#4flt+Ta>2wO$V$@LrbXRD+93$%=NfwP}&6c-@1zIXZ??x#Ya{tL4M
z#frl@&Om>54*I`9LS)!`Fh(qrmf}n|18_F57<LUVbe8s~z?lC^y9&czgR+YJZ{2@#
SLibmK8+Mx(0VRe14*myP<n6iu

delta 3366
zcmZ8kc{mhY7aztx#xlkb*^;sE+0&r13>ss3?S(|vv6K-q2E9ZR8oRMYlTo&mWn?Q$
zWDBXtzGUBpy!6TUeBV>={pa52-rs%hJ@@>6=bSqV&xM1{jDd7e0231v;FCV60t}-8
z9`}{BjEc0s2E^|`VVol-O96U!ysZ46hRm=JLA2CT3v#{5$4n%VtX<>ORE$XUAMU$`
z;0NN#qc6StA@{`(1XpV>koKh7np^)?Ue~cSdcrh0(-IXCYXi4*6?yF_JD#I!<|zZ|
zDijuZZpbYESWTBAt?wJ4cTR!((y2ENOH%@RH%MF$-*ee=IOB2(&HbO77AaZ6;wB1T
zp6%%y+zM+^<W^v5$&{~^)0O$87vndi+XT+#D(KF=IMM0Y(PFrNi4L8vyX~^+l1@4s
z8)#`&=6djUc~=#zQv6sr#YNheO@M=bmMgfatFN9aHP_Ob7N=<|IhSn3Mj8?m@MouD
z6}Cgl$!2Yu7C3Sr4cg49<UYihE5fazm}Q!xaL}GY*$O9SOFEd6;|22tyQo-mF}KDl
zX_ONPA3bHl(&Zb^K~7Jh3+gl2m#|Y{;Wc_C`TS<;2>Lzkmos+8zHr%gxfLlaS7n$9
zZ(T*!PkZX!&|c_hVO^HhAbvseS$zf-X&Mn9X=Utzi@*5_4qnf{a!M!dfm<Ze{r)GU
zCuE6=^-^c?%VG+k%o8Pja81t_xJMQ<+h74}AUvG^(8_bUnAbFcg-03~1UF9_1ZBD3
zC@reF)*-ec$e$H=g~3UwLEK1M>FNFa?@AxwcjuC;cRXH{p$I+S^mis3ELK$21r-O-
zUD7VN@2e%~E(tfNa@?ubWl^^aT7|gI{h7SME_P)1cH$uEr@LYiE({y9Kc*30wWb(O
zT$Nf|+BaTk4<ch48Vp8`232vGkt4_4(|E6<g^{sO!Jpn<Sk7-6^NL=NuOD{So?abF
z_K<Rj4q~=`6!>)x1p~fWwxZozJ)Ac@ikLBKG_;exe97T9X@JK4NV><Vq>^TBTWm}P
zU9k_=?A(u-TD7fm%XA8D<~gJ@GiGF}tHvr*XaN8*C;;#Wy@{3}X`FH3h&pS-T3O$G
z-@wqOXwS13S`-Tm7EVz1AMRNuz(nG?;>?UXeK(hQ3Uey7>M{fuqhx6(+S!F9nl=uE
zYFUHw2s$btgD7X-2xKDYRrbAWf~lC^w@o%=bz+ed>;j&7y!U79Zjq2FgA<U}%njx?
z1WQA^K4J}UW3Y+9ok7KEx~Haw@Mq`&r^7Ae3O4+L`DRUu%Jpgn%bJ_caWgwyH=oP%
zrhlxfWV~$(XBo&loG5!Uow*|U@o!YZ)a_Z14SjKg;9A3Td?qWx&9BKpq}j{2HqP{(
zzXPryEwsM~Hg2L9tL8NSEZ(a~Y8U+H3L_;Ew(UM+nU-t>#LY;5h7CD2LdP&d>knB8
zTAA(W`a6Pwx)ACMM!tUAU=^x^i9<B|o|F?Nq!-`e*LXJki7Oeu)z<iOLQRjxwnWZr
zT6>j80+GaVtMbP~Tm;mM)y`YlB^XHY5s@8vuB+#Q-`y=iN-retSTyK<q4hJgl`JXH
zY0vjv)^a(I1EmlxAZKB#vGj%u{j5<ftw*MNPsN?rAEcLY!Ho((?3Y(J?2NFNT<PKe
z9{nWk-s_8-)HLU)y?GNh?0nozrQas3t6p6tUct$ARJYr$|C+TuaI0SGU7?)e;9vPa
zzgoI4$b+rwU{TR~@S5(};hPHCCI$lo8!uDhShko+xaAo^_W99pa^$&GDxHnTU7%JX
zbWLN&=aycI%XXwr_qR-2)dZDQpi~T3-8tBnRFU>$Ed&=S%2U8<8^^KfT@>+%yUj6a
zZcKbt-P&PV^k${VlJA8F1p<nui?QtOoCRr|@zb&X{G8~kJ^|C?q_3a3`0uzBp9xab
zMra$B<Kp;s=TU5s^AFYc??e*ENfR?x%vX-onx|vm(iMzZ$-XoTvf5*YxDLK3TEfcu
zj=ZYA2zvE}Jn350sx0+>CbA7OFQ=wvUt;)B0R8b9Lb;y%R{6H6pSrphB5NqS!rpvs
zz4DIc1usP55rb^Up)zVt3&WUrW_V1~^V=m`GhC>YPFbt+P7w%F>U*!W<cS{t65p6l
z1|}~g`_ROL&!pa43(2egIpw>1;MZp{tb=jTJo`L)=813HGBRfN-E=B%39cQ}DO2N0
z&LfK4eU__)=xDoX+Ca5+!qQ@#_Y?fANZng=2cF2)a^pgZuDek=$Amn;2b%I~n~&b1
z5GVQ3$Tndd?b#*WgIJ|%Arg|dJo~!G%U=gqua9FI(r%7x_r<C-%MyA+yE1-k309p$
z+@9U`FR1ug8&|;ACi0hSIQWAHWbd06yZ!Z6Gy-K;g`1rFN&8Fr6W&<;0U!Xt!16EU
zqXGYGm6uq+CpnI&$}Eo4z0#t<O5dmbH1;)yc#AF>{m53hn|T5tqB3Aeu$a1a?rRPJ
z6jbzH9&uo-`qbX8g&Wjv0=reFBYf}tWvWf^T@9=#Q)v#o(NzzW`_3-_`*&?z71G<e
z^tLv#!?|sywpZFb>-D!sVLlZOXUpghTa;Z)<4-4tBz73ka&GE7e0(DsOPFf^ySH&)
zkTf~pg=$T*7P1ShYqlw6>fyJ|P8y#Q+Nm<W=ZHiQiU%%Y`|9>|cSGh4c4BTEq#c~t
z?7-~i-*jx956!=9Cc<gX;v7*?$GLmgI<euRb5;+B)BPUp+Lo461k9`Y>sAt`p=wn)
z48EDX+FTvFs-eC6ti$^nY>e$eN(#~l8r=eHl@_ted6f3?O`}-OWip+#6uqQIgjnUW
z)THKI3SQj8LqVcBq{r|}`ckqt$88hJEWbnsn53r>;F$3!KMMX7m$@OI3KY0E36`bF
zaC&$1`utLJ(rH<GA(@>aXbFA-$|9kg{{+qPfe33({yr6zs$r1|Vds-Wh6b3em7I>g
z?Z~)OlR`Zb{ZSPb=4-I?#PhFKylY2AQaZmA)0;pBf9q2BtL?j6A0u50{GdN3ON1^z
zlXxIL5PbeFYQAijfOF=thI@VH@~Z&6Ck)Q}eojI_*mz84?W@;KA`rNnVi$`wZZylt
zjeht9X&G(4<M3`rAf^3lqX9LihSH<I2p;Za2zs#}`lU!#vx)3gceqFQaMTd!r-+ov
zGv1xd{h59j_=|XYU2#3vj~y^_oOXUsIW!<*7p(%0g3KVF<6`p9q>G8Hu@dpG&ilt=
z>dI)fc*M(<uI}F6A78?~>ufAei@;YLQNGVIp%aDH!z-bis^`<Z%+ErvFi#6?&(PiX
z-Uo*gF6dkWQ{K^~g#_rZ@*|1oBe*G@RquOd3b@m6e6>KB+PxvBRno`2q%1vr6FC7b
zQX=7TGYK^sQ@H%j_qhlC2YxIW?Oy<@y!fV5^~q5N&I~^I@o(R4%VRhmq|yeS344kV
zef>{qp5w>AuvV4B9%}N_?Nt7{sv17tEzmXLq^g}o{~`SpaKR@9eR;|T=9l?9m;28z
z9&<3{-yHm}?uZfWLO4q;)LR9Pu><KdWfDQ}BRG)5_;#ig31y`XF-3yVi_!k0&>l%k
zD`v^q*p3%#4QZ$#)sZ5&ZPZr(JjrA}{cIjcw6HW_^JR^(e22GYC(Isnq50}cpm~f_
z8SmC*a9pUT%+2S?ZIXh1OHyukDrUILw>I`7x%C<QM8HW=Cb&v|iTz^cjunOkER{*T
zW2aOeIU8+1@-C85rqbp{wmKtWji&UKdO%vZdO@srXqapX;^1oY;-aBWak@+nR`jau
zt&797E`saSY-nOvEoj%l85sJ|h*v-zmDW#o#<a;^dtlpt<h5p|d_xk%LAH^MHp2+`
zO?pM-@eP++UiYV<<`2q!n%GOm9vuna0j)oGgaflaZ{@C6@b6BZKJwUZm}6_X?zh*#
znA;?R&1G-t&8`r{F)%i&ZDeRYwurwp5xFdMc$Jg#lf*Y=&m-L6nV>Wjo;-m4<$;9n
zv#Pd48cwd0s-1^Z)#12$N5=;%uk0TC#P>KRvi~mvp7f`VXM?@vv6uia7YxQrF4)K0
z%$SB23i$tD0R4Lz1^#2RDRF`GwD7M3eh;|3e_#|Dh|3W82?({mAJTukD0VCkzCS=n
z7NQZ$X<`$E^#u5PC_Ap*erv8Eq6_5A3FvpJasCDY0NhxKKAb`)px+6}@dt>Bjrg2X
eQ0Uj?Pu}eK9jxgA0Iokk3PeRNI78>j&i?>Q57pxU

diff --git a/unittests/test_table_importer.py b/unittests/test_table_importer.py
index 3d44b0dc..db40361c 100644
--- a/unittests/test_table_importer.py
+++ b/unittests/test_table_importer.py
@@ -21,12 +21,14 @@ import os
 import unittest
 from functools import partial
 from tempfile import NamedTemporaryFile
+import datetime
 
 import numpy as np
 import pandas as pd
 from caosadvancedtools.datainconsistency import DataInconsistencyError
 from caosadvancedtools.table_importer import (XLSImporter, assure_name_format,
                                               date_converter,
+                                              datetime_converter,
                                               win_path_converter,
                                               win_path_list_converter,
                                               yes_no_converter)
@@ -60,6 +62,17 @@ class ConverterTest(unittest.TestCase):
             r"\this\computer,\this\computer"),
                          ["/this/computer", "/this/computer"])
 
+    def test_datetime(self):
+        test_file = os.path.join(os.path.dirname(__file__), "date.xlsx")
+        self.importer = XLSImporter(converters={'d': datetime_converter,
+                                                }, obligatory_columns=['d'])
+
+        xls_file = pd.io.excel.ExcelFile(test_file)
+        df = xls_file.parse()
+        df = self.importer.read_xls(test_file)
+        assert df.shape[0] == 2
+        assert df.d.iloc[0] == datetime.datetime(1980,12,31,13,24,23)
+
     def test_date(self):
         test_file = os.path.join(os.path.dirname(__file__), "date.xlsx")
         self.importer = XLSImporter(converters={'a': date_converter,
-- 
GitLab