From 2bdd72866513ec86f97c9a343aa650041207711d Mon Sep 17 00:00:00 2001
From: Daniel <d.hornung@indiscale.com>
Date: Mon, 11 Mar 2024 12:25:38 +0100
Subject: [PATCH] WIP: Deferring foreign keys and inserting deeper refs into
context.
---
.../table_json_conversion/fill_xlsx.py | 30 +++++++++++++++---
.../data/indirect_data.xlsx | Bin 0 -> 5945 bytes
.../table_json_conversion/test_fill_xlsx.py | 10 ++++--
3 files changed, 33 insertions(+), 7 deletions(-)
create mode 100644 unittests/table_json_conversion/data/indirect_data.xlsx
diff --git a/src/caosadvancedtools/table_json_conversion/fill_xlsx.py b/src/caosadvancedtools/table_json_conversion/fill_xlsx.py
index 62aa922a..63c14af7 100644
--- a/src/caosadvancedtools/table_json_conversion/fill_xlsx.py
+++ b/src/caosadvancedtools/table_json_conversion/fill_xlsx.py
@@ -262,6 +262,8 @@ out: union[dict, None]
context = TemplateFiller.Context()
context.fill_from_data(data)
+ explode_later: List[dict] = []
+
insertables: Dict[str, Any] = {}
for name, content in data.items():
path = current_path + [name]
@@ -275,16 +277,23 @@ out: union[dict, None]
if isinstance(content[0], dict):
# An array of objects: must go into exploded sheet
for entry in content:
- self._handle_data(data=entry, current_path=path, context=next_context)
+ explode_later.append({"data": entry, "current_path": path,
+ "context": next_context})
continue
elif isinstance(content, dict):
if not current_path: # Special handling for top level
- self._handle_data(content, current_path=path, context=next_context)
+ explode_later.append({"data": content, "current_path": path,
+ "context": next_context})
continue
insert = self._handle_data(content, current_path=path, context=next_context.copy(),
only_collect_insertables=True)
assert isinstance(insert, dict)
assert not any(key in insertables for key in insert)
+
+ # print(insert)
+ # from IPython import embed
+ # embed()
+
insertables.update(insert)
continue
else: # scalars
@@ -301,8 +310,9 @@ out: union[dict, None]
insertables[path_str] = value
if only_collect_insertables:
return insertables
- if not current_path:
- return None
+ # if not current_path:
+ # print("returning early")
+ # return None
# actual data insertion
insert_row = None
@@ -318,11 +328,21 @@ out: union[dict, None]
sheet.cell(row=insert_row+1, column=col_index+1, value=value)
+ # continue in exploded sheets
+ for exploded_content in explode_later:
+ print(f"Exploding: {exploded_content}")
+ self._handle_data(**exploded_content)
+
# Insert foreign keys
if insert_row is not None and sheet is not None and _is_exploded_sheet(sheet):
foreigns = _get_foreign_key_columns(sheet)
for index, path in ((f.index, f.path) for f in foreigns.values()):
- value = context[path]
+ try:
+ value = context[path]
+ except KeyError as err:
+ print(f"\nCurrent context: {context._props}")
+ from IPython import embed
+ embed()
sheet.cell(row=insert_row+1, column=index+1, value=value)
return None
diff --git a/unittests/table_json_conversion/data/indirect_data.xlsx b/unittests/table_json_conversion/data/indirect_data.xlsx
new file mode 100644
index 0000000000000000000000000000000000000000..45ed38b8222d105f1958cef6a41c12d42a49c883
GIT binary patch
literal 5945
zcmZ`-1z1y$`W_(CUD6FAJvyaB8tF+$_vn<A?(UEn2na)^1f(13Q0bJC*1-VbPW|8O
zPyY9A&pGGWv%T-W^F8ku=haj}xkC&90MG%<77hkz(rV@*h)=H(2R`Dk1Y2mjfnD6W
zpSrkk`8YYMB}iZ)^Ww|h47t8<l<t%<mm1fueS_yu$(>{=`$i_PO{6o?nFf4MVHwlr
z!CD0_XM|4TIztK`fP^fjLEX6AnJUv3V&MxA+e#jd8&gs?tD2^Hi2=#}qIQj&pit<B
zOH=Km{$=;Iixq1(gdP>=#?5J|Vp`yk<$DJ0?2(wAiRNG7?#|kT9U{VHM+N{0{u!<X
z*v;x^ydx=ED&0Kz{p~Ru@86frU=$6<T4-Q~qVzT<Ei{;=Mr@QWHVRdPBDc|wc)f3~
z4!m8eab?a33N2`PQhGi3nnziSx^t+bLjg(`QR)yN>oezr%WdLK8*wSk3t4LX-Abbn
zS#5>nPz-~x@BvGQx^V)M@5gaMew~qHpGJ5}ki<N;*^szJctq5&%U?UJVrH3zO+XHj
zbHau7Z+b?l&9*kOzNSz_6j9}fWO}b0!M>dXp&)}oCEs|s{+m#Bq>o>L@Os}*UmH|s
zbpt>;YMt-|{bG~omdmrj?~%*8pT74U>^~H2LmwgyTnxWL{yRI_Px2V??*agMtN;KJ
zB0D~g+#a@8PFBCqyg&1^Zv=9m6DAHkgpT{zz?`u91Vb9?h}`YG=IgS<_SI>_2@K*L
zeJzv2g4z`Y@rN^T_EXfFbH(p)>|Bn@D{gHI`<@<fnQuo#rJeYgedeB@ZE56QTG0nG
z5lHipmqU*_Fd?cY^+tApZsCAIBEs&4&{%X8AL8NJCo3QdZpPW&y7@)DFgwShdy1B3
z0VW)_QsX*WMqpFAx=mWXbCO}5c&id5B7JF=cTsgGaL3-Oxg|-fu^mxrV2Cphr2$>g
zD7bt>-6(2Rr>m#K-sYj6L;u92QgG;CKpJYZGOT2Z1RaPYCYo<B>h%oPvtJlDH0QzZ
zM6t4d>?7PTJQ?dkb!SDZrJyVSn`3*aG)7&C-MHl^*j!C=S!MiVjj6MM<-J_nJj{5I
z#*~Bi;#5L(U-h^TPTR3YWOq$;U)8wBo*fJ8j~a*49a38vh4no@LX!GTsVO>ufs5Dd
zhe?zS6SfP_G>XsEEsN4mjtmY&q6x11dhg8IO_FEuE4bw+q2dCBdL~pgpFyP?Vph6R
zCZ8nDj<MBiQlRGygnU;FrSDyU{`j8RGFZyzmmeTH>W~&>SA|r*-;z6)dgZ>oQCOb(
zQB7vlrIiEY_>vLyG?LX>jL1!t%0}kd7z~ySVFr#J!ZG*M#cgQyGHQ*-#x@G#+!$LH
zVFm#(4Mk_hj};wn89t^dFv%AVa5irc2-hp)0Er%==SO7OYf>SXV(?}ZMV#05y&_m4
z3TSLYA6Sl$I;#TbNaz<CD9?qR<10|C<+*=xpK!cHgD6bEbXylnG7U(OGVd+(R|;_S
z&ibTx99QRwQxg%6q+#8aj>u$UA*>JyR8u0lW{1pO=S@i+rExP|JW5Wqd&^kv!7{SV
zN71POrSy!nofP-GMuq1IU{WPhQX|_?zEz+Pvuin`fiflJRCf`u%Sf7&3sSh%lZG_7
zR)m=u5%x{^s87}rTp7L+NR>1j_&Sp8IXat+OORqBg!t`fY!X2`O@#2Hoc&@~gF8P8
zOuCgjw?l`me>4R~%<wsKB%CQulRetNV<~^Q1LQ|5$uY!uAEswtSQ*JL`H}{RiF*8w
zMsm~72U94cuT`@q)F0_)Vos>5>O8bVB#0|u408DHn?%uMqYz_&P3KfBXi32t)N@(G
zJNJ>F>%N7)+sHt_4m6eRlSpC5*B`c$1;_grpP9m-Ifk5!q)93vZ>5=<u40Fd9`s;r
zTY{J|hIOW_gklH_WGJ0?9)Ew7{&5rf?tYUvXk!Oj<RQs11{XKosvE)LlWlZU*=<N`
zt7j2@^zHQr7j>FpY%v@06zT@=7t{imFk`O`MhMU2OxY$1nKl%s=KKXw;exE>cbHS!
zN#JhyPOlHen#kun?*g*)owhzrOk8etYSoq;if2&wN1YVY9~4DojdjaL`udw@*`)Vr
z@)J8bzGC7~Xf$j3c+5e(Vrj<t0_{0e3;9G_*Nhj`hr*sz{7f}Ye%oa$^NH#DM9TVn
znN__rl+Bymi9uVW%Pxz^rfO)(>UF0s^_VW`q_)joqK-s;?Aow;pb(otofL~u$s&!c
z>{0P4`Qx@NcI6&j;dXhx2ExpHt?OU<UDY2q5qdS?Ap_sFh*bDl@!FJ~EOTOGaDbu=
z3D?w&_h+_p^J@GUye=}mJIzjhxC|=dZHjG~elwP>34|@KUpS|VDXHf9_7Wg@$RjhR
zQ{5#c18!&@q$J%XLK;aMlsiHWwJf6w&bq*Qe}esvjbyZ1QY1*8<9UR-$s{u&aZD{X
zDm|Io`r%=!y?5Nl3yu({Fdd-f4B3!ubf*NLo9rhY=F^%v7Oe33HJy<biZ&KAm2N-h
zd`{Mzd6Q|0Ix-1`(~|5JnzxbexqeMu@~>_TW@+HSWf7No(Ipg<FGL)&rGcFRF)_ge
zRpxEYPz^Y9o>-m#aR^=R)$DvJY;gtq*>sf=#FCxhp$ghF+LXH%aiDMfrisMn)n%`%
z`{sDDeS-~7v5TyQmm}qKDEMY#Ux~zHf~9AqHXqJU<JiJ*_^FanO?a~#1WPv5F~t|I
z4{>U{dJ<OmW80Db9w1%FAqtX+(Rz&r0NndMK)k_j4(_&ARvzx$zb?O~NOd|GJja9I
z*B+CO7Yu)FRw=`z)KE_d(t1$^vj+}_8`Q6V5L7waw_&9((f&Ye2d0x;K0Oaa8-=GQ
zK_8%vZ^yrtscwC;cS2*4zR?j)5lx8sHWgzqT2RHet1?Y$(x`{LAfX@Y0|sYn2h>80
z{Cw>dIj0$GvZ%ZI#vu-mR&{H%H%8C5GzZF#5YrT*uhI`Q#q9K&+?ClPWwf;DFg_1~
zG!9B2BT%9?t(Vg?d~cW(=}q?6c#VB6K=?dLb)Ixz_Q=9IvIKxuSB?_z50uzj1KXN{
zDnH{4C*Ma$^)IRz)n7uxgCr&MO}&0e^I|g_)Go)P8&sTE#zrjQGc&-#hsSIKK?MhV
zAT3wJWCc5^xL9Zv(h{g4o5o-cita(nMY{Vq5=C-J8@Bv8)$%A_<93zrCw8ALD$+Vw
z*jWhM2~cKQ)7CWZH7OF{>dDCxe*$==O-T+_bUQUlihm<$J~3rtQ%MSl>Smsx8x$Kb
zq}_0&D`jt|qq;yhhNIcl@NlXlzT`i=cX)B{FoHLt<{6sB#?V6*uaaqB{pE$CpsTCz
zYzOaExx)AKT{cZcMcs;iJod?To#5rV%hyaCNtMJ&7L9FpN+ru+8B-^vC>2dkLNVD!
zb7j-Mow4lR`hFsP6T-eax%!$uhGlz}+Lf)UlA@DxmHD|AgYaxMnxxTiTk+A1%CdSN
zw_Jg35&rWJc|0_nBSBJvQ)Py>TC?t}_v^ABj70IEP?W0iTxR26ys0gLP{KQD|JK}T
z=#FnKLUY}i|5J0opPLILL+hvJ5Sv|kX4_NDI(gkFIeJ5OPs(D)nSuwYR6X{DHVq-L
z`NKS^yXk6V`{6yXA2)56s|H(#Vb65gf?2@6dM}eR?TweTE7yqK%0!F9koEOszNoHi
z)>J%Tt}P|pB;WUJ`N(h6dW77blRMZ!Mwx$%Y@PB_t$+_6h?e~+zE$Orhj(jFCl`a_
zoEg+jv`xFIx)vqKg)vs7`Gi=m%YGxC8(dNp4Mh%g`&3clZT`Rz^Vt3_-r&GqP8i*D
z14^`rL{3$@;W&b_WB<b)0T9LN@nxq(rG39qhH9sr<(T-y&H$w!QwYhtr5~5qSuJ#H
z=)9NV@eH@JZ;F%_Nn4dwW<X*OB}BgMtP_&=kzT}qp0TK2zWyf7j`UpnYt*aaYri!b
z>(S}w{-<3$L6d}AA~Bvk@ayw8=K@#(l@pU6CVZgp-0g}FPmflIg<>ryO@N-t1uX0R
z#|y>iAogNG%$R$W84{Gk1>H`9CJXbz&1;3s>?%ou3H!`bu$0>OO!mwo(w`fPzBbD3
z#F#3zn)tjvAm%`$9M=e^%>Mkc*~4A)+>o7e2A1+NeIsnQL&0-AVRxG3Fqph~Z8^jc
zFP^=?=_B_8Qo>PUpBWv`O_xr8R501RopgBRLASeN!|BdvJG*sZ<(hBCNyf)q12yMC
zA<q`2+MIDdLo@-U&XioRC@x2r8x2X%K{7Uc*^DTXiE~Lbxg>nf0Yas?#psA0h2uQh
z`OqH8Fdcpjm{>ZWLL0Iu?e#TZ78p0{(7Dn{RkjYtFF!=1v0rtxCW5tU(d5uW#4TC_
zNG_O-o!Etzk7u3wwrcf=KlM6N^~stjk>B*YEZ6~is)Nb>luk_D#Zep@Tf;`x?^3aE
zE-*7?EF5&sL_S%C@*_315z5fsEqiExF}s5Ix5`EBTd~{_Dz8AO9Orjub@%Xfv~vF`
z?L|Ffhj||2AMAY(%^sOsuO;6Rm3>hgFLPO|&zwRXBKgFSYcTMK2Xnu&(c<KHSKQu7
z3DK=UOz)YN)Qv>6QDZM1?X$hBl_zKMNr6EtF)&lr{(40U3lh&j2hgN<#^YHOW#u<n
zBaez-j>D}XX!W`^V=(J+=ln6ZH--WVrW~i@6jxf6J}{Y0QD_CR7<-ptWth19DKfS{
zWBf=SUO38U(8WW7f_`c_k=^@6@u-Z^G+k9^#WF)|!3>EJBb6niu2qeJM8citcZd4E
zPgR@})MXTWiG{)=G2SPKh7330W`FY98V1f^Y#ne7JkesEI66r_jI&x;BIeE!WFMs5
z+R>{+^AVdO0Oa!xac8}#p5SNlvW~;zZrRe7lvxfE5}NC0iN$HP8E?Vf^)|T(?z2jZ
zrW|kAi=ESd6RR<3o$teYaJlk!;^~d}y?$B_a1Za-#F32~l-<~|s#M;>s^Y~o$2ZnI
zV|!O*Pl$Yz2fUsqsc*Y_<Hx!FD04Gen#gaD6uP(;U&o;?_0Q?>3^csiP{ppyQJ0W1
zTCW(sKvNxP&ErZvW9DC19Nl%@vpKOAUPlT0I3Or)cV}IWBOVX|?_}r@P7a5|3r`G-
z;d0oqPT6u9_idux7}&$7S0mxd?hkHC`Kl|6@2k8j_RNoa(YLFYag?doZH{vH3!3;3
zQ1!9%I?>5Bm6=2USf*C>Sh)w->wu%w?A-CP(xZ|0VMj-E@Cfd?#$`zMBlj5O=9u1y
zMKh|32iPHos$;f>3^Q!2nFn}YDlx^CljvqU=rPE$D}rn2q)0u%^zsC4&!KIK8&~-Z
znL+A8LAS<!W=soJ2SSuY2!UcDuBL8Qj_%xCKhOCo@0`1Nu%&`}#|wDS#_6N@td2?V
zz>)Nd@jelv#>nTlL+2o_Nm%5*65@mlG<H(^!lezd759GNYug#kG$AQ9FeloLCd<@J
zbI=GxPD;tmTgcWLaY#yWky|&uw<IlhFV&BF@-ZJGG0&|b4GxBis=CV9DXH+`9*s-}
z0nbjc1>`p$F{Xnke5N+9y;hVbiWm$?bL0o7im4$%)cvresU4^VL!BRJ8}4eWgPE{k
z!5rsmUUw}JI*9i*5b$>u!Us3Rl#u{{cZeIs`(1@U_ZxFC*x_d#)@vK9bW7o1v9AV7
zrQYQ<$Z0c}4z;y|NSdaoz5OQHwNy59?I(oM8A{i7Uf1|`c;UkW=V(E~yR-i7ZVED>
zE}zh;Fs?j~USu@5W3cqZQb0u|QkefvGqwN#L{zQ}jICBE&xlQSkUjQXC&yGcyz@0!
zqD$o2Sc{<O(+(<}6PvP&w@4(09=6BGR%L`F;d6zB;;X%4v)1}aLM)G`p4Ot7a|Cv@
zsBh$s$|ly1HX4Ol+Gn6Y4JU3|-iFuPiidFE84q70;h<FWcQe8pq`kLod*q*4m5e=o
zB^%{_p%T|tSsR?rZG@E&Nrj=xIv!4u`+}jI({eZ8V+!Dzuo2@a(qxhBnNBjIb+?Yo
zKjenX`c3#a1+ISaq^TSvz}`ZKS!esjGlDKU#8b)ZqL=LN-+lYy<;OZUMd8l<smdSL
zOqEI-G;)IVch=+2IE?BL*^WkJo#J=ae=F>tx&NiM-V`;==REidP(ruhgt<w1E&atN
zP7}KA^XC&`AJTUB9XyLdxSEq|x`gNt{duBkUy?LhzbLC`>NCMh8;JP41bAs%tM36M
zjf9s0YxKnx^`Gm?m)nQkN3U6dVQ2UGmkqGSw&Sbm)}t<|9e;IQ;WqOj|ETqRKzn0o
zW*|CD3v3a(iLSwC81qWCba+U!t38}S<G!e7l+UD6?nuu2i_?(r;_bcF0nvNo%Y&Qi
zfTb~&<-hOMxtyt$2B9$hJHIU$kSy5Q!^+viRL9rF%H8;<%6ikr)Db1sX9^kev<G^v
z1kV}=vw!Wt3Y#@9`BG>O_5df&JEuyOm_H!QREXa{+&Yha6DX$8@ydPHf}W!I`~`6a
zj&M9s?ue>F)-C5;XkTaAs>=I?-tgx82hUlL(jG(U1@~BAeqYjtEgTExtY9T~6c5n!
z_Y~-&Q<n%pSQ4rby+G@PNZM^P>3Mu~)5@Kq8p4Z#vEavxdq~<q5=&72w!GM=cD}BX
zBG&zK=9+K!cIVPG!`%*><jhrHusZ?<3`JtN$_9IgB98rqavu(;B8j-?%E)<fbnZ93
zWe)J9F+?ZK-eEL};*hv}Hr`?^{ar;c@r<$5>j-70_Cw*Zej@jU5bX6uzvPmv?af8y
zS(V4!IiD%c;ouh{O%)_$VwC@FP$Km6>lKVh^8dFhZ=-Mfg}<=?Krj;YpXh(PhqvLk
z4Yfb;5yapB7klkC!R-gC{}6=WqW>cJua8%^S#Gb$e^^2h>jWZ<+bq8p=G!c{SCId(
zAlV`|5(LYih2%E$_JH^UMMap#{}>myfwvp(AK(E3_<xcAyZzn<-|icKz?Ou+!2hF*
l+~&DmlYe+L5iaR}sZdQ7RD_EP0AM2CVTh(x_*0I6e*qgvXl4Ka
literal 0
HcmV?d00001
diff --git a/unittests/table_json_conversion/test_fill_xlsx.py b/unittests/table_json_conversion/test_fill_xlsx.py
index 4af73e4d..1dec8494 100644
--- a/unittests/table_json_conversion/test_fill_xlsx.py
+++ b/unittests/table_json_conversion/test_fill_xlsx.py
@@ -78,10 +78,16 @@ def test_fill_xlsx():
template_file=rfp("data/multiple_refs_template.xlsx"),
known_good=rfp("data/multiple_refs_data.xlsx"),
schema=rfp("data/multiple_refs_schema.json"))
+
+
+def test_fill_indirect_reference():
+ # Run this test:
+ # pytest --pdb --pdbcls=IPython.terminal.debugger:Pdb test_fill_xlsx.py::test_fill_indirect_reference -s
fill_and_compare(json_file=rfp("data/indirect_data.json"),
template_file=rfp("data/indirect_template.xlsx"),
- known_good=rfp("data/multiple_refs_data.xlsx"),
- schema=rfp("data/indirect_schema.json"))
+ known_good=rfp("data/indirect_data.xlsx"),
+ schema=rfp("data/indirect_schema.json"),
+ custom_output=rfp("data/indirect_data.xlsx"))
def test_errors():
--
GitLab