From 4df3821fecf072195499ef6eb1cfdcb213ba489d Mon Sep 17 00:00:00 2001
From: Henrik tom Woerden <henrik.tom-woerden@ds.mpg.de>
Date: Fri, 2 Aug 2019 09:55:19 +0200
Subject: [PATCH] Integrationtest
---
.docker/Dockerfile | 18 ++
.docker/cert/all-certs.pkcs12 | Bin 0 -> 2389 bytes
.docker/cert/caosdb.cert.pem | 19 ++
.docker/cert/caosdb.jks | Bin 0 -> 2461 bytes
.docker/cert/caosdb.key.pem | 30 +++
.docker/docker-compose.yml | 33 ++++
.docker/pycaosdb.ini | 17 ++
.docker/tester.yml | 9 +
.docker/wait-for-it.sh | 182 ++++++++++++++++++
.gitignore | 6 +-
.gitlab-ci.yml | 83 ++++++++
integrationtests/crawl.py | 63 ++++++
integrationtests/extroot/.cerate_dir | 0
.../TestProject/2019-02-03/README.md | 15 ++
.../TestProject/2019-02-03/plot.py | 0
.../TestProject/2019-02-03/results.pdf | 0
.../2019-02-03_something/README.md | 21 ++
.../2019-02-03_something/analyse.py | 0
.../2019-02-03_something/images/lol1.png | 0
.../2019-02-03_something/images/lol2.png | 0
.../TestProject/2019-02-03/README.md | 9 +
.../TestProject/2019-02-03/datafile.dat | 0
.../2019-02-03_something/README.md | 15 ++
.../2019-02-03_something/usefull.xlsx | 0
.../2019-02-03_something/useless.xlsx | 0
.../2019-02-03_really_cool_finding/README.md | 9 +
.../datafile.dat | 0
.../2019_paper_on_exciting_stuff/README.md | 15 ++
.../2019_paper_on_exciting_stuff/usefull.xlsx | 0
.../2019_paper_on_exciting_stuff/useless.xlsx | 0
.../TestProject/2019-02-03/README.md | 9 +
.../TestProject/2019-02-03/sim.py | 0
.../TestProject/2019-02-03/snapshots.dat | 0
.../2019-02-03_something/README.md | 15 ++
.../2019-02-03_something/large_sim.py | 0
.../2019-02-03_something/parameters.p | 0
.../2019-02-03_something/timeseries.npy | 0
integrationtests/filldb.sh | 10 +
integrationtests/insert_model.py | 5 +
integrationtests/insert_record.py | 5 +
integrationtests/model.yml | 56 ++++++
integrationtests/test.sh | 2 +
integrationtests/test_crawler.py | 114 +++++++++++
src/caosadvancedtools/cfood.py | 10 +-
src/caosadvancedtools/crawler.py | 2 -
src/caosadvancedtools/read_md_header.py | 1 +
src/programs/crawl.py | 1 +
src/programs/example_cfood.py | 1 +
unittests/create_filetree.py | 11 +-
unittests/test_cfood.py | 3 +-
unittests/test_table_converter.py | 1 +
51 files changed, 772 insertions(+), 18 deletions(-)
create mode 100644 .docker/Dockerfile
create mode 100644 .docker/cert/all-certs.pkcs12
create mode 100644 .docker/cert/caosdb.cert.pem
create mode 100644 .docker/cert/caosdb.jks
create mode 100644 .docker/cert/caosdb.key.pem
create mode 100644 .docker/docker-compose.yml
create mode 100644 .docker/pycaosdb.ini
create mode 100644 .docker/tester.yml
create mode 100755 .docker/wait-for-it.sh
create mode 100644 .gitlab-ci.yml
create mode 100755 integrationtests/crawl.py
create mode 100644 integrationtests/extroot/.cerate_dir
create mode 100644 integrationtests/extroot/DataAnalysis/TestProject/2019-02-03/README.md
create mode 100644 integrationtests/extroot/DataAnalysis/TestProject/2019-02-03/plot.py
create mode 100644 integrationtests/extroot/DataAnalysis/TestProject/2019-02-03/results.pdf
create mode 100644 integrationtests/extroot/DataAnalysis/TestProject/2019-02-03_something/README.md
create mode 100644 integrationtests/extroot/DataAnalysis/TestProject/2019-02-03_something/analyse.py
create mode 100644 integrationtests/extroot/DataAnalysis/TestProject/2019-02-03_something/images/lol1.png
create mode 100644 integrationtests/extroot/DataAnalysis/TestProject/2019-02-03_something/images/lol2.png
create mode 100644 integrationtests/extroot/ExperimentalData/TestProject/2019-02-03/README.md
create mode 100644 integrationtests/extroot/ExperimentalData/TestProject/2019-02-03/datafile.dat
create mode 100644 integrationtests/extroot/ExperimentalData/TestProject/2019-02-03_something/README.md
create mode 100644 integrationtests/extroot/ExperimentalData/TestProject/2019-02-03_something/usefull.xlsx
create mode 100644 integrationtests/extroot/ExperimentalData/TestProject/2019-02-03_something/useless.xlsx
create mode 100644 integrationtests/extroot/Publications/Posters/2019-02-03_really_cool_finding/README.md
create mode 100644 integrationtests/extroot/Publications/Posters/2019-02-03_really_cool_finding/datafile.dat
create mode 100644 integrationtests/extroot/Publications/Theses/2019_paper_on_exciting_stuff/README.md
create mode 100644 integrationtests/extroot/Publications/Theses/2019_paper_on_exciting_stuff/usefull.xlsx
create mode 100644 integrationtests/extroot/Publications/Theses/2019_paper_on_exciting_stuff/useless.xlsx
create mode 100644 integrationtests/extroot/SimulationData/TestProject/2019-02-03/README.md
create mode 100644 integrationtests/extroot/SimulationData/TestProject/2019-02-03/sim.py
create mode 100644 integrationtests/extroot/SimulationData/TestProject/2019-02-03/snapshots.dat
create mode 100644 integrationtests/extroot/SimulationData/TestProject/2019-02-03_something/README.md
create mode 100644 integrationtests/extroot/SimulationData/TestProject/2019-02-03_something/large_sim.py
create mode 100644 integrationtests/extroot/SimulationData/TestProject/2019-02-03_something/parameters.p
create mode 100644 integrationtests/extroot/SimulationData/TestProject/2019-02-03_something/timeseries.npy
create mode 100755 integrationtests/filldb.sh
create mode 100755 integrationtests/insert_model.py
create mode 100644 integrationtests/insert_record.py
create mode 100644 integrationtests/model.yml
create mode 100755 integrationtests/test.sh
create mode 100755 integrationtests/test_crawler.py
diff --git a/.docker/Dockerfile b/.docker/Dockerfile
new file mode 100644
index 00000000..2188dae2
--- /dev/null
+++ b/.docker/Dockerfile
@@ -0,0 +1,18 @@
+FROM debian:latest
+RUN apt-get update && \
+ apt-get install curl python3 python3-pip git python-autopep8 \
+ python3-pytest -y
+COPY .docker/wait-for-it.sh /wait-for-it.sh
+RUN git clone https://gitlab.gwdg.de/bmp-caosdb/caosdb-pylib.git && \
+ cd caosdb-pylib && pip3 install .
+RUN git clone https://gitlab.gwdg.de/bmp-caosdb/caosdb-models.git && \
+ cd caosdb-models && pip3 install .
+ADD https://gitlab.com/api/v4/projects/13601752/repository/branches \
+ scifolder_version.txt
+RUN git clone https://gitlab.com/henrik_indiscale/scifolder.git && \
+ cd scifolder && pip3 install .
+COPY . /git
+RUN rm -r /git/.git && mv /git/.docker/pycaosdb.ini /git/integrationtests
+RUN cd /git && pip3 install .
+WORKDIR /git/integrationtests/
+CMD /wait-for-it.sh caosdb-server:10443 -t 120 -- ./test.sh
diff --git a/.docker/cert/all-certs.pkcs12 b/.docker/cert/all-certs.pkcs12
new file mode 100644
index 0000000000000000000000000000000000000000..382b9cfe7dd1cec7c9f98f96c1fea63a971585ab
GIT binary patch
literal 2389
zcmY+EdpHvcAIEoRj)pLDS-C`<tO+N|P%e#(N+j1rE|Vg2iDDLJ88%vSNiw%`H_Sb?
zki-h7Medghv7vH1Id47B`@ZLW|M>m$`8?0>zt0y#hMxfe!5A`J4hGjw#3iop0*V0h
z$#78!84mf00T?ov^N)h%lfk7wQ6UKU%RKnU05M81v44LM2f{E2h^P@?z>OOGx()(~
z0!U<VFxkz{!Z~xb^m4K}R`H#oAgCetHRQ!gYOWR!vG+|@jp=2n=xVT_NI{(oSX9BT
ztHWc8QqN_md=~swns`y;O{a4oSv9TbL#|^oo8@6mYAH*{D>0Jt`aW$qCS`|;!1$8S
zJER?3b;h`l6Sgr;JNb2ys%KnAt_jh>;&74da&??~dPDE9D1_yEJLJc`qLaQA+v-79
zdYch!ufbbs;C+GH>iP9nJa`)ukWSaxwmD)^Q_`TRGe^M4&*Tdb9q4|I0!52C#e_Yo
zEy~6u++^Lx=8vN-w3DaodCy`p6jqhNx#h)^wbh?_c@C}^DSf}Ngm3+yRI}^M=Q_iV
zD36RZ`44{n9+KqNCjCC)xqS9b*?=48PbXwL>P<DnuheJ=6k|^Kd$pY0+UB4xl$^sl
z*!d08zB~z^IznH%_rSx(fF#8TMocBOZ4Z<kmzywT2N5%N@~#Q}8s`@6iUW)ZzPT3d
zZ9fVSe{$ITir$CBwEdxkkD5^@J?v3ZloOGl+s&;sjkGL@`NW9Tw;>d}6CiODE90Dz
z=6h66oo??AFH1eLwr_kUx)oO8tj?S_k{2?YfxlTWe#}fphYt-Qwf609N<CN-B7FY<
zgkPU_n()cIPSAd1U;jceEb|E(@d)*7G}c-FguCO)4@rtD8Z3Mi8$Es!b?mAc_`FZ{
z>)TjK$rASNQ?>_O585i+P^KHS8H+H*+7J8G=o@PBDwn^AW#ORQFS^YmHy;hm(!2k}
zBH4c$YmZ_MMwG=S?Q-wjV*RBwgy0V*wv0@7lM|0Ciqce^;#`N0MxkZA!&rBexYzGf
zjEIk^1gCG+`ED6}OTfXMOVpt$7)<%rAx6P^fkd0l8OtDl)Y(HbeMpV8pa}kZ;^nIb
z-ZKr2E2}c5E0f;_1G~<pp%qN)hvNjp2dtz<JYS;r4&Ys^C&2O6nxF4gUg!Y|lv{)n
zTw4`|LdjlmoPJ;vw?f}F9skLQJr+kyzkVJPw}Z7L#LzB1{*J~UM6~m^V=3h}SlrxZ
ztmb^x85GMovqIMfn+Hes5B{yIE8OS=yy`g(nzYx(?f=7lCt&8DbY5cht8UT@)CHIR
zo@uKGRJ`8(i=ld(N)5yt$)^>+NLe#tk|yUl<#CGYA!a@aWrbP8i@qgMwtLl*hn7$n
zGW7WW1e#BV9)^&i=$}~o=V`!x`%kW7Ku|szvWy`^7XGhd@UMzDkRc6Ki42!t6+_67
zmu5k;B%#g9KFeNU$e4d!>j!%kKfs1sV<Wr2@6~`)5A}H3E-AIR9%eoH<H+DdXHC?^
zQRu6(HCt{ek_@#{SH<VkxsL;Jve?~swcOH&N5xY3G{Dv8Je+EHQKWZlyohR8@iNX|
zh8T`cS!-K-#O7)?ZfR%bZ0c6Ayy1O{b?ZLo9+nSZKzuYpgzH82n)m(ww|}N&XE}@q
zeSfFZe-s;lw8X*UUM(-VuN9txMl0GeYe6{SAGcyR0SpVySb-(2bi*ZJB#>A!VYs_k
zz#UgBpl_5a<L9qr8=KNe?PE=M@wxW{vt>OXqXgGrBfwv-)6ru6d*Ln?8Tf+oH6+1U
z91GAKqt}=;c?Fs(x1tFP=En{l=D^LXW<FFMA^?Y)Rrs^j(`Hs-d!7|v`rA>Tkvk<U
zX5Kf}u`HCeep~zQ8K=WX>)Yb(E?ZD!<4Zo-8Y&kaip{#hx=EX+4ibT72lAj&b?pXC
z3Qt)Rmg01GY3N0G?%lo<)8cDcd7f>x?gK{}dYMPtPg5PqmF~v;k0~obMRh8nK1xs3
zu3jgZmUK^A(bPU{+bb?bwiBnAWjj3(ZjRKaaimji`XY0qEr%=~+ymDtLhEEG92Q$S
zn#s--%w~(5WMR0@tKt;<k~bl$U*8jP)v%R3$`JZkZ5dw6I<Qt2Jt>o72VUtTibe&S
zByLo`b=s4)u)cqMI*(phv8WK+v`_DMZWC&Oi8^!q;f4#mKVr+pCV=w2&v}vm-E)=~
zc1!%+rB;5GJw!+*L@yi3)I828Sr|SYoCcvWg7wnpWNy!G+8ql*`9cU^d)bi(O!Fpo
z9<R?Y(rzTk+^lFkW&GhPu_61iDvo#8<?R~!V=9<#X5k>A-sXm2-Ca#5^o-XFD>E|o
zDEFb(1)tA4g#}CtJp9JJcN&MfMhrb1IJG*fa+LV}2EC1wt*Jut#i&7JiivxKqu+f#
zG0Aq%aAVF|Y+h(<h`jj`Q_<6;nVo}`g9=4wDaMBI3$@L49b*mjR*GSUR-hYg=F_5%
z`VN3Y^|c22bL8yR@yRd8YEq021vw#OkKaqSSy8j$hj3z<a`PZ&hGf-jluMlbhfQzU
zz$6)oR5R19?xYllv*zdPts@QiHzU5UFj1nf;x0W}Y5-NZh_*=s=38Iv@k;z6u~Abx
z7+m%2>HcByX&m$pA+5g7cS?7>_*jvDt^?>?Ers^Y5TE}7>>n#*lc-s@%JyY}t~mv9
zk6mub`{7yG*mXbh?SL{xKotjc(U(eV{6ozqouG4WN^K@N154Cy>AVFId8KLc>vfz;
zQq{u~qe@hsq}1KyKUm9ksWA@?QnBIl+BdhvkQ&0!k$ntg=Uhm0!}3Z+PSbm{XhV6H
z|3KkIx`*FaPZ<@&3si8REv194vGiNUxV_!3Y2lVikwcQ#nAFJuDozdHwkY?ctMGDv
zlFXBfX}i-Z=Se%HSw&v_&i1|()}ww6l0lam<Foj)0mTGn>U9gen{dgndM)7_C@YAH
zuyW|d6p6&_?s%8#RyguzG>DT=9#o2tzq4pw0+ROXXJ46Ume{c<tA~A|!lPVgc{(f;
zRe2W<gM`$&Hve(aEgbm}u0PvyOLsp;?Z4{?m(@i=WE^8=u0MgeEO_jqSqrg~mTDl*
zJB%(y1p|jce@B2p3Qzz<hNgePH}X`^^INCz+Gi(35vo@9o)A%nE8z+acpLs}CI1C;
CMQ6DH
literal 0
HcmV?d00001
diff --git a/.docker/cert/caosdb.cert.pem b/.docker/cert/caosdb.cert.pem
new file mode 100644
index 00000000..af44eed0
--- /dev/null
+++ b/.docker/cert/caosdb.cert.pem
@@ -0,0 +1,19 @@
+-----BEGIN CERTIFICATE-----
+MIIDETCCAfmgAwIBAgIUM1S4TR6nEsstWuXDjZGBv6sb0LAwDQYJKoZIhvcNAQEL
+BQAwGDEWMBQGA1UEAwwNY2Fvc2RiLXNlcnZlcjAeFw0xOTA3MTgxNDM3MzNaFw0x
+OTA4MTcxNDM3MzNaMBgxFjAUBgNVBAMMDWNhb3NkYi1zZXJ2ZXIwggEiMA0GCSqG
+SIb3DQEBAQUAA4IBDwAwggEKAoIBAQDMFzou21Jugj9Z3YUkhXANt2oGNvuCyt9S
+IfBZO5ZpgwSTcRs8UAuKDYAMmGMeQtqLfqEYoQFEeybSd9NsrFb0IgrUaKrSNAzs
+phjsGvXU1UbMRw0vva3l6r94ceM8ecw+x2RIsTwfJkfVL60XMm9DvTaQFLzjcKLf
+lOQbXG6jgCHtvO5MThk/C+TvsAZlRDErp7HgDqDFg+jnJC5YsEFQQyAYCLJXOotQ
+A5FnRbT8dtoDHy03mHyed5Ji3IgxKxE5mF0ygG8/FRfgzqImg4FMnCb+KBwom2xh
+y2hvtFnm3+kCbB+waDTUaPZwvNUONUZMfcl3PggX0IwipqWL8DBJAgMBAAGjUzBR
+MB0GA1UdDgQWBBSglY56eKgLeyW1o5UdZgqlzDJJEjAfBgNVHSMEGDAWgBSglY56
+eKgLeyW1o5UdZgqlzDJJEjAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUA
+A4IBAQAF4sn/C7wx9fq/9/v6UYb9DbOXaonS+1jOzf1lAOEwHtJh0M9SDTqvaeQz
+bshJ33ZFbwH3qIq7+OzGj9Yon9LN1GGEl4CcKKQRQQkjo7jt7X47C6PkrW6l23+i
+OrDRk7Oke9BJS/ZRKLmxvlaLYULpj02lOsbddfE4igXxRPCiouosc+HQ3kd6Mqhv
+5Gt0Yki3z4fvi+CWrtUjxYeMUJkraFAyKDKwFVZgDnWwyipGo9yh3a6iXD7duQcx
+FRk7zm1oaF3ia9hFfU1AVDfkgIlPLWW30CT4Jgi25/wpiGZiT1uXAgfoJCEDoYXp
+bQIM5KQqw0Kgjqc/KuLCRe1KeCzq
+-----END CERTIFICATE-----
diff --git a/.docker/cert/caosdb.jks b/.docker/cert/caosdb.jks
new file mode 100644
index 0000000000000000000000000000000000000000..355b78fd3b60ebdb92b111ee0d81bc6f1adb8add
GIT binary patch
literal 2461
zcmY+Ec{J1u8^_IJn6V~8=$f&NHNUwSLS-3bOCmM0(`0N>b_!$3Sg*phmG#PgGl>{8
z_8Q$0vczbJa%I;tp)A+?p7*`?+&`Z4ob&lU&-c#{K!O*5K@flhzX*e?B%e#(5&-dl
zN=a}NC<%V{5bFaZZnu9$+!vrEZtFvQ?r?B0zW;skBS7F%67&Z^f^GrIFvS1yVYw7k
z6kRwLTqptlP8+mA@kZ{cQI*+&U<g<5VHcDUf6MDzk1SbZZf0@qU8{@eqTDfatG%e_
zh69-*6+)}qxidkdn)O`zM;Fd2dpIFFc2>sS<?0?KN~|Tjwa<c6*Oa))UvIr(J*V<B
zXc2~r=Kc?{{IA^G-7}W#Zj{h?O~QnSBHT_{UPq^IdMtM)d#WU+v@k)v>Mm89XfsHF
z$q6S}r(3`A?0<=zUC^k!vskG!1&EODtA{gnL`vT+6fOk@*)F)Q;JaTnSJ<zKi}+i>
zpcKy)@b&f3GkhCwV&8|wYZczd$B3S$7}h<zGOc>-o*-K_u1XWt*k2_4Ysc)WdJxA0
z#%UZ6ej(&7`B`R9SV-q>&beU}2+?mhE65J?v%BjU9l8Fp-TnRN_HV&r>z$>A$h(bb
zZDZeD))eKDv{8EeDq+Zlm4u^C?4*rGZL`kNEL<;d_BODNNx3lfnX-$~S7lt)i{$ip
z#(xPY+`o~#8ygx=M+!9dzSg?y_dcRAJl!yFrnA5@Z31KeiPgqfX5=Mvx?YwgfK-Gj
z7;jiZe}F#5JIv<{hksu*8h$ixNziEdv14<3=3IT~jiDInP_y%`kQ6=JFSUAKMY`oJ
zrKdCS^NI>C&BmVvPxbV@12{eB%C6=m*-MwrIcCrjexlM(D|z73Ek*mZGzQEZWUtjF
ztqm;wAb--<x9!YMgaEtppbhkQxdH~q((x*u7}F<Oemhql`wY0TADO;aV$<uCZSn4m
z`U5_F#FIApQ$sZ*TnUlClRd5uL>y~4+tO@>$~=$PCc>Pya3#MzS6SbACKGrzw{Di$
z6h}KDp2jLFX?s{R`&dP-PU1UXerI>x=X10CMtP&_O%jy_dM8H{y}8XM<^woljC9E&
z<&1|Hw5~;v8osp1WEL4if{$P@XztnzivjcdSztm%oiWXZ)0@Ix>mN@^cmX`9vi=}U
zrG&DlzaBJ8ea9E;Ne%2((Z4l(ToBTIK4f#A)@_gdz*DP}?X$su4{ufPbVA_|C(qS`
zLEDJ?Kfd_a6NB5<qdNKqGbu;$y0XRQ>S9P|e>x{;>|F|V$U}JE^USlT{EED06W>)t
z?MBGS0ZhWMm3`^-SuO|Yj7k+M+9cg{m^JvMZJ@Shr({Wkm=(Jq1~y1$-pq79mSL*I
zv<HId7|FVCHxce-p<7OFCDB|Bx4e?%)qhqMH)x`V+<z<EXi4;xgkGyquyN2VcJzgU
z13=E5&YRi}gThe;21Z%i_pqWmOX0sGeS9mejS4u6ILUgW+_D|hMTuqG7R6WFsn<OT
z;>$<gY@y#GnR7})W@C+tMcNVG(t_6A?4d64Nh8#8KA@hCJ(}mI8=V2Zs+)6DJwi`i
zVBBTwWP|EKzvV&Aq;=k_5+!BT8mv5+?5C?X#Wzn5x2O2}%*9`NbcLeX(|Y9vvvsh6
z{M3!Pt-c-3I@|2;%`)~%EiWBu6%*NQxjgST{Wx#FXr_8fubAOcvQ31s#CtS^egfU$
z#qPTHUstm8%<*pErJ6zWpwSnKdq5R|Ngqb5dH4jHQ@<<wt1XWWsLPcq4B|oze}gln
zuMsAy&1-`l9uf*JEq|O;v<qmp(AVM{L0N3*nOiPa`!OWDyTf5fTa&2r%3nM%X7R$G
zbmuWx;K<*qgo|RJC%{}70QnCR!-zsf9EbteWHH)UO)V^d)z;Jo0DuG;`}>FoR!V~O
zAEItBi0g2*|C50J<yF{!yvko`x<Wd0z(HtJ@nm7%{9(+*%l~>cjs%H&rr-40x7xBk
zVD6;EtNNqnU$(5re@F$2)9UF7>)!c7P0u~%O9w5qh!;1YWRD?Xg+8uC!a-{A0cb!b
zc1I1ql3dddALtfKE)aocyT)zbTPk^a>_@^49mn9Qw{Wg-S%DI((9Zj*@o+5J)ND0$
zqRdS&<Y1*~A6Mvjnl+Lwr!ZM2k9(cm*dAWxN1zeh#Ja3526bR;$ul{zb%cNld7FK>
zO0>K|hEGvo>VS72PlcTIv<rMjwV?VB_O9L(>i8FOtlXW<Ex(Q3?3DOiq(exI8s`xw
zFdvzfl~)dp?`v}!x!&SU2A(3)I)2fOwccQqbJ5l%gXSOZU);rW_%m}UBLZ0xF;q+X
zW*q2iSAxHyYr~u3K`xUAS0aR-)IAs!D>ixEUA&;Qr?Rac8MbpJb8k~N(=4rwpp&&q
z-`sxV)MuBOC^oyjSrs+n_%Xh8>SG%5)L3XvqfPN+oV|)hn?iX^6(#thHJYgCmEH4T
zw@I9k2p>8g1zwKL^}1U?4D7+zpbcez#yO70)IGsUS>f+IZW%OFVC8sz7m5qdFO)$C
zK6NeSLbe+{SqpaV6zY1V<-rgF*6qY^d1p_XB_lV7yKt5ujMqsDYd4=G8OP6F@JU(`
zkt|WL*y*70kSO<zQ=B!>?QPA0?kaiw(#;^t6cOd)&k(BBA@oTtY6P5Hi~8Iju+sI9
z2K%P5!9~6fsI5HC3H?Y*UFhoDUOd~X<xi{k$IVQtOyjO6n8TD-?*TJ~Mf%h-#@E>b
z%s2W@bc;*3CJpH{y~8O!(j$l(yVS4XVSqwohd1)^xA!{kKIWQ%>A%FGF4V{fcoX6m
zwLyeaa#;q1{b@4LUQBJh8jGL_xN6lGEh4p`R;?SFVdgQHQEw$)XZlvUBz_xj3qYM8
z<%2lJBdfNNCU+f(^vhn5)4vmF;;t1p9CI5r+N8oH6p8LW?<^MFqi$1<kT-mbM-0HJ
zaE*XVVJo-)5Q{{Fu#;39TB&H~=;l8SH*;Tv&KXdKb1D}<=t*B-jxj%U>BpH0IoFiE
zjvJ8S>9E>jj@f-5t2;e&Gk@@9P#pu&E&7`GrEKrVU1o22QmJI^P*3JlHAc;Qb_*&D
zVhwW&`eYo-wy*tZk)<kqD{)8LfKwTa?*Z94EC_>XxXn|cNIKeIjd!<YWdhjs)C0}{
zNB|DwRuzPRk8^WDMN1F?u2Yl0Ldxt$Z!Ymo>D1`=`9MWCyBQC0-UCiUC+vTfly`IV
Nn;hE(LqN%;e*q8neR%)?
literal 0
HcmV?d00001
diff --git a/.docker/cert/caosdb.key.pem b/.docker/cert/caosdb.key.pem
new file mode 100644
index 00000000..3459474b
--- /dev/null
+++ b/.docker/cert/caosdb.key.pem
@@ -0,0 +1,30 @@
+-----BEGIN RSA PRIVATE KEY-----
+Proc-Type: 4,ENCRYPTED
+DEK-Info: AES-256-CBC,0B42601A73984CD9F1DF2F786F894EA0
+
+5ZY5CR8ho3a9449XIUmSpIsCZZWsVQ1pWX+SJPSUlk2A9N3BHdY0AdvCx+sTF7p4
+bSoAulnAEFlIeXqimtK+InH0d3HLbvn41gvEbFTPjZdejxro6m9DvSHCpyDZK9TH
+m5A2aRsJioeibETUA7uvld/d9Szdz//GPN2YzWqJpzXZtTWHH/tR2J2L6KE+wBDa
+IViryBCv0+9e6l2WeEjXqqYZhznR/gUsxc2xEs7x8wEpWBVlmVzYWuSt7OA5oOm/
+r1OAR9LEH0PIISUv2h+P7ccOZJD/JE2YDM3IBdClxLv2MRyj/88ougcV5gv688+f
+6XxT2+LfCLd+viqGcFZ5a6tT1wpUthVWWvUWPna6SlVVrsj/7bCc6y2Jptff+Kqv
+eL8Gl1dWZXwe0JXr+Ss5KfIddbY0pJv8FHjCsN+Nkdozo7Wdufm1Xkw+s1f2td6u
+bHwbo1Y4mX5nm9I2OkAMhGubSPalGZRUvr9cfl1/1V/QgX0jPmJmNmH5yFUdc5MQ
+QIdcpZmz8xIH1XAdJWTbzoYKzKnXTHpZFaZL1acMrK2CERMxDStfe9W/DSRy6/qn
+AsQbh6tDk+Fe05EQwpZBZQ1YlrfAru7QDaDE9+uO+mxkf1UHynfPHNp2jZAe6XBt
+p97GXUruLR8rBOAGX19EuoG2sSek44dLW5IsLGbR1MpTlq48YZv2z3TrESNJHKAM
+VAxg5kwqvYocykdFrKmggbdBv3oo6g59RxiufKparlkI7YZFhks6V1gujKW+JChc
+Eoy8e2EJJP0pY0Pb3ZE9QkK7ZX4otXNRgqqnf2f7kTiGMRBSDSD5xGUw0syXAS3G
+Ji/4pMFh8ALsYUOOOYUISWAUB6O8/438SIq+g+TNHYVbrli/WK9ip0+RGPmJ+Ulm
+iWQMQOcsSYTBqDytvr0241JDBuq9RjMBJE4FO2ltr7L4gmkN8+d/WGnvu4onJBoE
+8ApriArJjuHBP1iex3uygJKD3sdiuAuukrPlEla1xjywH3QSzj/SWABIu4QJAr3a
+70jEFGL04eblvBPGN2hnXEQCxHqRJLpFH1huWMlO3PuoRnGxIuxymkEOOEWfswLO
+r9qLbkUWmFFFqJBNB3T4kfzuZDqiCQnS8+QNvhTiKCWLcUswXTirSac8B5cWDaqd
+/Nf119n3ezZkpzgKr0kw+kLxlU2Tbl7UBZ7TRadxklXaAmEo7z+Wh7WKmbcUMqit
+xh6P7P0ZWcLYIiXCC5gYr6jzU+Rjx4ZGF5+Q0LcXC3Ai6Ujz4zdrVRLBiw27zKgB
+RsYBCYRmQ2qO0AzHFBWixl7IVGYi+QFKU0SFrPLzme+yOJjjLAv2Xzu24KFiwHLz
+8MWCd0yM8UjXAql+jCk54O6e4QwER2Rfr6vxlH6x90diy8eJQXejBEVNIfyOpyST
+Mqn2jc5vWanVxRk/SgrZ0pOJ8TBeGSzolJF9MBPjZ9PJTbu3WzJpuguQ4CeAAV7E
+0+rbPGALnuDKmrsnyIb9WX+fEpT7oBPEKV84XyG1+UkBrfDaQvK3SujeRvnOaZLG
+uSNc8yX9CZmt2OJTRWK5fKnR5fGzAnMR7cRUYNvcWJ0T6B4wj8cwF4wY9laGzMO7
+-----END RSA PRIVATE KEY-----
diff --git a/.docker/docker-compose.yml b/.docker/docker-compose.yml
new file mode 100644
index 00000000..97fe0c64
--- /dev/null
+++ b/.docker/docker-compose.yml
@@ -0,0 +1,33 @@
+version: '3.7'
+services:
+ sqldb:
+ image: mariadb:10.4
+ environment:
+ MYSQL_ROOT_PASSWORD: caosdb1234
+ networks:
+ - caosnet
+ caosdb-server:
+ image: "$CI_REGISTRY/caosdb:$CAOSDB_TAG"
+ user: 999:999
+ depends_on:
+ - sqldb
+ networks:
+ - caosnet
+ volumes:
+ - type: bind
+ source: "$EXEPATH/.docker/cert"
+ target: /opt/caosdb/cert
+ - type: bind
+ source: "$EXEPATH/integrationtests/extroot"
+ target: /opt/caosdb/mnt/extroot
+ read_only: true
+ ports:
+ # - "from_outside:from_inside"
+ - "10443:10443"
+ - "10080:10080"
+ environment:
+ DEBUG: 1
+
+networks:
+ caosnet:
+ driver: bridge
diff --git a/.docker/pycaosdb.ini b/.docker/pycaosdb.ini
new file mode 100644
index 00000000..b57f2ebd
--- /dev/null
+++ b/.docker/pycaosdb.ini
@@ -0,0 +1,17 @@
+[IntegrationTests]
+test_server_side_scripting.bin_dir=../caosdb-server/test_scripting/bin/
+
+[Connection]
+url=https://caosdb-server:10443
+username=admin
+cacert=../.docker/cert/caosdb.cert.pem
+#cacert=/etc/ssl/cert.pem
+debug=0
+
+passwordmethod=plain
+password=caosdb
+
+ssl_insecure=True
+timeout=500
+[Container]
+debug=0
diff --git a/.docker/tester.yml b/.docker/tester.yml
new file mode 100644
index 00000000..7eb8d4d6
--- /dev/null
+++ b/.docker/tester.yml
@@ -0,0 +1,9 @@
+version: '3.6'
+services:
+ tester:
+ image: "$CI_REGISTRY_IMAGE:latest"
+ networks:
+ - docker_caosnet
+networks:
+ docker_caosnet:
+ external: true
diff --git a/.docker/wait-for-it.sh b/.docker/wait-for-it.sh
new file mode 100755
index 00000000..d69e99f1
--- /dev/null
+++ b/.docker/wait-for-it.sh
@@ -0,0 +1,182 @@
+#!/usr/bin/env bash
+# License:
+# From https://github.com/vishnubob/wait-for-it
+# The MIT License (MIT)
+# Use this script to test if a given TCP host/port are available
+
+WAITFORIT_cmdname=${0##*/}
+
+echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi }
+
+usage()
+{
+ cat << USAGE >&2
+Usage:
+ $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args]
+ -h HOST | --host=HOST Host or IP under test
+ -p PORT | --port=PORT TCP port under test
+ Alternatively, you specify the host and port as host:port
+ -s | --strict Only execute subcommand if the test succeeds
+ -q | --quiet Don't output any status messages
+ -t TIMEOUT | --timeout=TIMEOUT
+ Timeout in seconds, zero for no timeout
+ -- COMMAND ARGS Execute command with args after the test finishes
+USAGE
+ exit 1
+}
+
+wait_for()
+{
+ if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+ echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+ else
+ echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout"
+ fi
+ WAITFORIT_start_ts=$(date +%s)
+ while :
+ do
+ if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then
+ nc -z $WAITFORIT_HOST $WAITFORIT_PORT
+ WAITFORIT_result=$?
+ else
+ (echo > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1
+ WAITFORIT_result=$?
+ fi
+ if [[ $WAITFORIT_result -eq 0 ]]; then
+ WAITFORIT_end_ts=$(date +%s)
+ echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds"
+ break
+ fi
+ sleep 1
+ done
+ return $WAITFORIT_result
+}
+
+wait_for_wrapper()
+{
+ # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692
+ if [[ $WAITFORIT_QUIET -eq 1 ]]; then
+ timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+ else
+ timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT &
+ fi
+ WAITFORIT_PID=$!
+ trap "kill -INT -$WAITFORIT_PID" INT
+ wait $WAITFORIT_PID
+ WAITFORIT_RESULT=$?
+ if [[ $WAITFORIT_RESULT -ne 0 ]]; then
+ echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT"
+ fi
+ return $WAITFORIT_RESULT
+}
+
+# process arguments
+while [[ $# -gt 0 ]]
+do
+ case "$1" in
+ *:* )
+ WAITFORIT_hostport=(${1//:/ })
+ WAITFORIT_HOST=${WAITFORIT_hostport[0]}
+ WAITFORIT_PORT=${WAITFORIT_hostport[1]}
+ shift 1
+ ;;
+ --child)
+ WAITFORIT_CHILD=1
+ shift 1
+ ;;
+ -q | --quiet)
+ WAITFORIT_QUIET=1
+ shift 1
+ ;;
+ -s | --strict)
+ WAITFORIT_STRICT=1
+ shift 1
+ ;;
+ -h)
+ WAITFORIT_HOST="$2"
+ if [[ $WAITFORIT_HOST == "" ]]; then break; fi
+ shift 2
+ ;;
+ --host=*)
+ WAITFORIT_HOST="${1#*=}"
+ shift 1
+ ;;
+ -p)
+ WAITFORIT_PORT="$2"
+ if [[ $WAITFORIT_PORT == "" ]]; then break; fi
+ shift 2
+ ;;
+ --port=*)
+ WAITFORIT_PORT="${1#*=}"
+ shift 1
+ ;;
+ -t)
+ WAITFORIT_TIMEOUT="$2"
+ if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi
+ shift 2
+ ;;
+ --timeout=*)
+ WAITFORIT_TIMEOUT="${1#*=}"
+ shift 1
+ ;;
+ --)
+ shift
+ WAITFORIT_CLI=("$@")
+ break
+ ;;
+ --help)
+ usage
+ ;;
+ *)
+ echoerr "Unknown argument: $1"
+ usage
+ ;;
+ esac
+done
+
+if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then
+ echoerr "Error: you need to provide a host and port to test."
+ usage
+fi
+
+WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15}
+WAITFORIT_STRICT=${WAITFORIT_STRICT:-0}
+WAITFORIT_CHILD=${WAITFORIT_CHILD:-0}
+WAITFORIT_QUIET=${WAITFORIT_QUIET:-0}
+
+# check to see if timeout is from busybox?
+WAITFORIT_TIMEOUT_PATH=$(type -p timeout)
+WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH)
+if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then
+ WAITFORIT_ISBUSY=1
+ WAITFORIT_BUSYTIMEFLAG="-t"
+
+else
+ WAITFORIT_ISBUSY=0
+ WAITFORIT_BUSYTIMEFLAG=""
+fi
+
+if [[ $WAITFORIT_CHILD -gt 0 ]]; then
+ wait_for
+ WAITFORIT_RESULT=$?
+ exit $WAITFORIT_RESULT
+else
+ if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then
+ wait_for_wrapper
+ WAITFORIT_RESULT=$?
+ else
+ wait_for
+ WAITFORIT_RESULT=$?
+ fi
+fi
+
+if [[ $WAITFORIT_CLI != "" ]]; then
+ if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then
+ echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess"
+ exit $WAITFORIT_RESULT
+ fi
+ exec "${WAITFORIT_CLI[@]}"
+else
+ exit $WAITFORIT_RESULT
+fi
+
diff --git a/.gitignore b/.gitignore
index 046cdde9..51c99625 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,5 @@
+__pycache__
+.tox
.coverage
+cache.db
*.egg-info
-.tox
-__pycache__
-
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
new file mode 100644
index 00000000..fccefd25
--- /dev/null
+++ b/.gitlab-ci.yml
@@ -0,0 +1,83 @@
+#
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2018 Research Group Biomedical Physics,
+# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+# Copyright (C) 2019 Henrik tom Wörden
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+
+variables:
+ CI_REGISTRY_IMAGE: $CI_REGISTRY/caosdb-advanced-testenv
+ # With dind the docker deamon is available on the network
+ DOCKER_HOST: tcp://docker:2375/
+ # When using dind, it's wise to use the overlayfs driver for
+ # improved performance.
+ DOCKER_DRIVER: overlay2
+
+services:
+ - docker:dind
+
+stages:
+ - setup
+ - style
+ - test
+
+test:
+ tags: [docker]
+ stage: test
+ image:
+ name: docker/compose:1.24.1
+ entrypoint: ["/bin/sh", "-c"]
+ script:
+ - if [[ "$CAOSDB_TAG" == "" ]]; then
+ CAOSDB_TAG=master-latest;
+ fi
+ - echo $CAOSDB_TAG
+ - docker login -u testuser -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+ - docker pull $CI_REGISTRY_IMAGE:latest
+ - EXEPATH=`pwd` CAOSDB_TAG=$CAOSDB_TAG docker-compose
+ -f .docker/docker-compose.yml up -d
+ - docker-compose -f .docker/tester.yml run tester
+ - rc=$?
+ - docker-compose -f .docker/docker-compose.yml down
+ - exit $rc
+
+build-testenv:
+ tags: [docker]
+ image: docker:latest
+ stage: setup
+ script:
+ - docker login -u testuser -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+ # use here general latest or specific branch latest...
+ - docker pull $CI_REGISTRY_IMAGE:latest || true
+ - docker build
+ --pull
+ --file .docker/Dockerfile
+ --cache-from $CI_REGISTRY_IMAGE:latest
+ -t $CI_REGISTRY_IMAGE:latest .
+ - docker push $CI_REGISTRY_IMAGE:latest
+
+
+style:
+ tags: [docker]
+ stage: style
+ image: $CI_REGISTRY_IMAGE:latest
+ script:
+ - autopep8 -r --diff --exit-code .
+ allow_failure: true
diff --git a/integrationtests/crawl.py b/integrationtests/crawl.py
new file mode 100755
index 00000000..df76f6cf
--- /dev/null
+++ b/integrationtests/crawl.py
@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+# encoding: utf-8
+#
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2018 Research Group Biomedical Physics,
+# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+
+import argparse
+from argparse import RawTextHelpFormatter
+
+import caosdb as db
+
+from caosadvancedtools.crawler import Crawler
+from scifolder import (AnalysisCFood, ExperimentCFood, PublicationCFood,
+ SimulationCFood)
+
+
+def get_parser():
+ parser = argparse.ArgumentParser(description=__doc__,
+ formatter_class=RawTextHelpFormatter)
+
+ parser.add_argument("path")
+
+ return parser
+
+
+def access(path):
+ return "extroot" + path
+
+
+if __name__ == "__main__":
+ parser = get_parser()
+ args = parser.parse_args()
+
+ print("Starting query...")
+ files = Crawler.query_files(args.path)
+ print("Query done...")
+ config = db.configuration.get_config()
+ c = Crawler(food=[
+ AnalysisCFood(use_cache=True, access=access),
+ ExperimentCFood(use_cache=True, access=access),
+ PublicationCFood(use_cache=True, access=access),
+ SimulationCFood(use_cache=True, access=access),
+ ])
+ c.crawl(files)
diff --git a/integrationtests/extroot/.cerate_dir b/integrationtests/extroot/.cerate_dir
new file mode 100644
index 00000000..e69de29b
diff --git a/integrationtests/extroot/DataAnalysis/TestProject/2019-02-03/README.md b/integrationtests/extroot/DataAnalysis/TestProject/2019-02-03/README.md
new file mode 100644
index 00000000..e557f203
--- /dev/null
+++ b/integrationtests/extroot/DataAnalysis/TestProject/2019-02-03/README.md
@@ -0,0 +1,15 @@
+---
+responsible:
+- Only Responsible
+description: A description of another example analysis.
+
+data:
+- filename: "/ExperimentalData/TestProject/2019-02-03/*.dat"
+ description: an example reference to a results file
+
+scripts:
+- filename: plot.py
+ description: a plotting script
+results:
+- filename: results.pdf
+...
diff --git a/integrationtests/extroot/DataAnalysis/TestProject/2019-02-03/plot.py b/integrationtests/extroot/DataAnalysis/TestProject/2019-02-03/plot.py
new file mode 100644
index 00000000..e69de29b
diff --git a/integrationtests/extroot/DataAnalysis/TestProject/2019-02-03/results.pdf b/integrationtests/extroot/DataAnalysis/TestProject/2019-02-03/results.pdf
new file mode 100644
index 00000000..e69de29b
diff --git a/integrationtests/extroot/DataAnalysis/TestProject/2019-02-03_something/README.md b/integrationtests/extroot/DataAnalysis/TestProject/2019-02-03_something/README.md
new file mode 100644
index 00000000..c342053a
--- /dev/null
+++ b/integrationtests/extroot/DataAnalysis/TestProject/2019-02-03_something/README.md
@@ -0,0 +1,21 @@
+---
+responsible:
+- First Person
+description: A description of an example analysis.
+
+results:
+- filename: "images/*.png"
+ description: an example reference to a results file
+
+data:
+- filename: /ExperimentalData/TestProject/2019-02-03_something/use*.xlsx
+ description: an example reference to data files
+
+scripts:
+- analyse.py
+
+tags:
+- collagen
+- time sweep
+- frequency sweep
+...
diff --git a/integrationtests/extroot/DataAnalysis/TestProject/2019-02-03_something/analyse.py b/integrationtests/extroot/DataAnalysis/TestProject/2019-02-03_something/analyse.py
new file mode 100644
index 00000000..e69de29b
diff --git a/integrationtests/extroot/DataAnalysis/TestProject/2019-02-03_something/images/lol1.png b/integrationtests/extroot/DataAnalysis/TestProject/2019-02-03_something/images/lol1.png
new file mode 100644
index 00000000..e69de29b
diff --git a/integrationtests/extroot/DataAnalysis/TestProject/2019-02-03_something/images/lol2.png b/integrationtests/extroot/DataAnalysis/TestProject/2019-02-03_something/images/lol2.png
new file mode 100644
index 00000000..e69de29b
diff --git a/integrationtests/extroot/ExperimentalData/TestProject/2019-02-03/README.md b/integrationtests/extroot/ExperimentalData/TestProject/2019-02-03/README.md
new file mode 100644
index 00000000..348a6fec
--- /dev/null
+++ b/integrationtests/extroot/ExperimentalData/TestProject/2019-02-03/README.md
@@ -0,0 +1,9 @@
+---
+responsible:
+- Only Responsible
+description: A description of another example experiment.
+
+results:
+- filename: "/ExperimentalData/TestProject/2019-02-03/*.dat"
+ description: an example reference to a results file
+...
diff --git a/integrationtests/extroot/ExperimentalData/TestProject/2019-02-03/datafile.dat b/integrationtests/extroot/ExperimentalData/TestProject/2019-02-03/datafile.dat
new file mode 100644
index 00000000..e69de29b
diff --git a/integrationtests/extroot/ExperimentalData/TestProject/2019-02-03_something/README.md b/integrationtests/extroot/ExperimentalData/TestProject/2019-02-03_something/README.md
new file mode 100644
index 00000000..9e63c07c
--- /dev/null
+++ b/integrationtests/extroot/ExperimentalData/TestProject/2019-02-03_something/README.md
@@ -0,0 +1,15 @@
+---
+responsible:
+- First Person
+- Second Person
+description: A description of an example experiment.
+
+results:
+- filename: "*.xlsx"
+ description: an example reference to a results file
+
+tags:
+- collagen
+- time sweep
+- frequency sweep
+...
diff --git a/integrationtests/extroot/ExperimentalData/TestProject/2019-02-03_something/usefull.xlsx b/integrationtests/extroot/ExperimentalData/TestProject/2019-02-03_something/usefull.xlsx
new file mode 100644
index 00000000..e69de29b
diff --git a/integrationtests/extroot/ExperimentalData/TestProject/2019-02-03_something/useless.xlsx b/integrationtests/extroot/ExperimentalData/TestProject/2019-02-03_something/useless.xlsx
new file mode 100644
index 00000000..e69de29b
diff --git a/integrationtests/extroot/Publications/Posters/2019-02-03_really_cool_finding/README.md b/integrationtests/extroot/Publications/Posters/2019-02-03_really_cool_finding/README.md
new file mode 100644
index 00000000..348a6fec
--- /dev/null
+++ b/integrationtests/extroot/Publications/Posters/2019-02-03_really_cool_finding/README.md
@@ -0,0 +1,9 @@
+---
+responsible:
+- Only Responsible
+description: A description of another example experiment.
+
+results:
+- filename: "/ExperimentalData/TestProject/2019-02-03/*.dat"
+ description: an example reference to a results file
+...
diff --git a/integrationtests/extroot/Publications/Posters/2019-02-03_really_cool_finding/datafile.dat b/integrationtests/extroot/Publications/Posters/2019-02-03_really_cool_finding/datafile.dat
new file mode 100644
index 00000000..e69de29b
diff --git a/integrationtests/extroot/Publications/Theses/2019_paper_on_exciting_stuff/README.md b/integrationtests/extroot/Publications/Theses/2019_paper_on_exciting_stuff/README.md
new file mode 100644
index 00000000..9e63c07c
--- /dev/null
+++ b/integrationtests/extroot/Publications/Theses/2019_paper_on_exciting_stuff/README.md
@@ -0,0 +1,15 @@
+---
+responsible:
+- First Person
+- Second Person
+description: A description of an example experiment.
+
+results:
+- filename: "*.xlsx"
+ description: an example reference to a results file
+
+tags:
+- collagen
+- time sweep
+- frequency sweep
+...
diff --git a/integrationtests/extroot/Publications/Theses/2019_paper_on_exciting_stuff/usefull.xlsx b/integrationtests/extroot/Publications/Theses/2019_paper_on_exciting_stuff/usefull.xlsx
new file mode 100644
index 00000000..e69de29b
diff --git a/integrationtests/extroot/Publications/Theses/2019_paper_on_exciting_stuff/useless.xlsx b/integrationtests/extroot/Publications/Theses/2019_paper_on_exciting_stuff/useless.xlsx
new file mode 100644
index 00000000..e69de29b
diff --git a/integrationtests/extroot/SimulationData/TestProject/2019-02-03/README.md b/integrationtests/extroot/SimulationData/TestProject/2019-02-03/README.md
new file mode 100644
index 00000000..348a6fec
--- /dev/null
+++ b/integrationtests/extroot/SimulationData/TestProject/2019-02-03/README.md
@@ -0,0 +1,9 @@
+---
+responsible:
+- Only Responsible
+description: A description of another example experiment.
+
+results:
+- filename: "/ExperimentalData/TestProject/2019-02-03/*.dat"
+ description: an example reference to a results file
+...
diff --git a/integrationtests/extroot/SimulationData/TestProject/2019-02-03/sim.py b/integrationtests/extroot/SimulationData/TestProject/2019-02-03/sim.py
new file mode 100644
index 00000000..e69de29b
diff --git a/integrationtests/extroot/SimulationData/TestProject/2019-02-03/snapshots.dat b/integrationtests/extroot/SimulationData/TestProject/2019-02-03/snapshots.dat
new file mode 100644
index 00000000..e69de29b
diff --git a/integrationtests/extroot/SimulationData/TestProject/2019-02-03_something/README.md b/integrationtests/extroot/SimulationData/TestProject/2019-02-03_something/README.md
new file mode 100644
index 00000000..9e63c07c
--- /dev/null
+++ b/integrationtests/extroot/SimulationData/TestProject/2019-02-03_something/README.md
@@ -0,0 +1,15 @@
+---
+responsible:
+- First Person
+- Second Person
+description: A description of an example experiment.
+
+results:
+- filename: "*.xlsx"
+ description: an example reference to a results file
+
+tags:
+- collagen
+- time sweep
+- frequency sweep
+...
diff --git a/integrationtests/extroot/SimulationData/TestProject/2019-02-03_something/large_sim.py b/integrationtests/extroot/SimulationData/TestProject/2019-02-03_something/large_sim.py
new file mode 100644
index 00000000..e69de29b
diff --git a/integrationtests/extroot/SimulationData/TestProject/2019-02-03_something/parameters.p b/integrationtests/extroot/SimulationData/TestProject/2019-02-03_something/parameters.p
new file mode 100644
index 00000000..e69de29b
diff --git a/integrationtests/extroot/SimulationData/TestProject/2019-02-03_something/timeseries.npy b/integrationtests/extroot/SimulationData/TestProject/2019-02-03_something/timeseries.npy
new file mode 100644
index 00000000..e69de29b
diff --git a/integrationtests/filldb.sh b/integrationtests/filldb.sh
new file mode 100755
index 00000000..906e499c
--- /dev/null
+++ b/integrationtests/filldb.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+# fill database
+python3 -m caosadvancedtools.loadFiles /opt/caosdb/mnt/extroot/ExperimentalData
+python3 -m caosadvancedtools.loadFiles /opt/caosdb/mnt/extroot/DataAnalysis
+python3 -m caosadvancedtools.loadFiles /opt/caosdb/mnt/extroot/SimulationData
+python3 -m caosadvancedtools.loadFiles /opt/caosdb/mnt/extroot/Publications
+python3 insert_model.py
+python3 insert_record.py
+python3 crawl.py /
diff --git a/integrationtests/insert_model.py b/integrationtests/insert_model.py
new file mode 100755
index 00000000..1dcf5bdb
--- /dev/null
+++ b/integrationtests/insert_model.py
@@ -0,0 +1,5 @@
+#!/usr/bin/env python3
+from caosmodels.parser import parse_model_from_yaml
+
+model = parse_model_from_yaml("model.yml")
+model.sync_data_model(noquestion=True)
diff --git a/integrationtests/insert_record.py b/integrationtests/insert_record.py
new file mode 100644
index 00000000..5927b058
--- /dev/null
+++ b/integrationtests/insert_record.py
@@ -0,0 +1,5 @@
+import caosdb as db
+
+r = db.Record(name="TestProject")
+r.add_parent(name="Project")
+r.insert()
diff --git a/integrationtests/model.yml b/integrationtests/model.yml
new file mode 100644
index 00000000..6fd888dc
--- /dev/null
+++ b/integrationtests/model.yml
@@ -0,0 +1,56 @@
+Experiment:
+ obligatory_properties:
+ date:
+ datatype: DATETIME
+ description: 'date of the experiment'
+ identifier:
+ datatype: TEXT
+ description: 'identifier of the experiment'
+ # TODO empty recommended_properties is a problem
+ #recommended_properties:
+ responsible:
+Project:
+Person:
+ obligatory_properties:
+ firstName:
+ datatype: TEXT
+ description: 'First name of a Person.'
+ lastName:
+ datatype: TEXT
+ description: 'LastName of a Person.'
+ responsible:
+responsible:
+ datatype: REFERENCE
+results:
+ datatype: REFERENCE
+data:
+ datatype: REFERENCE
+scripts:
+ datatype: REFERENCE
+Source:
+Simulation:
+ obligatory_properties:
+ date:
+ identifier:
+ responsible:
+Analysis:
+ obligatory_properties:
+ date:
+ identifier:
+ responsible:
+Publication:
+Thesis:
+ inherit_from_suggested:
+ - Publication
+Article:
+ inherit_from_suggested:
+ - Publication
+Poster:
+ inherit_from_suggested:
+ - Publication
+Presentation:
+ inherit_from_suggested:
+ - Publication
+Report:
+ inherit_from_suggested:
+ - Publication
diff --git a/integrationtests/test.sh b/integrationtests/test.sh
new file mode 100755
index 00000000..8774500a
--- /dev/null
+++ b/integrationtests/test.sh
@@ -0,0 +1,2 @@
+./filldb.sh
+py.test-3 test_crawler.py
diff --git a/integrationtests/test_crawler.py b/integrationtests/test_crawler.py
new file mode 100755
index 00000000..5797ba42
--- /dev/null
+++ b/integrationtests/test_crawler.py
@@ -0,0 +1,114 @@
+#!/usr/bin/env python3
+import os
+import unittest
+
+import caosdb as db
+
+
+def get_entity_with_id(eid):
+ return db.execute_query("FIND "+str(eid), unique=True)
+
+
+class CrawlerTest(unittest.TestCase):
+ def test_experiment(self):
+ ########################
+ # # first experiment # #
+ ########################
+ # TODO saving an empty string as value in a text property leads to a
+ # vanishing of the property
+ # thus an x is used here. Needs to be fixed.
+ exp = db.execute_query(
+ "FIND Experiment with date=2019-02-03 and identifier='x'",
+ unique=True)
+
+ # There should be a Project with name TestProject which is referenced
+ project_id = exp.get_property("Project").value
+ project = get_entity_with_id(project_id)
+ assert project.name == "TestProject"
+ assert "Project" in [p.name for p in project.get_parents()]
+
+ # There should be a datafile as result attached with path datafile.dat
+ datfile_id = exp.get_property("results").value[0]
+ datfile = get_entity_with_id(datfile_id)
+ assert os.path.basename(datfile.path) == "datafile.dat"
+
+ #########################
+ # # second experiment # #
+ #########################
+ exp = db.execute_query(
+ "FIND Experiment with date=2019-02-03 and identifier='something'",
+ unique=True)
+
+ # Should be the same project
+ assert project_id == exp.get_property("Project").value
+
+ # Should have two data files
+ datfile_ids = exp.get_property("results").value
+ datafile_names = []
+
+ for did in datfile_ids:
+ datfile = get_entity_with_id(did)
+ datafile_names.append(os.path.basename(datfile.path))
+ assert "usefull.xlsx" in datafile_names
+ assert "useless.xlsx" in datafile_names
+
+ def test_analysis(self):
+ ######################
+ # # first analysis # #
+ ######################
+ ana = db.execute_query(
+ "FIND Analysis with date=2019-02-03 and identifier='x'",
+ unique=True)
+
+ # There should be a Project with name TestProject which is referenced
+ project_id = ana.get_property("Project").value
+ project = get_entity_with_id(project_id)
+ assert "Project" in [p.name for p in project.get_parents()]
+
+ # There should be a file as result attached with path results.pdf
+ datfile_id = ana.get_property("results").value[0]
+ datfile = get_entity_with_id(datfile_id)
+ assert os.path.basename(datfile.path) == "results.pdf"
+
+ # There should be a file as script attached with path plot.py
+ datfile_id = ana.get_property("scripts").value[0]
+ datfile = get_entity_with_id(datfile_id)
+ assert os.path.basename(datfile.path) == "plot.py"
+
+ #######################
+ # # second analysis # #
+ #######################
+ ana = db.execute_query(
+ "FIND Analysis with date=2019-02-03 and identifier='something'",
+ unique=True)
+
+ # Should be the same project
+ assert project_id == ana.get_property("Project").value
+
+ # Should have two data files
+ datfile_ids = ana.get_property("results").value
+ datafile_names = []
+
+ for did in datfile_ids:
+ datfile = get_entity_with_id(did)
+ datafile_names.append(os.path.basename(datfile.path))
+ assert "lol1.png" in datafile_names
+ assert "lol2.png" in datafile_names
+
+ # There should be a file as script attached with path plot.py
+ datfile_id = ana.get_property("scripts").value[0]
+ datfile = get_entity_with_id(datfile_id)
+ assert os.path.basename(datfile.path) == "analyse.py"
+
+ def test_publication(self):
+ #########################
+ # # first publication # #
+ #########################
+ pub = db.execute_query("FIND really_cool_finding", unique=True)
+
+ # There should be a Project with name TestProject which is referenced
+
+ ##########################
+ # # second publication # #
+ ##########################
+ pub = db.execute_query("FIND paper_on_exciting_stuff ", unique=True)
diff --git a/src/caosadvancedtools/cfood.py b/src/caosadvancedtools/cfood.py
index 23e60d8e..1f5602a0 100644
--- a/src/caosadvancedtools/cfood.py
+++ b/src/caosadvancedtools/cfood.py
@@ -31,9 +31,10 @@ from copy import deepcopy
from datetime import datetime
import caosdb as db
-from caosadvancedtools.cache import Cache
from caosdb.exceptions import TransactionError
+from caosadvancedtools.cache import Cache
+
ENTITIES = {}
@@ -107,8 +108,9 @@ class AbstractCFood(object):
for key, identifiable in entities.items():
if identifiable is None:
- print("THIS IS STRANGE. No identifiables found in {}.".format(
- crawled_file})
+ print("THIS IS STRANGE. No identifiables found in {}.".format(
+ crawled_file))
+
continue
existing = None
@@ -209,7 +211,7 @@ class AbstractCFood(object):
# is using the unique keyword
try:
r = q.execute(unique=True)
- except TransactionError as er:
+ except TransactionError:
r = None
if r is not None:
diff --git a/src/caosadvancedtools/crawler.py b/src/caosadvancedtools/crawler.py
index 79c4ee07..d908b828 100644
--- a/src/caosadvancedtools/crawler.py
+++ b/src/caosadvancedtools/crawler.py
@@ -57,5 +57,3 @@ class Crawler(object):
print("{} FILES TO BE PROCESSED.".format(len(files)))
return files
-
-
diff --git a/src/caosadvancedtools/read_md_header.py b/src/caosadvancedtools/read_md_header.py
index 95096ed3..c4c278cc 100644
--- a/src/caosadvancedtools/read_md_header.py
+++ b/src/caosadvancedtools/read_md_header.py
@@ -57,6 +57,7 @@ def _clean_header(header):
class NoValidHeader(Exception):
pass
+
def get_header(filename):
"""Open an md file identified by filename and read out the yaml
header.
diff --git a/src/programs/crawl.py b/src/programs/crawl.py
index a21d4434..47d28d6c 100755
--- a/src/programs/crawl.py
+++ b/src/programs/crawl.py
@@ -29,6 +29,7 @@ import argparse
import caosdb as db
from argparse import RawTextHelpFormatter
+
def get_parser():
parser = argparse.ArgumentParser(description=__doc__,
formatter_class=RawTextHelpFormatter)
diff --git a/src/programs/example_cfood.py b/src/programs/example_cfood.py
index 4b063457..32d1114a 100644
--- a/src/programs/example_cfood.py
+++ b/src/programs/example_cfood.py
@@ -27,6 +27,7 @@ import caosdb as db
from caosadvancedtools.cfood import AbstractCFood
+
class ExampleCFood(AbstractCFood):
def create_identifiables(self, crawled_file, match):
print("create_identifiables")
diff --git a/unittests/create_filetree.py b/unittests/create_filetree.py
index c606da9e..6f95618d 100644
--- a/unittests/create_filetree.py
+++ b/unittests/create_filetree.py
@@ -36,14 +36,14 @@ def main(folder, dry=True):
if not os.path.exists(base_path) and not dry:
os.system("mkdir -p "+base_path)
- for ii, ser in enumerate(["Series_{}".format(i) for i in range(3)]):
+ for ii, ser in enumerate(["Series_{}".format(i) for i in range(3)]):
series_path = os.path.join(base_path, ser)
- print("Series:\n"+ser +"\n")
+ print("Series:\n"+ser + "\n")
if not dry:
os.mkdir(series_path)
for date in [datetime.today()-timedelta(days=i)-timedelta(weeks=50*ii) for i in range(10)]:
#import IPython
- #IPython.embed()
+ # IPython.embed()
exp_path = os.path.join(series_path, "Exp_"+str(date.date()))
print("Exp: "+os.path.basename(exp_path))
if not dry:
@@ -62,11 +62,6 @@ def main(folder, dry=True):
os.system("touch "+d_path)
-
-
-
-
-
def get_parser():
parser = argparse.ArgumentParser(description=__doc__,
formatter_class=RawTextHelpFormatter)
diff --git a/unittests/test_cfood.py b/unittests/test_cfood.py
index 1fcf418e..8894e151 100644
--- a/unittests/test_cfood.py
+++ b/unittests/test_cfood.py
@@ -39,6 +39,7 @@ class CFoodTest(unittest.TestCase):
def test_check_existence(self):
assert AbstractCFood.find_existing(self.exp) is None
+
class CFoodTestExist(CFoodTest):
def setUp(self):
super().setUp()
@@ -46,7 +47,7 @@ class CFoodTestExist(CFoodTest):
def test_check_existence(self):
res = AbstractCFood.find_existing(self.exp)
- assert res.id == self.exp.id
+ assert res.id == self.exp.id
def tearDown(self):
self.exp.delete()
diff --git a/unittests/test_table_converter.py b/unittests/test_table_converter.py
index ca90f002..9f0d618c 100644
--- a/unittests/test_table_converter.py
+++ b/unittests/test_table_converter.py
@@ -82,6 +82,7 @@ class ToTsvTest(unittest.TestCase):
class IntegrationTest(unittest.TestCase):
""" converts tsv to a container and back and compares origin with
result """
+
def test_backandforth(self):
cont = from_tsv(TEST_TABLE, "Measurement")
tempfile = NamedTemporaryFile(delete=False)
--
GitLab