diff --git a/.docker/Dockerfile b/.docker/Dockerfile
index 914e5f61138a664784ef1f8f323c101ff5f1c08a..d8c975783da47cc975fd071dfef604223f4e4fe5 100644
--- a/.docker/Dockerfile
+++ b/.docker/Dockerfile
@@ -30,7 +30,7 @@ RUN git clone \
cd scifolder && pip3 install .
COPY . /git
RUN rm -r /git/.git \
- && mv /git/.docker/pycaosdb.ini /git/integrationtests/full_test
+ && mv /git/.docker/pycaosdb.ini /git/integrationtests
RUN cd /git && pip3 install .
-WORKDIR /git/integrationtests/full_test
+WORKDIR /git/integrationtests
CMD /wait-for-it.sh caosdb-server:10443 -t 500 -- ./test.sh
diff --git a/.docker/docker-compose.yml b/.docker/docker-compose.yml
index 74a00dc343db24bd4fe28f6f02eab732d20f83da..e859e4337653a41dd2e17a819760b18fe2185c5e 100644
--- a/.docker/docker-compose.yml
+++ b/.docker/docker-compose.yml
@@ -18,7 +18,7 @@ services:
source: "$EXEPATH/.docker/cert"
target: /opt/caosdb/cert
- type: bind
- source: "$EXEPATH/integrationtests/full_test/extroot"
+ source: "$EXEPATH/integrationtests/extroot"
target: /opt/caosdb/mnt/extroot
read_only: true
ports:
diff --git a/README_SETUP.md b/README_SETUP.md
index 526fe900bc7896f24289d506757d4e3a8c195f59..1400ac5e7428db0143b5180228b097dfbd661f78 100644
--- a/README_SETUP.md
+++ b/README_SETUP.md
@@ -3,16 +3,29 @@
pip install . --user
pip install tox --user
+In order to run the tests you need to install the [scifolder
+package](https://gitlab.com/henrik_indiscale/scifolder) by Henrik tom
+Wörden.
+
# Run Unit Tests
tox
# Run Integration Tests Locally
-1. Change directory to `integrationtests/full_test/`.
-2. Mount `extroot` to the folder that will be used as extroot. E.g. `sudo mount
- -o bind extroot ../../../caosdb-deploy/profiles/empty/paths/extroot`.
-3. Start an empty CaosDB instance (with the mounted extroot).
+1. Change directory to `integrationtests/`.
+2. Mount `extroot` to the folder that will be used as
+ extroot. E.g. `sudo mount -o bind extroot
+ ../../caosdb-deploy/profiles/empty/paths/extroot` (or whatever path
+ the extroot of the empty profile to be used is located at).
+3. Start an empty (!) CaosDB instance (with the mounted extroot). The
+ database will be cleared during testing, so it's important to use
+ an empty insctance.
4. Run `test.sh`.
# Code Formatting
autopep8 -i -r ./
+
+# Dependencies
+
+test:
+- scifolder package from https://gitlab.com/henrik_indiscale/scifolder
diff --git a/integrationtests/clear_database.py b/integrationtests/clear_database.py
new file mode 100644
index 0000000000000000000000000000000000000000..138cf4e6abb256d5710cd2b32f55a1fb51f3fbed
--- /dev/null
+++ b/integrationtests/clear_database.py
@@ -0,0 +1,46 @@
+# -*- coding: utf-8 -*-
+#
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2020 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2020 Florian Spreckelsen <f.spreckelsen@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public
+# License along with this program. If not, see
+# <https://www.gnu.org/licenses/>.
+#
+# ** end header
+#
+"""Clear the database before and after the integration tests."""
+import caosdb as db
+
+
+def clear_all():
+ """First remove Records, then RecordTypes, then Properties, finally
+ files. Since there may be no entities, execute all deletions
+ without raising errors.
+
+ """
+ db.execute_query("FIND Record").delete(
+ raise_exception_on_error=False)
+ db.execute_query("FIND RecordType").delete(
+ raise_exception_on_error=False)
+ db.execute_query("FIND Property").delete(
+ raise_exception_on_error=False)
+ db.execute_query("FIND File").delete(
+ raise_exception_on_error=False)
+
+
+if __name__ == "__main__":
+ clear_all()
diff --git a/integrationtests/full_test/crawl.py b/integrationtests/crawl.py
similarity index 100%
rename from integrationtests/full_test/crawl.py
rename to integrationtests/crawl.py
diff --git a/integrationtests/full_test/example_table.csv b/integrationtests/example_table.csv
similarity index 100%
rename from integrationtests/full_test/example_table.csv
rename to integrationtests/example_table.csv
diff --git a/integrationtests/full_test/extroot/.cerate_dir b/integrationtests/extroot/.cerate_dir
similarity index 100%
rename from integrationtests/full_test/extroot/.cerate_dir
rename to integrationtests/extroot/.cerate_dir
diff --git a/integrationtests/full_test/extroot/DataAnalysis/2010_TestProject/2019-02-03/README.md b/integrationtests/extroot/DataAnalysis/2010_TestProject/2019-02-03/README.md
similarity index 100%
rename from integrationtests/full_test/extroot/DataAnalysis/2010_TestProject/2019-02-03/README.md
rename to integrationtests/extroot/DataAnalysis/2010_TestProject/2019-02-03/README.md
diff --git a/integrationtests/full_test/extroot/DataAnalysis/2010_TestProject/2019-02-03/plot.py b/integrationtests/extroot/DataAnalysis/2010_TestProject/2019-02-03/plot.py
similarity index 100%
rename from integrationtests/full_test/extroot/DataAnalysis/2010_TestProject/2019-02-03/plot.py
rename to integrationtests/extroot/DataAnalysis/2010_TestProject/2019-02-03/plot.py
diff --git a/integrationtests/full_test/extroot/DataAnalysis/2010_TestProject/2019-02-03/results.pdf b/integrationtests/extroot/DataAnalysis/2010_TestProject/2019-02-03/results.pdf
similarity index 100%
rename from integrationtests/full_test/extroot/DataAnalysis/2010_TestProject/2019-02-03/results.pdf
rename to integrationtests/extroot/DataAnalysis/2010_TestProject/2019-02-03/results.pdf
diff --git a/integrationtests/full_test/extroot/DataAnalysis/2010_TestProject/2019-02-03_something/README.md b/integrationtests/extroot/DataAnalysis/2010_TestProject/2019-02-03_something/README.md
similarity index 100%
rename from integrationtests/full_test/extroot/DataAnalysis/2010_TestProject/2019-02-03_something/README.md
rename to integrationtests/extroot/DataAnalysis/2010_TestProject/2019-02-03_something/README.md
diff --git a/integrationtests/full_test/extroot/DataAnalysis/2010_TestProject/2019-02-03_something/analyse.py b/integrationtests/extroot/DataAnalysis/2010_TestProject/2019-02-03_something/analyse.py
similarity index 100%
rename from integrationtests/full_test/extroot/DataAnalysis/2010_TestProject/2019-02-03_something/analyse.py
rename to integrationtests/extroot/DataAnalysis/2010_TestProject/2019-02-03_something/analyse.py
diff --git a/integrationtests/full_test/extroot/DataAnalysis/2010_TestProject/2019-02-03_something/images/lol1.png b/integrationtests/extroot/DataAnalysis/2010_TestProject/2019-02-03_something/images/lol1.png
similarity index 100%
rename from integrationtests/full_test/extroot/DataAnalysis/2010_TestProject/2019-02-03_something/images/lol1.png
rename to integrationtests/extroot/DataAnalysis/2010_TestProject/2019-02-03_something/images/lol1.png
diff --git a/integrationtests/full_test/extroot/DataAnalysis/2010_TestProject/2019-02-03_something/images/lol2.png b/integrationtests/extroot/DataAnalysis/2010_TestProject/2019-02-03_something/images/lol2.png
similarity index 100%
rename from integrationtests/full_test/extroot/DataAnalysis/2010_TestProject/2019-02-03_something/images/lol2.png
rename to integrationtests/extroot/DataAnalysis/2010_TestProject/2019-02-03_something/images/lol2.png
diff --git a/integrationtests/full_test/extroot/ExperimentalData/2010_TestProject/2019-02-03/README.md b/integrationtests/extroot/ExperimentalData/2010_TestProject/2019-02-03/README.md
similarity index 100%
rename from integrationtests/full_test/extroot/ExperimentalData/2010_TestProject/2019-02-03/README.md
rename to integrationtests/extroot/ExperimentalData/2010_TestProject/2019-02-03/README.md
diff --git a/integrationtests/full_test/extroot/ExperimentalData/2010_TestProject/2019-02-03/datafile.dat b/integrationtests/extroot/ExperimentalData/2010_TestProject/2019-02-03/datafile.dat
similarity index 100%
rename from integrationtests/full_test/extroot/ExperimentalData/2010_TestProject/2019-02-03/datafile.dat
rename to integrationtests/extroot/ExperimentalData/2010_TestProject/2019-02-03/datafile.dat
diff --git a/integrationtests/full_test/extroot/ExperimentalData/2010_TestProject/2019-02-03/not_treated.file b/integrationtests/extroot/ExperimentalData/2010_TestProject/2019-02-03/not_treated.file
similarity index 100%
rename from integrationtests/full_test/extroot/ExperimentalData/2010_TestProject/2019-02-03/not_treated.file
rename to integrationtests/extroot/ExperimentalData/2010_TestProject/2019-02-03/not_treated.file
diff --git a/integrationtests/full_test/extroot/ExperimentalData/2010_TestProject/2019-02-03_something/README.md b/integrationtests/extroot/ExperimentalData/2010_TestProject/2019-02-03_something/README.md
similarity index 100%
rename from integrationtests/full_test/extroot/ExperimentalData/2010_TestProject/2019-02-03_something/README.md
rename to integrationtests/extroot/ExperimentalData/2010_TestProject/2019-02-03_something/README.md
diff --git a/integrationtests/full_test/extroot/ExperimentalData/2010_TestProject/2019-02-03_something/usefull.xlsx b/integrationtests/extroot/ExperimentalData/2010_TestProject/2019-02-03_something/usefull.xlsx
similarity index 100%
rename from integrationtests/full_test/extroot/ExperimentalData/2010_TestProject/2019-02-03_something/usefull.xlsx
rename to integrationtests/extroot/ExperimentalData/2010_TestProject/2019-02-03_something/usefull.xlsx
diff --git a/integrationtests/full_test/extroot/ExperimentalData/2010_TestProject/2019-02-03_something/useless.xlsx b/integrationtests/extroot/ExperimentalData/2010_TestProject/2019-02-03_something/useless.xlsx
similarity index 100%
rename from integrationtests/full_test/extroot/ExperimentalData/2010_TestProject/2019-02-03_something/useless.xlsx
rename to integrationtests/extroot/ExperimentalData/2010_TestProject/2019-02-03_something/useless.xlsx
diff --git a/integrationtests/full_test/extroot/Publications/Posters/2019-02-03_really_cool_finding/README.md b/integrationtests/extroot/Publications/Posters/2019-02-03_really_cool_finding/README.md
similarity index 100%
rename from integrationtests/full_test/extroot/Publications/Posters/2019-02-03_really_cool_finding/README.md
rename to integrationtests/extroot/Publications/Posters/2019-02-03_really_cool_finding/README.md
diff --git a/integrationtests/full_test/extroot/Publications/Posters/2019-02-03_really_cool_finding/poster.pdf b/integrationtests/extroot/Publications/Posters/2019-02-03_really_cool_finding/poster.pdf
similarity index 100%
rename from integrationtests/full_test/extroot/Publications/Posters/2019-02-03_really_cool_finding/poster.pdf
rename to integrationtests/extroot/Publications/Posters/2019-02-03_really_cool_finding/poster.pdf
diff --git a/integrationtests/full_test/extroot/Publications/Theses/2019_paper_on_exciting_stuff/README.md b/integrationtests/extroot/Publications/Theses/2019_paper_on_exciting_stuff/README.md
similarity index 100%
rename from integrationtests/full_test/extroot/Publications/Theses/2019_paper_on_exciting_stuff/README.md
rename to integrationtests/extroot/Publications/Theses/2019_paper_on_exciting_stuff/README.md
diff --git a/integrationtests/full_test/extroot/SimulationData/2010_TestProject/2019-02-03/README.md b/integrationtests/extroot/SimulationData/2010_TestProject/2019-02-03/README.md
similarity index 100%
rename from integrationtests/full_test/extroot/SimulationData/2010_TestProject/2019-02-03/README.md
rename to integrationtests/extroot/SimulationData/2010_TestProject/2019-02-03/README.md
diff --git a/integrationtests/full_test/extroot/SimulationData/2010_TestProject/2019-02-03/sim.py b/integrationtests/extroot/SimulationData/2010_TestProject/2019-02-03/sim.py
similarity index 100%
rename from integrationtests/full_test/extroot/SimulationData/2010_TestProject/2019-02-03/sim.py
rename to integrationtests/extroot/SimulationData/2010_TestProject/2019-02-03/sim.py
diff --git a/integrationtests/full_test/extroot/SimulationData/2010_TestProject/2019-02-03/snapshots.dat b/integrationtests/extroot/SimulationData/2010_TestProject/2019-02-03/snapshots.dat
similarity index 100%
rename from integrationtests/full_test/extroot/SimulationData/2010_TestProject/2019-02-03/snapshots.dat
rename to integrationtests/extroot/SimulationData/2010_TestProject/2019-02-03/snapshots.dat
diff --git a/integrationtests/full_test/extroot/SimulationData/2010_TestProject/2019-02-03_something/README.md b/integrationtests/extroot/SimulationData/2010_TestProject/2019-02-03_something/README.md
similarity index 100%
rename from integrationtests/full_test/extroot/SimulationData/2010_TestProject/2019-02-03_something/README.md
rename to integrationtests/extroot/SimulationData/2010_TestProject/2019-02-03_something/README.md
diff --git a/integrationtests/full_test/extroot/SimulationData/2010_TestProject/2019-02-03_something/large_sim.py b/integrationtests/extroot/SimulationData/2010_TestProject/2019-02-03_something/large_sim.py
similarity index 100%
rename from integrationtests/full_test/extroot/SimulationData/2010_TestProject/2019-02-03_something/large_sim.py
rename to integrationtests/extroot/SimulationData/2010_TestProject/2019-02-03_something/large_sim.py
diff --git a/integrationtests/full_test/extroot/SimulationData/2010_TestProject/2019-02-03_something/parameters.p b/integrationtests/extroot/SimulationData/2010_TestProject/2019-02-03_something/parameters.p
similarity index 100%
rename from integrationtests/full_test/extroot/SimulationData/2010_TestProject/2019-02-03_something/parameters.p
rename to integrationtests/extroot/SimulationData/2010_TestProject/2019-02-03_something/parameters.p
diff --git a/integrationtests/full_test/extroot/SimulationData/2010_TestProject/2019-02-03_something/timeseries.npy b/integrationtests/extroot/SimulationData/2010_TestProject/2019-02-03_something/timeseries.npy
similarity index 100%
rename from integrationtests/full_test/extroot/SimulationData/2010_TestProject/2019-02-03_something/timeseries.npy
rename to integrationtests/extroot/SimulationData/2010_TestProject/2019-02-03_something/timeseries.npy
diff --git a/integrationtests/full_test/filldb.sh b/integrationtests/filldb.sh
similarity index 100%
rename from integrationtests/full_test/filldb.sh
rename to integrationtests/filldb.sh
diff --git a/integrationtests/full_test/insert_model.py b/integrationtests/insert_model.py
similarity index 100%
rename from integrationtests/full_test/insert_model.py
rename to integrationtests/insert_model.py
diff --git a/integrationtests/full_test/model.yml b/integrationtests/model.yml
similarity index 100%
rename from integrationtests/full_test/model.yml
rename to integrationtests/model.yml
diff --git a/integrationtests/single_tests/test_cfood.py b/integrationtests/single_tests/test_cfood.py
deleted file mode 100644
index a489b9b68a028ce273c20b44d3545b973d54c0b8..0000000000000000000000000000000000000000
--- a/integrationtests/single_tests/test_cfood.py
+++ /dev/null
@@ -1,37 +0,0 @@
-#!/usr/bin/env python
-# encoding: utf-8
-#
-# ** header v3.0
-# This file is a part of the CaosDB Project.
-#
-# Copyright (C) 2018 Research Group Biomedical Physics,
-# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <https://www.gnu.org/licenses/>.
-#
-# ** end header
-import unittest
-from tempfile import NamedTemporaryFile
-
-import caosdb as db
-
-from caosadvancedtools.cfood import AbstractCFood
-
-
-class CFoodTest(unittest.TestCase):
- def setUp(self):
- pass
-
- def test_check_existence(self):
- pass
diff --git a/integrationtests/single_tests/test_crawler.py b/integrationtests/single_tests/test_crawler.py
deleted file mode 100644
index 9e5f4905f286ea3fe53d69c88adf4cd9b82c7690..0000000000000000000000000000000000000000
--- a/integrationtests/single_tests/test_crawler.py
+++ /dev/null
@@ -1,108 +0,0 @@
-#!/usr/bin/env python
-# encoding: utf-8
-#
-# This file is a part of the CaosDB Project.
-#
-# Copyright (C) 2018 Research Group Biomedical Physics,
-# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as
-# published by the Free Software Foundation, either version 3 of the
-# License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program. If not, see <https://www.gnu.org/licenses/>.
-#
-import unittest
-from copy import deepcopy
-from tempfile import NamedTemporaryFile
-
-import caosdb as db
-
-from caosadvancedtools.crawler import Crawler
-
-
-def seek_and_destroy(names):
- for name in names:
- db.execute_query("FIND "+name).delete(raise_exception_on_error=False)
-
-
-class CrawlerTest(unittest.TestCase):
- def setUp(self):
- # TODO replace by something more reasonable
- seek_and_destroy(["Experiment", "Analysis", "Publication", "species"])
- self.rts = db.Container().extend([
- db.RecordType(name="Experiment").insert(),
- db.RecordType(name="Analysis").insert(),
- db.RecordType(name="Publication").insert(),
- db.Property(name="species", datatype=db.TEXT).insert(),
- ])
- self.exp = db.Record()
- self.exp.add_parent(name="Experiment")
- self.exp.add_property(name="species", value="microunicorn")
- self.ana = db.Record()
- self.ana.add_parent(name="Analysis")
- self.pub = db.Record()
- self.pub.add_parent(name="Publication")
-
- def test_check_existence(self):
- assert Crawler.find_existing(self.exp) is None
-
- def test_find_or_insert_identifiables(self):
- tmpexp = db.Record()
- tmpexp.add_parent(name="Experiment")
- tmpexp.add_property(name="species", value="microunicorn")
- tmpana = db.Record()
- tmpana.add_parent(name="Analysis")
- tmpexp.insert()
- tmpana.insert()
- self.ana.id = tmpana.id
- # exp inserted/no id; ana inserted/id; pub missing
- identifiables = db.Container().extend([self.exp, self.ana, self.pub])
- old_id = id(identifiables[0])
- reference_to_first = identifiables[0]
- assert reference_to_first is identifiables[0]
- Crawler.find_or_insert_identifiables(identifiables)
-
- for el in identifiables:
- assert el.is_valid()
-
- # check whether instance is the same
- assert reference_to_first is identifiables[0]
- assert old_id == id(identifiables[0])
- # order must not be changed
- assert identifiables[0].get_parents()[0].name == "Experiment"
- assert identifiables[1].get_parents()[0].name == "Analysis"
- assert identifiables[2].get_parents()[0].name == "Publication"
-
- def tearDown(self):
- for el in [self.exp, self.ana, self.pub, self.rts]:
- try:
- el.delete()
- except BaseException:
- pass
-
-
-class CrawlerTestExist(CrawlerTest):
- def setUp(self):
- super().setUp()
- self.exp.insert()
- self.ana.insert()
- self.pub.insert()
-
- def test_check_existence(self):
- res = Crawler.find_existing(self.exp)
- assert res.id == self.exp.id
-
- def tearDown(self):
- for el in [self.exp, self.ana, self.pub, self.rts]:
- try:
- el.delete()
- except BaseException:
- pass
diff --git a/integrationtests/full_test/test.sh b/integrationtests/test.sh
similarity index 75%
rename from integrationtests/full_test/test.sh
rename to integrationtests/test.sh
index ccaaad5e7d867eaa3319e158f5eaefbdedd40c85..9712e0e7db654c1d7a508e740b936800363e4b8e 100755
--- a/integrationtests/full_test/test.sh
+++ b/integrationtests/test.sh
@@ -2,10 +2,15 @@
OUT=/tmp/crawler.output
ls
rm -rf cache.db
+echo "Clearing database"
+python3 clear_database.py
+echo "Testing crawler without cfoods"
+python3 -m pytest test_crawler_basics.py
+
echo "Filling the database"
./filldb.sh
echo "Testing the crawler database"
-python3 -m pytest test_crawler.py
+python3 -m pytest test_crawler_with_cfoods.py
echo "make a change"
pushd extroot
egrep -liRZ 'A description of another example' . | xargs -0 -l sed -i -e 's/A description of another example/A description of this example/g'
@@ -15,8 +20,8 @@ echo "run crawler"
# check whether there was something UNAUTHORIZED
set -e
grep "UNAUTHORIZED UPDATE" $OUT
-# get the id of the run
-RUN_ID=$(grep "./crawl.py -a " $OUT | awk '{ print $3 }')
+# get the id of the run which is the last field of the output string
+RUN_ID=$(grep "run id:" $OUT | awk '{ print $NF }')
echo $RUN_ID
echo "run crawler again"
echo "./crawl.py -a $RUN_ID /"
@@ -35,9 +40,12 @@ python3 test_table.py
# TODO the following test deletes lots of the data inserted by the crawler
echo "Testing im and export"
python3 test_im_und_export.py
+# Better safe than sorry:
+python3 clear_database.py
# Test correct display of data model errors:
echo "Testing recognition of data model problems ... "
+python3 -m pytest test_datamodel_problems.py
python3 -m pytest test_crawl_with_datamodel_problems.py
# Obsolete due to teardown in the above test.
diff --git a/integrationtests/full_test/test_crawl_with_datamodel_problems.py b/integrationtests/test_crawl_with_datamodel_problems.py
similarity index 100%
rename from integrationtests/full_test/test_crawl_with_datamodel_problems.py
rename to integrationtests/test_crawl_with_datamodel_problems.py
diff --git a/integrationtests/test_crawler_basics.py b/integrationtests/test_crawler_basics.py
new file mode 100644
index 0000000000000000000000000000000000000000..85fca282c8546ad1e7f6a708a2eaf46e374a528f
--- /dev/null
+++ b/integrationtests/test_crawler_basics.py
@@ -0,0 +1,130 @@
+#!/usr/bin/env python
+# encoding: utf-8
+#
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2018 Research Group Biomedical Physics,
+# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+# Copyright (C) 2020 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2020 Florian Spreckelsen <f.spreckelsen@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
+"""Check basic crawler functionality without actually using
+cfoods. This is tested in test_crawler_with_cfoods.py.
+
+"""
+import unittest
+
+import caosdb as db
+
+from caosadvancedtools.crawler import Crawler
+from caosadvancedtools.guard import INSERT
+from caosadvancedtools.guard import global_guard as guard
+
+
+def setup_module():
+ """Clear all test entities. Allow insertions."""
+ guard.set_level(INSERT)
+ try:
+ db.execute_query("FIND Test*").delete()
+ except Exception:
+ pass
+
+
+class CrawlerTest(unittest.TestCase):
+ def setUp(self):
+ """Clear possibly existing entities and create the necessary Records,
+ RecordTypes, and Properties.
+
+ """
+ setup_module()
+ self.rts = db.Container().extend([
+ db.RecordType(name="Test_Type_1").insert(),
+ db.RecordType(name="Test_Type_2").insert(),
+ db.RecordType(name="Test_Type_3").insert(),
+ db.Property(name="Test_Prop", datatype=db.TEXT).insert(),
+ ])
+ self.rec1 = db.Record()
+ self.rec1.add_parent(name="Test_Type_1")
+ self.rec1.add_property(name="Test_Prop", value="Test")
+ self.rec2 = db.Record()
+ self.rec2.add_parent(name="Test_Type_2")
+ self.rec3 = db.Record()
+ self.rec3.add_parent(name="Test_Type_3")
+
+ def test_check_existence(self):
+ # This hasn't been inserted yet:
+ assert Crawler.find_existing(self.rec1) is None
+
+ def test_find_or_insert_identifiables(self):
+ """Create identical Records that have to be identified by the
+ crawler.
+
+ """
+ same_as_rec1 = db.Record()
+ same_as_rec1.add_parent(name="Test_Type_1")
+ same_as_rec1.add_property(name="Test_Prop", value="Test")
+ same_as_rec2 = db.Record()
+ same_as_rec2.add_parent(name="Test_Type_2")
+ same_as_rec1.insert()
+ same_as_rec2.insert()
+ # copy the id
+ self.rec2.id = same_as_rec2.id
+ # Insert rec1, rec2, and rec3. rec1 already exists in the
+ # database but doesn't have an id yet, rec2 exists, rec3 is
+ # missing entirely.
+ identifiables = db.Container().extend(
+ [self.rec1, self.rec2, self.rec3])
+ old_id = id(identifiables[0])
+ reference_to_first = identifiables[0]
+ assert reference_to_first is identifiables[0]
+ Crawler.find_or_insert_identifiables(identifiables)
+
+ for el in identifiables:
+ assert el.is_valid()
+
+ # check whether instance is the same
+ assert reference_to_first is identifiables[0]
+ assert old_id == id(identifiables[0])
+ # order must not be changed
+ assert identifiables[0].get_parents()[0].name == "Test_Type_1"
+ assert identifiables[1].get_parents()[0].name == "Test_Type_2"
+ assert identifiables[2].get_parents()[0].name == "Test_Type_3"
+
+ def tearDown(self):
+ setup_module()
+ # Delete nameless entities
+ for el in [self.rec1, self.rec2, self.rec3]:
+ try:
+ el.delete()
+ except BaseException:
+ pass
+
+
+class CrawlerTestExist(CrawlerTest):
+ """Same tests as above, but now insert Record during setup."""
+
+ def setUp(self):
+ super().setUp()
+ self.rec1.insert()
+ self.rec2.insert()
+ self.rec3.insert()
+
+ def test_check_existence(self):
+ # Now this exists
+ res = Crawler.find_existing(self.rec1)
+ assert res.id == self.rec1.id
diff --git a/integrationtests/full_test/test_crawler.py b/integrationtests/test_crawler_with_cfoods.py
similarity index 90%
rename from integrationtests/full_test/test_crawler.py
rename to integrationtests/test_crawler_with_cfoods.py
index cf399bfb59cd194e38f8df81f162ac01e52f296d..cc6f06771c48b41cdbba928051e0620e60586587 100755
--- a/integrationtests/full_test/test_crawler.py
+++ b/integrationtests/test_crawler_with_cfoods.py
@@ -1,4 +1,27 @@
#!/usr/bin/env python3
+# encoding: utf-8
+#
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2018 Research Group Biomedical Physics,
+# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+# Copyright (C) 2019,2020 Indiscale GmbH <info@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
import os
import unittest
diff --git a/integrationtests/single_tests/test_datamodel_problems.py b/integrationtests/test_datamodel_problems.py
similarity index 98%
rename from integrationtests/single_tests/test_datamodel_problems.py
rename to integrationtests/test_datamodel_problems.py
index 93a403d87b6411390108ace5378f1bf3223ec20e..d9af69c79007bae5212f16d18363ff18b4ba9d32 100644
--- a/integrationtests/single_tests/test_datamodel_problems.py
+++ b/integrationtests/test_datamodel_problems.py
@@ -4,7 +4,7 @@
# ** header v3.0
# This file is a part of the CaosDB Project.
#
-# Copyright (C) Indiscale, GmbH <info@indiscale.com>
+# Copyright (C) 2020 Indiscale GmbH <info@indiscale.com>
# Copyright (C) 2020 Florian Spreckelsen <f.spreckelsen@indiscale.com>
#
# This program is free software: you can redistribute it and/or modify
diff --git a/integrationtests/full_test/test_im_und_export.py b/integrationtests/test_im_und_export.py
similarity index 100%
rename from integrationtests/full_test/test_im_und_export.py
rename to integrationtests/test_im_und_export.py
diff --git a/integrationtests/full_test/test_table.py b/integrationtests/test_table.py
similarity index 100%
rename from integrationtests/full_test/test_table.py
rename to integrationtests/test_table.py