Merge branch 'f-clean-up-tests' into 'dev'

Clean up and join integration tests

See merge request caosdb/caosdb-advanced-user-tools!27

integrationtests/full_test/test.sh → integrationtests/test.sh

@@ -2,10 +2,15 @@
 OUT=/tmp/crawler.output
 ls 
 rm -rf cache.db
+echo "Clearing database"
+python3 clear_database.py
+echo "Testing crawler without cfoods"
+python3 -m pytest test_crawler_basics.py
+
 echo "Filling the database"
 ./filldb.sh
 echo "Testing the crawler database"
-python3 -m pytest test_crawler.py
+python3 -m pytest test_crawler_with_cfoods.py
 echo "make a change"
 pushd extroot
 egrep -liRZ 'A description of another example' . | xargs -0 -l sed -i -e 's/A description of another example/A description of this example/g'
@@ -15,8 +20,8 @@ echo "run crawler"
 # check whether there was something UNAUTHORIZED
 set -e
 grep "UNAUTHORIZED UPDATE" $OUT
-# get the id of the run
-RUN_ID=$(grep "./crawl.py -a " $OUT | awk '{ print $3 }')
+# get the id of the run which is the last field of the output string
+RUN_ID=$(grep "run id:" $OUT | awk '{ print $NF }')
 echo $RUN_ID
 echo "run crawler again"
 echo "./crawl.py -a $RUN_ID /"
@@ -35,9 +40,12 @@ python3 test_table.py
 # TODO the following test deletes lots of the data inserted by the crawler
 echo "Testing im and export"
 python3 test_im_und_export.py
+# Better safe than sorry:
+python3 clear_database.py
 
 # Test correct display of data model errors:
 echo "Testing recognition of data model problems ... "
+python3 -m pytest test_datamodel_problems.py
 python3 -m pytest test_crawl_with_datamodel_problems.py
 
 # Obsolete due to teardown in the above test.

integrationtests/single_tests/test_crawler.py → integrationtests/test_crawler_basics.py

 #!/usr/bin/env python
 # encoding: utf-8
 #
+# ** header v3.0
 # This file is a part of the CaosDB Project.
 #
 # Copyright (C) 2018 Research Group Biomedical Physics,
 # Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+# Copyright (C) 2020 Indiscale GmbH <info@indiscale.com>
+# Copyright (C) 2020 Florian Spreckelsen <f.spreckelsen@indiscale.com>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as
@@ -19,52 +22,73 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program. If not, see <https://www.gnu.org/licenses/>.
 #
+# ** end header
+"""Check basic crawler functionality without actually using
+cfoods. This is tested in test_crawler_with_cfoods.py.
+
+"""
 import unittest
-from copy import deepcopy
-from tempfile import NamedTemporaryFile
 
 import caosdb as db
 
 from caosadvancedtools.crawler import Crawler
+from caosadvancedtools.guard import INSERT
+from caosadvancedtools.guard import global_guard as guard
 
 
-def seek_and_destroy(names):
-    for name in names:
-        db.execute_query("FIND "+name).delete(raise_exception_on_error=False)
+def setup_module():
+    """Clear all test entities.  Allow insertions."""
+    guard.set_level(INSERT)
+    try:
+        db.execute_query("FIND Test*").delete()
+    except Exception:
+        pass
 
 
 class CrawlerTest(unittest.TestCase):
     def setUp(self):
-        # TODO replace by something more reasonable
-        seek_and_destroy(["Experiment", "Analysis", "Publication", "species"])
+        """Clear possibly existing entities and create the necessary Records,
+        RecordTypes, and Properties.
+
+        """
+        setup_module()
         self.rts = db.Container().extend([
-            db.RecordType(name="Experiment").insert(),
-            db.RecordType(name="Analysis").insert(),
-            db.RecordType(name="Publication").insert(),
-            db.Property(name="species", datatype=db.TEXT).insert(),
+            db.RecordType(name="Test_Type_1").insert(),
+            db.RecordType(name="Test_Type_2").insert(),
+            db.RecordType(name="Test_Type_3").insert(),
+            db.Property(name="Test_Prop", datatype=db.TEXT).insert(),
         ])
-        self.exp = db.Record()
-        self.exp.add_parent(name="Experiment")
-        self.exp.add_property(name="species", value="microunicorn")
-        self.ana = db.Record()
-        self.ana.add_parent(name="Analysis")
-        self.pub = db.Record()
-        self.pub.add_parent(name="Publication")
+        self.rec1 = db.Record()
+        self.rec1.add_parent(name="Test_Type_1")
+        self.rec1.add_property(name="Test_Prop", value="Test")
+        self.rec2 = db.Record()
+        self.rec2.add_parent(name="Test_Type_2")
+        self.rec3 = db.Record()
+        self.rec3.add_parent(name="Test_Type_3")
 
     def test_check_existence(self):
-        assert Crawler.find_existing(self.exp) is None
+        # This hasn't been inserted yet:
+        assert Crawler.find_existing(self.rec1) is None
 
     def test_find_or_insert_identifiables(self):
-        tmpexp = db.Record()
-        tmpexp.add_parent(name="Experiment")
-        tmpexp.add_property(name="species", value="microunicorn")
-        tmpana = db.Record()
-        tmpana.add_parent(name="Analysis")
-        tmpexp.insert()
-        tmpana.insert()
-        self.ana.id = tmpana.id
-        # exp inserted/no id; ana inserted/id; pub missing
-        identifiables = db.Container().extend([self.exp, self.ana, self.pub])
+        """Create identical Records that have to be identified by the
+        crawler.
+
+        """
+        same_as_rec1 = db.Record()
+        same_as_rec1.add_parent(name="Test_Type_1")
+        same_as_rec1.add_property(name="Test_Prop", value="Test")
+        same_as_rec2 = db.Record()
+        same_as_rec2.add_parent(name="Test_Type_2")
+        same_as_rec1.insert()
+        same_as_rec2.insert()
+        # copy the id
+        self.rec2.id = same_as_rec2.id
+        # Insert rec1, rec2, and rec3. rec1 already exists in the
+        # database but doesn't have an id yet, rec2 exists, rec3 is
+        # missing entirely.
+        identifiables = db.Container().extend(
+            [self.rec1, self.rec2, self.rec3])
         old_id = id(identifiables[0])
         reference_to_first = identifiables[0]
         assert reference_to_first is identifiables[0]
@@ -77,12 +101,14 @@ class CrawlerTest(unittest.TestCase):
         assert reference_to_first is identifiables[0]
         assert old_id == id(identifiables[0])
         # order must not be changed
-        assert identifiables[0].get_parents()[0].name == "Experiment"
-        assert identifiables[1].get_parents()[0].name == "Analysis"
-        assert identifiables[2].get_parents()[0].name == "Publication"
+        assert identifiables[0].get_parents()[0].name == "Test_Type_1"
+        assert identifiables[1].get_parents()[0].name == "Test_Type_2"
+        assert identifiables[2].get_parents()[0].name == "Test_Type_3"
 
     def tearDown(self):
-        for el in [self.exp, self.ana, self.pub, self.rts]:
+        setup_module()
+        # Delete nameless entities
+        for el in [self.rec1, self.rec2, self.rec3]:
             try:
                 el.delete()
             except BaseException:
@@ -90,19 +116,15 @@ class CrawlerTest(unittest.TestCase):
 
 
 class CrawlerTestExist(CrawlerTest):
+    """Same tests as above, but now insert Record during setup."""
+
     def setUp(self):
         super().setUp()
-        self.exp.insert()
-        self.ana.insert()
-        self.pub.insert()
+        self.rec1.insert()
+        self.rec2.insert()
+        self.rec3.insert()
 
     def test_check_existence(self):
-        res = Crawler.find_existing(self.exp)
-        assert res.id == self.exp.id
-
-    def tearDown(self):
-        for el in [self.exp, self.ana, self.pub, self.rts]:
-            try:
-                el.delete()
-            except BaseException:
-                pass
+        # Now this exists
+        res = Crawler.find_existing(self.rec1)
+        assert res.id == self.rec1.id

integrationtests/full_test/test_crawler.py → integrationtests/test_crawler_with_cfoods.py

 #!/usr/bin/env python3
+# encoding: utf-8
+#
+# ** header v3.0
+# This file is a part of the CaosDB Project.
+#
+# Copyright (C) 2018 Research Group Biomedical Physics,
+# Max-Planck-Institute for Dynamics and Self-Organization Göttingen
+# Copyright (C) 2019,2020 Indiscale GmbH <info@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+# ** end header
 import os
 import unittest
 

integrationtests/single_tests/test_datamodel_problems.py → integrationtests/test_datamodel_problems.py

@@ -4,7 +4,7 @@
 # ** header v3.0
 # This file is a part of the CaosDB Project.
 #
-# Copyright (C) Indiscale, GmbH <info@indiscale.com>
+# Copyright (C) 2020 Indiscale GmbH <info@indiscale.com>
 # Copyright (C) 2020 Florian Spreckelsen <f.spreckelsen@indiscale.com>
 #
 # This program is free software: you can redistribute it and/or modify