more tests

tests/test_administration.py

@@ -52,6 +52,7 @@ def setup_module():
 
 def teardown_module():
     switch_to_admin_user()
+    admin.set_server_property("AUTH_OPTIONAL", "FALSE")
     try:
         admin._delete_user(name=test_user)
     except Exception as e:

tests/test_file.py

@@ -35,6 +35,7 @@ from sys import maxsize as maxint
 
 from lxml import etree
 from pytest import raises, mark
+import caosdb as db
 from caosdb import execute_query, get_config, get_connection
 from caosdb.common import models
 from caosdb.exceptions import EntityError, TransactionError
@@ -72,6 +73,10 @@ def teardown():
         os.remove("test2.dat")
     except Exception as e:
         print(e)
+    try:
+        os.remove("test.dat.tmp")
+    except BaseException:
+        pass
 
     # remove directories
     body = get_connection().retrieve(
@@ -277,9 +282,8 @@ def test_file6():
 def test_file7():
     try:
         # upload file to testfiles2/testsub/testfile...
-        upload_file = open("test.dat", "w")
+        with open("test.dat", "w") as upload_file:
             upload_file.write("hello world\n")
-        upload_file.close()
         file_ = models.File(name="Testfidb.dble",
                             description="Testfile Desc",
                             path="testfiles2/testsub/testfile" +
@@ -300,13 +304,11 @@ def test_file7():
             print("drop off box not on this system.")
         else:
             os.mkdir(path)
-            pickup_file = open(path + "/testpickup1.dat", "w")
+            with open(path + "/testpickup1.dat", "w") as pickup_file:
                 pickup_file.write("hello world\n")
-            pickup_file.close()
             os.mkdir(path + "/subfolder")
-            pickup_file = open(path + "/subfolder/testpickup2.dat", "w")
+            with open(path + "/subfolder/testpickup2.dat", "w") as pickup_file:
                 pickup_file.write("hello world\n")
-            pickup_file.close()
 
             folder_ = models.File(
                 name="PickupTestfolder",
@@ -325,12 +327,9 @@ def test_file7():
 
 
 def test_consistency_file_was_modified():
-    try:
-
     # insert new test file
-        upload_file = open("test.dat", "w")
+    with open("test.dat", "w") as upload_file:
         upload_file.write("hello world\n")
-        upload_file.close()
     file_ = models.File(name="TestConsistency1",
                         description="Testfile Desc",
                         path="debug/test_file_storage_consistency",
@@ -358,8 +357,7 @@ def test_consistency_file_was_modified():
 
     # download file again and check if it is still the same (just to be
     # sure that the server only simulated the consistency breach
-        d = open(file_.download(target="test.dat.tmp"), "r")
-
+    with open(file_.download(target="test.dat.tmp"), "r") as d:
         r = d.read()
         assert r == "hello world\n"
 
@@ -368,32 +366,11 @@ def test_consistency_file_was_modified():
     assert c.messages["Info", 0] is not None
     assert c.messages["Info", 0][0] == "File system is consistent."
 
-    finally:
-        # clean up file record
-        try:
-            file_.delete()
-        except BaseException:
-            pass
-        # clean up local files
-        try:
-            d.close()
-        except BaseException:
-            pass
-        try:
-            os.remove("test.dat")
-        except BaseException:
-            pass
-        try:
-            os.remove("test.dat.tmp")
-        except BaseException:
-            pass
-
 
 def test_consistency_file_does_not_exist():
-    try:
     with open("test.dat", "w") as upload_file:
         upload_file.write("hello world\n")
-        file_ = models.File(name="TestConsistency1",
+    file_ = models.File(name="TestConsistency2",
                         description="Testfile Desc",
                         path="debug/test_file_storage_consistency",
                         file="test.dat")
@@ -425,24 +402,6 @@ def test_consistency_file_does_not_exist():
     assert c.messages["Info", 0] is not None
     assert c.messages["Info", 0][0] == "File system is consistent."
 
-    finally:
-        try:
-            file_.delete()
-        except BaseException:
-            pass
-        try:
-            d.close()
-        except BaseException:
-            pass
-        try:
-            os.remove("test.dat")
-        except BaseException:
-            pass
-        try:
-            os.remove("test.dat.tmp")
-        except BaseException:
-            pass
-
 
 def test_consistency_unknown_file():
     c = runCheck(None, None)
@@ -714,7 +673,7 @@ def test_insert_files_in_dir_regex():
         except BaseException:
             pass
 
-
+@mark.skip()
 def test_thumbnails():
     file_ = models.File(name="TestFile",
                         description="Testfile Desc",
@@ -814,3 +773,132 @@ def test_auto_create_parent_dirs():
     file_1 = dir_c.xpath("file")[0]
     assert file_1.get("name") == "test.dat"
     assert file_1.get("url")[-26:] == "/FileSystem/A/B/C/test.dat"
+
+
+@mark.skip()
+def test_import_file():
+    path = get_config().get("IntegrationTests",
+                            "test_files.test_import_files_in_dir.local") + "testfolder/"
+
+    try:
+        os.makedirs(path)
+        os.makedirs(path + "subfolder/")
+        with open(path + "subfolder/some_data.csv", "w") as test_file1:
+            test_file1.write("hello, world\n")
+
+        file_ = db.File(description="Testfile Desc",
+                        import_file=True,
+                        path="subfolder/some_data.csv")
+        file_.insert()
+
+        file_ = db.File(description="Testfile Desc",
+                        import_file=True,
+                        path="subfolder/some_data.csv")
+        file_.insert()
+
+        file_.delete()
+    finally:
+        # we need to delete this one manually, bc file_.delete() will not
+        # delete the file
+        try:
+            shutil.rmtree(path)
+        except BaseException:
+            pass
+
+@mark.skip()
+def test_import_directoy():
+    path = get_config().get("IntegrationTests",
+                            "test_files.test_import_files_in_dir.local") + "testfolder/"
+
+    try:
+        os.makedirs(path)
+        os.makedirs(path + "subfolder/")
+        with open(path + "subfolder/some_data.csv", "w") as test_file1:
+            test_file1.write("hello, world\n")
+
+        c = models.Container()
+    finally:
+        try:
+            c.delete()
+        except BaseException:
+            pass
+        try:
+            shutil.rmtree(path)
+        except BaseException:
+            pass
+
+
+def test_consistency_file_was_modified_2():
+    # this test is different from test_consistency_file_was_modified in that
+    # the server does actually corrupt the file.
+    # insert new test file
+    with open("test.dat", "w") as upload_file:
+        upload_file.write("hello world\n")
+    file_ = models.File(name="TestConsistency3",
+                        description="Testfile Desc",
+                        path="debug/test_file_storage_consistency",
+                        file="test.dat")
+    file_.insert()
+    checksum1 = file_.checksum
+    print(checksum1)
+    qfile = execute_query("FIND FILE TestConsistency3", unique=True)
+    assert_equal(qfile.id, file_.id)
+    assert_equal(qfile.checksum, checksum1)
+
+    assert file_.id is not None
+    assert file_.is_valid()
+
+    # run consistency check (no consistency warning)
+    c = runCheck(None, None)
+    assert c.messages["Info", 0] is not None
+    assert c.messages["Info", 0][0] == "File system is consistent."
+
+    # when in debug mode, the server offers a special option
+    #
+    #     '-c FILE_WAS_MODIFIED_2'
+    #
+    # which simulates a modified file.
+    c = runCheck(None, "-c FILE_WAS_MODIFIED_2")
+
+    assert c.messages["Error", 0] is not None
+    assert c.messages["Error",
+                      0][0] == 'debug/test_file_storage_consistency: File was modified.'
+
+    # old checksum is still stored
+    qfile = execute_query("FIND FILE TestConsistency3", unique=True)
+    assert_equal(qfile.id, file_.id)
+    assert_equal(qfile.checksum, checksum1)
+
+    # download fails because the checksum changed.
+    with raises(db.exceptions.ConsistencyError) as exc:
+        file_.download(target="test.dat.tmp")
+
+    file_.download(target="test.dat.tmp", check_hash=False)
+    assert file_.checksum == checksum1
+    checksum2 = db.File.sha512("test.dat.tmp")
+    file_.checksum = checksum2
+    assert checksum2 != checksum1
+
+    # old checksum is still stored
+    qfile = execute_query("FIND FILE TestConsistency3", unique=True)
+    assert_equal(qfile.id, file_.id)
+    assert_equal(qfile.checksum, checksum1)
+
+    # here the client may validate the new file
+    file_.file = None
+    file_.update()
+    assert checksum2 == file_.checksum
+
+    # new checksum is stored
+    qfile = execute_query("FIND FILE TestConsistency3", unique=True)
+    assert_equal(qfile.id, file_.id)
+    assert_equal(qfile.checksum, checksum2)
+
+    # no ConsistencyError anymore
+    file_.download(target="test.dat.tmp")
+    assert checksum2 == file_.checksum
+
+    # new checksum is still stored
+    qfile = execute_query("FIND FILE TestConsistency3", unique=True)
+    assert_equal(qfile.id, file_.id)
+    assert_equal(qfile.checksum, checksum2)

tests/test_permissions.py

@@ -45,7 +45,9 @@ easy_pw = "1234"
 
 
 def setup_module():
-    switch_to_admin_user()
+    d = db.execute_query("FIND *")
+    if len(d) > 0:
+        d.delete()
     insert_test_user()
 
 
@@ -62,6 +64,9 @@ def teardown_module():
         os.remove("test2.dat")
     except BaseException:
         pass
+    d = db.execute_query("FIND *")
+    if len(d) > 0:
+        d.delete()
 
 
 @mark.skip
@@ -486,13 +491,12 @@ def test_update_role():
 
 
 def test_update_move_file():
-    upload_file = open("test.dat", "w")
+    with open("test.dat", "w") as upload_file:
         upload_file.write("hello world\n")
-    upload_file.close()
 
     f = db.File(
         name="TestFile",
-        path="/permissiontestfiles/test.dat",
+        path="/testfiles/permissiontestfiles/test.dat",
         file="test.dat").insert()
     assert f.is_valid()
 
@@ -500,22 +504,22 @@ def test_update_move_file():
     grant_permission(f, "UPDATE:FILE:MOVE")
 
     '''SUCCESS'''
-    f.path = "/otherpermissiontestfiles/test.dat"
+    f.path = "/testfiles/otherpermissiontestfiles/test.dat"
     f.update()
 
     f2 = db.execute_query("FIND TestFile", unique=True)
-    assert_equal(f2.path, "/otherpermissiontestfiles/test.dat")
+    assert_equal(f2.path, "/testfiles/otherpermissiontestfiles/test.dat")
 
     deny_permission(f, "UPDATE:FILE:MOVE")
 
     '''FAILURE'''
-    f.path = "/againotherpermissiontestfiles/test.dat"
+    f.path = "/testfiles/againotherpermissiontestfiles/test.dat"
     with raises(db.TransactionError) as te:
         f.update()
     assert te.value.has_error(db.AuthorizationError)
 
     f2 = db.execute_query("FIND TestFile", unique=True)
-    assert f2.path == "/otherpermissiontestfiles/test.dat"
+    assert f2.path == "/testfiles/otherpermissiontestfiles/test.dat"
 
 
 def test_update_add_file():
@@ -529,7 +533,7 @@ def test_update_add_file():
     grant_permission(f, "RETRIEVE:ENTITY")
     '''FAILURE'''
 
-    f.path = "/permissiontestfiles/newtest.dat"
+    f.path = "/testfiles/permissiontestfiles/newtest.dat"
     f.file = upload_file
     with raises(db.TransactionError) as te:
         f.update()
@@ -542,13 +546,13 @@ def test_update_add_file():
     grant_permission(f, "UPDATE:FILE:ADD")
 
     f2 = db.execute_query("FIND TestFile", unique=True)
-    f2.path = "/permissiontestfiles/newtest.dat"
+    f2.path = "/testfiles/permissiontestfiles/newtest.dat"
     f2.file = upload_file
     f2.update()
     assert f2.is_valid()
 
     f2 = db.execute_query("FIND TestFile", unique=True)
-    assert_equal(f2.path, "/permissiontestfiles/newtest.dat")
+    assert_equal(f2.path, "/testfiles/permissiontestfiles/newtest.dat")
 
 
 def test_update_change_file():
@@ -563,7 +567,7 @@ def test_update_change_file():
     f = db.File(
         name="TestFile",
         file=upload_file,
-        path="permissiontestfiles/test.dat").insert()
+        path="testfiles/permissiontestfiles/test.dat").insert()
     assert f.is_valid()
     grant_permission(f, "RETRIEVE:ENTITY")
     grant_permission(f, "RETRIEVE:FILE")
@@ -925,7 +929,7 @@ def test_download_file():
     f = db.File(
         name="TestFile",
         file=upload_file,
-        path="permissiontestfiles/test.dat").insert()
+        path="testfiles/permissiontestfiles/test.dat").insert()
     assert f.is_valid()
 
     '''FAILURE'''

tests/test_query.py

@@ -45,7 +45,7 @@ def setup_module():
 
 def setup():
     try:
-        db.execute_query("FIND Test*").delete()
+        db.execute_query("FIND *").delete()
     except Exception as e:
         print(e)
 
@@ -633,9 +633,8 @@ def store_file(path, name=None, f=None):
 
 def test_stored_at_wildcards():
 
-    upload_file = open("test.dat", "w")
+    with open("test.dat", "w") as upload_file:
         upload_file.write("hello world\n")
-    upload_file.close()
 
     file1 = store_file("test1.dat", f=upload_file)