Add tests for file import

tests/test_file.py

@@ -189,36 +189,6 @@ def test_file4():
             pass
 
 
-def test_upload_complete_folder():
-    file1_ = models.File(name="Testfile1",
-                         description="Testfile Desc",
-                         path="testfiles/testfile1.dat" + hex(randint(0,
-                                                                      maxint)),
-                         file="test.dat")
-    file2_ = models.File(name="Testfile2",
-                         description="Testfile Desc",
-                         path="testfiles/testfile2.dat" + hex(randint(0,
-                                                                      maxint)),
-                         file="test.dat")
-    c = models.Container()
-    c.extend([file1_, file2_])
-    c.insert()
-    assert file1_.id is not None
-    assert file2_.id is not None
-    c.delete()
-
-    c = models.Container()
-    folder_ = models.File(name="Testfolder",
-                          description="Testfolder Desc",
-                          path="testfiles/testfolder" + hex(randint(0,
-                                                                    maxint)),
-                          file="testfolder")
-    c.append(folder_)
-    c.insert()
-    assert folder_.id is not None
-    c.delete()
-
-
 def test_file6():
     try:
         # upload file to testfiles2/testfile...
@@ -674,7 +644,6 @@ def test_insert_files_in_dir_regex():
             pass
 
 
-@mark.xfail(reason="Needs file system refactoring")
 def test_auto_create_parent_dirs():
     file_ = models.File(name="TestFile",
                         path="A/B/C/test.dat",
@@ -755,38 +724,128 @@ def test_auto_create_parent_dirs():
     assert file_1.get("url")[-26:] == "/FileSystem/A/B/C/test.dat"
 
 
-@mark.xfail(reason="Needs file system refactoring")
 def test_import_file():
+    downloaded_file = None
     path = get_config().get("IntegrationTests",
                             "test_files.test_import_files_in_dir.local") + "testfolder/"
 
     try:
         os.makedirs(path)
         os.makedirs(path + "subfolder/")
-        with open(path + "subfolder/some_data.csv", "w") as test_file1:
+        local_file = os.path.join(path, "subfolder", "some_data.csv")
+        with open(local_file, "w") as test_file1:
             test_file1.write("hello, world\n")
+        checksum = db.File.sha512(local_file)
 
-        file_ = db.File(description="Testfile Desc",
-                        import_file=True,
-                        path="subfolder/some_data.csv")
+        file_ = db.File(import_file=True, # server knows where to look
+                        path="testfolder/subfolder/some_data.csv")
+        file_.checksum = checksum
         file_.insert()
 
-        file_ = db.File(description="Testfile Desc",
-                        import_file=True,
-                        path="subfolder/some_data.csv")
+        re = db.execute_query("FIND FILE WHICH IS STORED AT /testfolder/subfolder/some_data.csv", unique=True)
+        assert re.id == file_.id
+        assert re.checksum == checksum
+
+        downloaded_file = re.download()
+        assert db.File.sha512(downloaded_file) == checksum
+
+    finally:
+        try:
+            os.remove(downloaded_file)
+        except BaseException:
+            pass
+
+
+def test_import_non_existing_file():
+    file_ = db.File(import_file=True,
+                    path="testfolder/subfolder/does_not_exist.csv")
+    with raises(TransactionError) as te:
         file_.insert()
+    cm = te.value.errors[0]
+    errors = cm.entity.get_errors()
+    assert errors[0].description == 'This file does not exist. It cannot be imported.'
+
+
+def test_import_checksum_mismatch():
+    path = get_config().get("IntegrationTests",
+                            "test_files.test_import_files_in_dir.local") + "testfolder/"
+
+    try:
+        os.makedirs(path)
+        os.makedirs(path + "subfolder/")
+        local_file = os.path.join(path, "subfolder", "some_data.csv")
+        with open(local_file, "w") as test_file1:
+            test_file1.write("hello, world\n")
+
+        file_ = db.File(import_file=True, # server knows where to look
+                        path="testfolder/subfolder/some_data.csv")
+        file_.checksum = "abc123" # this is a wrong checksum
+
+        with raises(TransactionError) as te:
+            file_.insert()
+        cm = te.value.errors[0]
+        errors = cm.entity.get_errors()
+        assert errors[0].description == 'Checksum test failed. File is corrupted.'
+
 
-        file_.delete()
     finally:
-        # we need to delete this one manually, bc file_.delete() will not
-        # delete the file
         try:
             shutil.rmtree(path)
         except BaseException:
             pass
 
 
-@mark.xfail(reason="Needs file system refactoring")
+def test_import_existing():
+    path = get_config().get("IntegrationTests",
+                            "test_files.test_import_files_in_dir.local") + "testfolder/"
+
+    os.makedirs(path)
+    os.makedirs(path + "subfolder/")
+    local_file = os.path.join(path, "subfolder", "some_data.csv")
+    with open(local_file, "w") as test_file1:
+        test_file1.write("hello, world\n")
+
+    file_ = db.File(import_file=True,
+                    path="testfolder/subfolder/some_data.csv")
+
+    # insert once
+    file_.insert()
+
+    # insert again
+    file2 = db.File(import_file=True,
+                    path="testfolder/subfolder/some_data.csv")
+
+    with raises(TransactionError) as te:
+        file2.insert()
+    cm = te.value.errors[0]
+    errors = cm.entity.get_errors()
+    assert errors[0].description == 'This target path does already exist.'
+
+
+def test_import_creates_parents():
+    path = get_config().get("IntegrationTests",
+                            "test_files.test_import_files_in_dir.local") + "testfolder/"
+
+    os.makedirs(path)
+    os.makedirs(path + "subfolder/")
+    local_file = os.path.join(path, "subfolder", "some_data.csv")
+    with open(local_file, "w") as test_file1:
+        test_file1.write("hello, world\n")
+
+    file_ = db.File(import_file=True,
+                    path="testfolder/subfolder/some_data.csv")
+
+    file_.insert()
+
+    re = db.execute_query("FIND Entity WHICH IS STORED AT /testfolder/subfolder*")
+    for e in re:
+        if e.role == "Directory":
+            assert e.path == "/testfolder/subfolder"
+        else:
+            assert e.role == "File"
+            assert e.path == "/testfolder/subfolder/some_data.csv"
+
+
 def test_import_directoy():
     path = get_config().get("IntegrationTests",
                             "test_files.test_import_files_in_dir.local") + "testfolder/"
@@ -797,19 +856,63 @@ def test_import_directoy():
         with open(path + "subfolder/some_data.csv", "w") as test_file1:
             test_file1.write("hello, world\n")
 
-        c = models.Container()
+        directory = db.common.models.Directory(import_file=True, path="testfolder/subfolder/")
+        directory.insert()
+
+        re = db.execute_query("FIND Entity WHICH IS STORED AT /testfolder/subfolder", unique=True)
+        print(re)
+        assert re.id == directory.id
+        assert re.role == "Directory"
+        assert re.path == "/testfolder/subfolder"
+
     finally:
         try:
-            c.delete()
+            shutil.rmtree(path)
         except BaseException:
             pass
+
+
+def test_import_directoy_recursively():
+    path = get_config().get("IntegrationTests",
+                            "test_files.test_import_files_in_dir.local") + "testfolder/"
+
+    try:
+        os.makedirs(path)
+        os.makedirs(path + "subfolder/")
+        with open(path + "subfolder/some_data.csv", "w") as test_file1:
+            test_file1.write("hello, world\n")
+
+        directory = db.common.models.Directory(recursive_import=True, path="testfolder/subfolder/")
+
+        c = db.Container()
+        c.append(directory)
+        c.insert()
+
+        assert len(c) == 2
+        for e in c:
+            if e.role == "Directory":
+                assert e.path == "testfolder/subfolder"
+            else:
+                assert e.role == "File"
+                assert e.path == "testfolder/subfolder/some_data.csv"
+
+        re = db.execute_query("FIND Entity WHICH IS STORED AT /testfolder/subfolder*")
+        for e in re:
+            if e.role == "Directory":
+                assert e.path == "testfolder/subfolder"
+            else:
+                assert e.role == "File"
+                assert e.path == "testfolder/subfolder/some_data.csv"
+
+    finally:
         try:
             shutil.rmtree(path)
         except BaseException:
             pass
 
 
-@mark.xfail(reason="Needs file system refactoring")
+
+
 def test_consistency_file_was_modified_2():
     # this test is different from test_consistency_file_was_modified in that
     # the server does actually corrupt the file.