ENH: alpha done

src/caosadvancedtools/converter/import_labfolder.py → src/caosadvancedtools/converter/labfolder.py

@@ -21,7 +21,6 @@
 
 """ Imports labfolder exports """
 
-import argparse
 import os
 import re
 import shutil
@@ -38,7 +37,14 @@ from bs4 import BeautifulSoup
 
 import caosdb as db
 
-#crawler = Crawler()
+RERUN = False
+# crawler = Crawler()
+
+print("""
+WARNING: This is an ALPHA version. Parsing of the by labfolder exported data
+might not work correctly! There might be missing elements! Check the result
+carefully before inserting it.
+""")
 
 
 def create_project(project):
@@ -77,12 +83,34 @@ def add_property_from_data_element(dbrecord, element):
     quant = val_or_none(element.find_all(attrs={'class': 'element-quantity'}))
     val = val_or_none(element.find_all(attrs={'class': 'element-value'}))
 
+    print("tit", title)
+    print("qu", quant)
     if quant is not None:
-        title = title+"-"+quant
+        quant = quant.strip(": ")
+        title = title+" - "+quant
+    res = db.execute_query("FIND PROPERTY '{}'".format(title))
+    if len(res) == 0:
+        p = db.Property(name=title, unit=unit, datatype=db.DOUBLE)
+        p.insert()
+    try:
+        val = float(val)
+    except TypeError:
+        print("Value is no float!!!", val)
+        return
     dbrecord.add_property(name=title, value=val, unit=unit)
 
 
-def create_entry(entry, dbproject):
+def create_file(name, filename, root):
+    local_path = os.path.join(root, filename)
+    local_path = os.path.normpath(local_path)
+    if not os.path.exists(local_path):
+        raise ValueError("FILE DOES NOT EXIST: ", local_path)
+    f = db.File(path=local_path, file=local_path, name=name)
+    return f
+
+
+def create_entry(entry, dbproject, root):
+    cont = db.Container()
     dbrecord = db.Record()
     dbrecord.add_parent(name="LabbookEntry")
     dbrecord.add_property(name="Project", value=dbproject)
@@ -108,10 +136,24 @@ def create_entry(entry, dbproject):
             attrs={'class': 'dd_entry_cell_file_download'})
 
         if len(download) > 0:
-            local_path = (download[0].parent).attrs['data-filename']
-            f = db.File(path=local_path,
-                        file=local_path)
-            dbrecord.add_property(name="accompaningFile", value=f)
+            name = ((download[0].parent).attrs['data-filename']).strip('"')
+            if name == "blank.png":
+                continue
+            if len(download[0].find_all("img")) > 0:
+                filename = download[0].find_all("img")[0].attrs['src']
+            elif len(download[0].find_all("a")) > 0:
+                filename = download[0].find_all("a")[0].attrs['href']
+            else:
+                raise ValueError("could not get filename")
+            print(name)
+            print(filename)
+            f = create_file(name, filename, root)
+            if RERUN:
+                f.retrieve()
+            else:
+                f.insert()
+            dbrecord.add_property(name="associatedFile", value=f)
+            cont.append(f)
 
             continue
 
@@ -128,11 +170,16 @@ def create_entry(entry, dbproject):
             attrs={'class': 'table-el-container'})
 
         if len(tables) > 0:
-            local_path = (tables[0]).find_all(
+            name = (tables[0]).find_all(
                         attrs={'class': 'table-el-filename'}
                     )[0].getText().strip()
-            f = db.File(path=local_path, file=local_path)
+            f = create_file(name, name, root)
+            if RERUN:
+                f.retrieve()
+            else:
+                f.insert()
             dbrecord.add_property(name="table", value=f)
+            cont.append(f)
 
             continue
 
@@ -144,31 +191,48 @@ def create_entry(entry, dbproject):
 
             continue
 
-    print(dbrecord)
+    cont.extend([dbrecord, person])
 
+    return cont
 
-def main(args):
-    """The main function."""
 
-    if not os.path.exists(args.file):
-        raise ValueError("File does not exist")
+def treat_project(path):
+    with open(os.path.join(path, "index.html")) as fp:
+        tree = BeautifulSoup(fp, features="lxml")
 
-    with open(args.file) as inpu:
-        text = inpu.read()
+    cont = db.Container()
+    project = tree.find_all(id="eln_project_content")
 
-    tree = BeautifulSoup(text, features="lxml")
-    project = tree.find_all(id="eln_project_content")[0]
+    if len(project) == 0:
+        return
+    else:
+        project = project[0]
     dbproject = create_project(project)
+    cont.append(dbproject)
 
     for entry in project.find_all(lambda x: x.has_attr('data-id')):
-        create_entry(entry, dbproject)
+        recs = create_entry(entry, dbproject, path)
+        cont.extend(recs)
 
+    print(cont)
+    cont.insert()
     # import IPython
     # IPython.embed()
 
 
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument("file", default="./labfolder_example.html", nargs="?")
-    args = parser.parse_args()
-    sys.exit(main(args))
+def import_data(folder):
+    """imports the data of a labfolder export"""
+
+    if not os.path.exists(folder):
+        raise ValueError("folder does not exist")
+
+    projects_folder = os.path.join(folder, "projects")
+
+    if not os.path.exists(projects_folder):
+        raise ValueError("folder does not contain a projects folder")
+
+    for root, dirs, files in os.walk(projects_folder):
+        print(root, dirs, files)
+
+        if "index.html" in files:
+            treat_project(root)

unittests/model.yml

+Project:
+   obligatory_properties:
+      projectId:
+         datatype: INTEGER
+         description: 'UID of this project'
+Person:
+   recommended_properties:
+      firstName:
+         datatype: TEXT 
+         description: 'first name'
+      lastName:
+         datatype: TEXT 
+         description: 'last name'
+LabbookEntry:
+   recommended_properties:
+      Project:
+      entryId:
+         datatype: INTEGER
+         description: 'UID of this entry'
+      responsible:
+         datatype: Person
+         description: 'the person responsible for these notes'
+      textElement:
+         datatype: TEXT
+         description: 'a text element of a labbook recording'
+      associatedFile:
+         datatype: FILE
+         description: 'A file associated with this recording'
+      table:
+         datatype: FILE
+         description: 'A table document associated with this recording'

unittests/test_labfolder_import.py

+#!/usr/bin/env python3
+#
+# This file is a part of the CaosDB Project.
+#
+# Copyright (c) 2020 IndiScale GmbH
+# Copyright (c) 2020 Daniel Hornung <d.hornung@indiscale.com>
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+
+""" Imports labfolder exports """
+
+import argparse
+import sys
+
+import caosmodels
+from caosmodels.parser import parse_model_from_yaml
+
+from caosadvancedtools.converter import labfolder
+
+
+def main(args):
+    """The main function."""
+    model = parse_model_from_yaml("./model.yml")
+
+    model.sync_data_model()
+    labfolder.import_data(args.folder)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("folder", default="./example_labfolder_data",
+                        nargs="?", help='folder that contains the data')
+    args = parser.parse_args()
+    sys.exit(main(args))