From 52c13e088fce5fdd3cdecf983f2d93d0ed057416 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Henrik=20tom=20W=C3=B6rden?= <henrik@trineo.org>
Date: Sat, 4 Jan 2020 10:27:11 +0100
Subject: [PATCH] ENH: added files for import and export
---
.gitignore | 1 +
src/caosadvancedtools/export_related.py | 160 +++++++++++++++++++++++
src/caosadvancedtools/import_from_xml.py | 78 +++++++++++
3 files changed, 239 insertions(+)
create mode 100644 src/caosadvancedtools/export_related.py
create mode 100644 src/caosadvancedtools/import_from_xml.py
diff --git a/.gitignore b/.gitignore
index 51c99625..c68adb8f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,3 +3,4 @@ __pycache__
.coverage
cache.db
*.egg-info
+.docker/cert
diff --git a/src/caosadvancedtools/export_related.py b/src/caosadvancedtools/export_related.py
new file mode 100644
index 00000000..75fe828a
--- /dev/null
+++ b/src/caosadvancedtools/export_related.py
@@ -0,0 +1,160 @@
+import argparse
+
+from lxml import etree
+
+import caosdb as db
+
+
+def type_converted_list(val, pr):
+ # TODO exists in apiutils
+ """Convert a list to a python list of the correct type."""
+ prrealpre = pr.replace("<", "<").replace(">", ">")
+ prreal = prrealpre[prrealpre.index("<") + 1:prrealpre.rindex(">")]
+ lst = [type_converted_value(i, prreal) for i in val]
+
+ return ([i[0] for i in lst], lst[0][1])
+
+
+def type_converted_value(val, pr):
+ # TODO exists in apiutils
+ """Convert val to the correct type which is indicated by the database
+ type string in pr.
+
+ Returns a tuple with two entries:
+ - the converted value
+ - True if the value has to be interpreted as an id acting as a reference
+ """
+
+ if val is None:
+ return (None, False)
+ elif pr == db.DOUBLE:
+ return (float(val), False)
+ elif pr == db.BOOLEAN:
+ return (bool(val), False)
+ elif pr == db.INTEGER:
+ return (int(val), False)
+ elif pr == db.TEXT:
+ return (val, False)
+ elif pr == db.FILE:
+ return (int(val), True)
+ elif pr == db.REFERENCE:
+ return (int(val), True)
+ elif pr == db.DATETIME:
+ return (val, False)
+ elif pr == db.TIMESPAN:
+ return (val, False)
+ elif pr[0:4] == "LIST":
+ return type_converted_list(val, pr)
+ else:
+ return (int(val), True)
+
+
+"""
+allows to create a revisionOf record
+"""
+
+
+def id_query(ids):
+ q = "FIND Entity with " + " OR ".join(["id={}".format(id) for id in ids])
+
+ return db.execute_query(q)
+
+
+def retrieve_multiple_entities(entities):
+ collection = db.Container()
+ step = 20
+
+ for i in range(len(entities)//step+1):
+ collection.extend(id_query(entities[i:i+step]))
+
+ return collection
+
+
+def get_ids_of_related_entities(entity):
+ entities = []
+
+ for par in entity.parents:
+ entities.append(par.id)
+
+ for prop in entity.properties:
+ entities.append(prop.id)
+ value, isref = type_converted_value(prop.value, prop.datatype)
+
+ if isref:
+ if isinstance(value, list):
+ entities.extend(value)
+ else:
+ entities.append(value)
+
+ return entities
+
+
+def recursively_collect_related(entity):
+ all_entities = db.Container()
+ all_entities.append(entity)
+ ids = set([entity.id])
+ new_entities = [entity]
+
+ while new_entities:
+ new_ids = set()
+
+ for ent in new_entities:
+ new_ids.update(get_ids_of_related_entities(ent))
+ new_ids = [id for id in new_ids if id not in ids]
+ new_entities = retrieve_multiple_entities(list(new_ids))
+ ids.update([e.id for e in new_entities])
+ all_entities.extend(new_entities)
+
+ return all_entities
+
+
+def invert_ids(entity):
+ apply_to_ids(entity, lambda x: x*-1)
+
+
+def apply_to_ids(entity, func):
+ entity.id = func(entity.id)
+
+ for par in entity.parents:
+ par.id = func(par.id)
+
+ for prop in entity.properties:
+ prop.id = func(prop.id)
+ value, isref = type_converted_value(prop.value, prop.datatype)
+
+ if isref:
+ if isinstance(value, list):
+ prop.value = [func(el) for el in prop.value]
+ else:
+ prop.value = func(prop.value)
+
+
+def main(rec_id):
+ ent = db.execute_query("FIND {}".format(rec_id), unique=True)
+ cont = recursively_collect_related(ent)
+
+ for ent in cont:
+ invert_ids(ent)
+ xml = etree.tounicode(cont.to_xml(
+ local_serialization=True), pretty_print=True)
+
+ with open("test.txt", "w") as fi:
+ fi.write(xml)
+
+
+def defineParser():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument(
+ '-i',
+ '--id',
+ type=int,
+ help='the id of the record that shall be copied and then changed')
+
+ return parser
+
+
+if __name__ == "__main__":
+ parser = defineParser()
+ args = parser.parse_args()
+
+ main(args.id)
diff --git a/src/caosadvancedtools/import_from_xml.py b/src/caosadvancedtools/import_from_xml.py
new file mode 100644
index 00000000..d25cd302
--- /dev/null
+++ b/src/caosadvancedtools/import_from_xml.py
@@ -0,0 +1,78 @@
+import argparse
+from tempfile import NamedTemporaryFile
+
+from caosmodels.data_model import DataModel
+from lxml import etree
+
+import caosdb as db
+from export_related import apply_to_ids
+
+
+def main(filename):
+ cont = db.Container()
+ with open(filename) as fi:
+ cont = cont.from_xml(fi.read())
+
+ tmpfile = NamedTemporaryFile(delete=False)
+ tmpfile.close()
+
+ model = []
+
+ for el in cont:
+ if isinstance(el, db.File):
+ el.file = tmpfile.name
+
+ if (isinstance(el, db.Property) or isinstance(el, db.RecordType)):
+ model.append(el)
+
+ for el in model:
+ cont.remove(el)
+
+ for el in model:
+ if el.name is not None:
+ ret = db.execute_query("FIND {}".format(el.name))
+ # TODO incorrect
+
+ # if len(ret) > 0 and not isinstance(ret[0], type(el)):
+ # print("The following entities have the same name as the current one:")
+
+ # for r in ret:
+ # print(r)
+
+ # if "y" == input("Do you want to insert\n"+str(el)):
+ # el.insert(unique=False)
+ id_mapping = {}
+
+ for el in model:
+ id_mapping[el.id] = el
+
+ datamodel = DataModel()
+ datamodel.extend(model)
+ datamodel.sync_data_model()
+
+ def replace_by_new(old):
+ if old in id_mapping:
+ return id_mapping[old].id
+ else:
+ return old
+
+ print(cont[:5])
+
+ for el in cont:
+ apply_to_ids(el, replace_by_new)
+ print(cont[:5])
+ cont.insert()
+
+
+def defineParser():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("file", help='file to be imported')
+
+ return parser
+
+
+if __name__ == "__main__":
+ parser = defineParser()
+ args = parser.parse_args()
+
+ main(args.file)
--
GitLab