From b8a94313d8ef36ad6b58450bff8d84ac098426c7 Mon Sep 17 00:00:00 2001
From: Timm Fitschen <t.fitschen@indiscale.com>
Date: Thu, 29 Oct 2020 11:15:26 +0100
Subject: [PATCH] WIP: introduce ListOfReferenceField
---
djaosdb/caosdb_client.py | 32 ++++++---
djaosdb/models/__init__.py | 5 +-
djaosdb/models/fields.py | 122 ++++++++++++++++++++++++++++++++-
djaosdb/sql2mongo/operators.py | 41 +++++++++--
tests/test_new.py | 114 +++++++++++++++++++++++++++++-
5 files changed, 297 insertions(+), 17 deletions(-)
diff --git a/djaosdb/caosdb_client.py b/djaosdb/caosdb_client.py
index 22c2673..5200bd5 100644
--- a/djaosdb/caosdb_client.py
+++ b/djaosdb/caosdb_client.py
@@ -123,7 +123,7 @@ class DefaultCaosDBClientDelegate:
if sub is not None:
subquery = self._get_filter_clause(sub)
result += " WITH (" + subquery + ")"
- return result
+ return " (" + result + " )"
def _get_filter_clause(self, fil):
LOGGER.debug("enter _get_filter_clause(%s)", fil)
@@ -183,9 +183,16 @@ class DefaultCaosDBClientDelegate:
rows = self._get_property_values(res, projection)
return FindResult(rows, projection, sort, limit, skip)
+ def find_auth(self, record_type, *args, **kwargs):
+ if record_type == "caosdb_auth_user":
+ name = kwargs["filter"]["elements"][-1]["v"]
+ print("find " + name)
+ res = self._caosdb.administration._retrieve_user(name)
+ print(res)
+
def list_record_type_names(self):
res = self._caosdb.execute_query("SELECT name FROM RECORDTYPE")
- return [e.name for e in res if e.name is not None]
+ return [e.name for e in res if e.name is not None] + ["caosdb_auth_user"]
def list_property_names(self):
res1 = self._caosdb.execute_query("SELECT name FROM PROPERTY")
@@ -214,6 +221,9 @@ class DefaultCaosDBClientDelegate:
c.insert()
+ def insert_auth(self, record_type : str, records: list):
+ print(records)
+
def insert_many(self, record_type : str, records : list):
c = self._caosdb.Container()
property_names = self.list_property_names()
@@ -439,7 +449,7 @@ class CaosDBClient:
# TODO
raise Exception("NOT IMPLEMENTED")
- def aggregate(self, *args, **kwargs):
+ def aggregate(self, record_type, *args, **kwargs):
"""aggregate
Parameters
@@ -451,8 +461,10 @@ class CaosDBClient:
Returns
-------
"""
- LOGGER.debug("aggregate(%s, %s)", args, kwargs)
- return self._delegate.aggregate(*args, **kwargs)
+ if record_type.startswith("caosdb_auth_"):
+ return self._delegate.aggregate_auth(record_type, records)
+ LOGGER.debug("aggregate(%s, %s, %s)", record_type, args, kwargs)
+ return self._delegate.aggregate(record_type, *args, **kwargs)
def insert_many(self, record_type : str, records : list):
"""insert_many
@@ -470,6 +482,8 @@ class CaosDBClient:
result : InsertManyResult
"""
LOGGER.debug("insert_many(%s, %s)", record_type, records)
+ if record_type.startswith("caosdb_auth_"):
+ return self._delegate.insert_auth(record_type, records)
return self._delegate.insert_many(record_type, records)
def update(self, *args, **kwargs):
@@ -532,7 +546,7 @@ class CaosDBClient:
# TODO
raise Exception("NOT IMPLEMENTED")
- def find(self, *args, **kwargs):
+ def find(self, record_type, *args, **kwargs):
"""find
Parameters
@@ -545,5 +559,7 @@ class CaosDBClient:
-------
result : FindResult
"""
- LOGGER.debug("find(%s, %s)", args, kwargs)
- return self._delegate.find(*args, **kwargs)
+ LOGGER.debug("find(%s, %s, %s)", record_type, args, kwargs)
+ if record_type.startswith("caosdb_auth_"):
+ return self._delegate.find_auth(record_type, *args, **kwargs)
+ return self._delegate.find(record_type, *args, **kwargs)
diff --git a/djaosdb/models/__init__.py b/djaosdb/models/__init__.py
index 069a3da..d9ff742 100644
--- a/djaosdb/models/__init__.py
+++ b/djaosdb/models/__init__.py
@@ -5,12 +5,13 @@ from .manager import DjaosdbManager
from .fields import (
ArrayField, DjongoManager,
EmbeddedField, ArrayReferenceField, ObjectIdField,
- GenericObjectIdField, JSONField
+ GenericObjectIdField, JSONField,
+ ListOfReferencesField
)
__all__ = django_models + [
'DjaosdbManager',
'DjongoManager', 'ArrayField',
'EmbeddedField', 'ArrayReferenceField', 'ObjectIdField',
- 'GenericObjectIdField', 'JSONField'
+ 'GenericObjectIdField', 'JSONField', 'ListOfReferencesField',
]
diff --git a/djaosdb/models/fields.py b/djaosdb/models/fields.py
index 94a6061..46a0a1d 100644
--- a/djaosdb/models/fields.py
+++ b/djaosdb/models/fields.py
@@ -24,7 +24,7 @@ from django.db import connections as caosdb_connections
from django.db import router, connections, transaction
from django.db.models import (
Manager, Model, Field, AutoField,
- ForeignKey, BigAutoField)
+ ForeignKey, BigAutoField, ManyToManyField)
from django.utils import version
from django.db.models.fields.related import RelatedField
from django.forms import modelform_factory
@@ -1090,3 +1090,123 @@ class ArrayReferenceField(ForeignKey):
initial = initial()
defaults['initial'] = [i.pk for i in initial]
return super().formfield(**defaults)
+
+def create_many_to_many_intermediary_model(field, klass):
+ """Copied from django.db.models.fields.related
+ .create_many_to_many_intermediary_model and tweaked for use with caosdb
+ List<REFERENCE> properties.
+ """
+ def set_managed(model, related, through):
+ through._meta.managed = model._meta.managed or related._meta.managed
+
+ to_model = resolve_relation(klass, field.remote_field.model)
+ name = '%s_%s' % (klass._meta.object_name, field.name)
+ lazy_related_operation(set_managed, klass, to_model, name)
+
+ to = make_model_tuple(to_model)[1]
+ from_ = klass._meta.model_name
+ if to == from_:
+ to = 'to_%s' % to
+ from_ = 'from_%s' % from_
+
+ meta = type('Meta', (), {
+ 'db_table': field._get_m2m_db_table(klass._meta),
+ 'auto_created': klass,
+ 'app_label': klass._meta.app_label,
+ 'db_tablespace': klass._meta.db_tablespace,
+ 'unique_together': (from_, to),
+ 'verbose_name': _('%(from)s-%(to)s relationship') % {'from': from_, 'to': to},
+ 'verbose_name_plural': _('%(from)s-%(to)s relationships') % {'from': from_, 'to': to},
+ 'apps': field.model._meta.apps,
+ })
+ # Construct and return the new class.
+ return type(name, (Model,), {
+ 'Meta': meta,
+ '__module__': klass.__module__,
+ from_: ForeignKey(
+ klass,
+ related_name='%s+' % name,
+ db_column = field.lh_column,
+ db_tablespace=field.db_tablespace,
+ db_constraint=field.remote_field.db_constraint,
+ on_delete=CASCADE,
+ ),
+ to: ForeignKey(
+ to_model,
+ related_name='%s+' % name,
+ db_column = field.rh_column,
+ db_tablespace=field.db_tablespace,
+ db_constraint=field.remote_field.db_constraint,
+ on_delete=CASCADE,
+ )
+ })
+
+
+class ListOfReferencesField(ManyToManyField):
+ """Provide a many-to-many relation by using caosdb's LIST<REFERENCE>
+ properties.
+ """
+
+ def __init__(self, to, related_name=None, related_query_name=None,
+ limit_choices_to=None, symmetrical=None, through=None,
+ through_fields=None, db_constraint=True, db_table=None,
+ db_column=None,
+ swappable=True, **kwargs):
+
+ self.lh_column = "id"
+ self.rh_column = db_column
+
+ super().__init__(to, related_name=related_name,
+ related_query_name=related_query_name,
+ limit_choices_to=limit_choices_to,
+ symmetrical=symmetrical, through=through,
+ through_fields=through_fields,
+ db_constraint=db_constraint, db_table=db_table,
+ swappable=swappable, **kwargs)
+
+ def contribute_to_class(self, cls, name, **kwargs):
+ """Copied from
+ django.db.models.fields.related.ManyToManyField.contribute_to_class
+
+ There is no difference in the code necessary since the only change is
+ that the local (tweaked) function
+ create_many_to_many_intermediary_model is being called instead of the
+ standard function.
+ """
+ # To support multiple relations to self, it's useful to have a non-None
+ # related name on symmetrical relations for internal reasons. The
+ # concept doesn't make a lot of sense externally ("you want me to
+ # specify *what* on my non-reversible relation?!"), so we set it up
+ # automatically. The funky name reduces the chance of an accidental
+ # clash.
+ if self.remote_field.symmetrical and (
+ self.remote_field.model == RECURSIVE_RELATIONSHIP_CONSTANT or
+ self.remote_field.model == cls._meta.object_name
+ ):
+ self.remote_field.related_name = "%s_rel_+" % name
+ elif self.remote_field.is_hidden():
+ # If the backwards relation is disabled, replace the original
+ # related_name with one generated from the m2m field name. Django
+ # still uses backwards relations internally and we need to avoid
+ # clashes between multiple m2m fields with related_name == '+'.
+ self.remote_field.related_name = "_%s_%s_+" % (cls.__name__.lower(), name)
+
+ super().contribute_to_class(cls, name, **kwargs)
+
+ # The intermediate m2m model is not auto created if:
+ # 1) There is a manually specified intermediate, or
+ # 2) The class owning the m2m field is abstract.
+ # 3) The class owning the m2m field has been swapped out.
+ if not cls._meta.abstract:
+ if self.remote_field.through:
+ def resolve_through_model(_, model, field):
+ field.remote_field.through = model
+ lazy_related_operation(resolve_through_model, cls, self.remote_field.through, field=self)
+ elif not cls._meta.swapped:
+ self.remote_field.through = create_many_to_many_intermediary_model(self, cls)
+
+ # Add the descriptor for the m2m relation.
+ setattr(cls, self.name, ManyToManyDescriptor(self.remote_field, reverse=False))
+
+ # Set up the accessor for the m2m table name for the relation.
+ self.m2m_db_table = partial(self._get_m2m_db_table, cls._meta)
diff --git a/djaosdb/sql2mongo/operators.py b/djaosdb/sql2mongo/operators.py
index f93c29c..02f84e7 100644
--- a/djaosdb/sql2mongo/operators.py
+++ b/djaosdb/sql2mongo/operators.py
@@ -85,6 +85,7 @@ class _BinaryOp(_Op):
def evaluate(self):
pass
+
class _InNotInOp(_BinaryOp):
def _fill_in(self, token):
@@ -190,7 +191,6 @@ class LikeOp(_BinaryOp):
"negation": False}
-
class IsOp(_BinaryOp):
def __init__(self, *args, **kwargs):
@@ -269,6 +269,22 @@ class NotOp(_UnaryOp):
self.lhs.rhs = self.rhs
+class NotNullOp(_UnaryOp):
+ def __init__(self, *args, **kwargs):
+ super().__init__(name="NOT", *args, **kwargs)
+ self._identifier = SQLToken.token2sql(self.statement, self.query)
+
+ def negate(self):
+ raise SQLDecodeError
+
+ def evaluate(self):
+ pass
+
+ def to_mongo(self):
+ return {"type": "pov", "p": self._identifier.field,
+ "o": " IS ", "v": "NOT NULL", "negation": False}
+
+
def simplify_and_or(oper, elems):
result = []
for elem in elems:
@@ -300,6 +316,10 @@ class _AndOrOp(_Op):
def evaluate(self):
if not (self.lhs and self.rhs):
+ print(self)
+ print(self.lhs)
+ print(self.rhs)
+ print(self.rhs.to_mongo())
raise SQLDecodeError
if isinstance(self.lhs, _AndOrOp):
@@ -384,6 +404,7 @@ class _StatementParser:
self._op = None
def _token2op(self,
+ prev_op: _Op,
tok: Token,
statement: SQLStatement) -> '_Op':
op = None
@@ -417,6 +438,9 @@ class _StatementParser:
elif tok.match(tokens.Keyword, 'IS'):
op = IsOp(**kw)
+ elif tok.match(tokens.Keyword, "EXISTS"):
+ raise Exception("EXISTS!!!")
+
elif isinstance(tok, Comparison):
op = CmpOp(tok, self.query)
@@ -434,7 +458,7 @@ class _StatementParser:
pass
elif isinstance(tok, Identifier):
- pass
+ op = NotNullOp(tok, self.query)
elif tok.match(tokens.Whitespace, r"\s", True):
pass
else:
@@ -444,6 +468,8 @@ class _StatementParser:
def _statement2ops(self):
def link_op():
+ print("0", prev_op)
+ print("1", op)
if prev_op is not None:
prev_op.rhs = op
op.lhs = prev_op
@@ -453,9 +479,13 @@ class _StatementParser:
prev_op = None
op = None
for tok in statement:
- op = self._token2op(tok, statement)
+ op = self._token2op(prev_op, tok, statement)
+ print("###", tok, op)
if not op:
continue
+ if isinstance(op, _InNotInOp) and prev_op is not None and isinstance(prev_op, NotNullOp):
+ self._ops.remove(prev_op)
+ prev_op = prev_op.lhs
link_op()
if isinstance(op, CmpOp):
self._cmp_ops.append(op)
@@ -530,11 +560,11 @@ class CmpOp(_Op):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._identifier = SQLToken.token2sql(self.statement.left, self.query)
-
if isinstance(self.statement.right, Identifier):
raise SQLDecodeError('Join using WHERE not supported')
self._operator = OPERATOR_MAP[self.statement.token_next(0)[1].value]
+
index = re_index(self.statement.right.value)
self._constant = self.params[index] if index is not None else None
@@ -543,6 +573,9 @@ class CmpOp(_Op):
else:
self._field_ext = None
+
+
+
def negate(self):
self.is_negated = True
diff --git a/tests/test_new.py b/tests/test_new.py
index 26049f2..901c443 100644
--- a/tests/test_new.py
+++ b/tests/test_new.py
@@ -445,7 +445,7 @@ def test_query_generation_conjuction():
assert query == ('SELECT id, django_content_type.app_label, '
'django_content_type.model, codename FROM RECORD '
'"auth_permission" WITH ( auth_group_permissions.'
- 'group_id="226" AND REFERENCES django_content_type )')
+ 'group_id="226" AND ( REFERENCES django_content_type ) )')
def test_parse_select_join_with_reverse_on_clause():
sql = """
@@ -607,7 +607,6 @@ def test_count_subquery():
distict = pipeline.pop("distinct")
filters = pipeline.pop("filter")
assert pipeline == {}
- print(filters)
assert filters == {
'type': 'and',
'elements': [
@@ -697,3 +696,114 @@ def test_remove_self_join():
'p': 'col4',
'o': '=',
'v': '209'}]}
+
+def test_user():
+ params = ('admin',)
+ sql = ('SELECT "caosdb_auth_user"."id", "caosdb_auth_user"."password", '
+ '"caosdb_auth_user"."last_login", "caosdb_auth_user"."is_superuser", '
+ '"caosdb_auth_user"."username", "caosdb_auth_user"."first_name", '
+ '"caosdb_auth_user"."last_name", "caosdb_auth_user"."email", '
+ '"caosdb_auth_user"."is_staff", "caosdb_auth_user"."is_active", '
+ '"caosdb_auth_user"."date_joined" FROM "caosdb_auth_user" WHERE '
+ '("caosdb_auth_user"."is_active" AND "caosdb_auth_user"."is_staff" AND '
+ '"caosdb_auth_user"."is_superuser" AND "caosdb_auth_user"."username" = %(0)s) '
+ 'LIMIT 21')
+ cached_record_types = [
+ "caosdb_auth_user",
+ ]
+ connection = _MockConnection(cached_record_types=cached_record_types)
+ q = Query(connection=connection, sql=sql, params=params)
+ caosdb_params = q._query._to_caosdb()[2]
+ print(caosdb_params["filter"])
+ assert caosdb_params["filter"] == {
+ 'type': 'and',
+ 'elements': [{
+ 'type': 'pov',
+ 'negation': False,
+ 'p': 'is_active',
+ 'o': ' IS ',
+ 'v': 'NOT NULL',
+ },{
+ 'type': 'pov',
+ 'negation': False,
+ 'p': 'is_staff',
+ 'o': ' IS ',
+ 'v': 'NOT NULL',
+ }, {
+ 'type': 'pov',
+ 'negation': False,
+ 'p': 'is_superuser',
+ 'o': ' IS ',
+ 'v': 'NOT NULL',
+ }, {'type': 'pov',
+ 'p': 'username',
+ 'o': '=',
+ 'v': 'admin',
+ 'negation': False}]}
+
+def test_join_where():
+ params = ('973',)
+ sql = ('SELECT "Language"."id", "Language"."name", "Language"."description" FROM '
+ '"Language" , "Material" WHERE (("Material"."Language"="Language"."id") AND '
+ '"Language"."id" IN (%s))')
+ assert False, "TODO"
+
+def test_select_count_group_by():
+ sql = """
+ SELECT "Language"."id", "Language"."name", "Language"."description",
+ COUNT("Material"."id") AS "used"
+ FROM "Language"
+ LEFT OUTER JOIN "Material"
+ ON ("Language"."id" = "Material"."language")
+ WHERE "Language"."id" IN (%(0)s)
+ GROUP BY "Language"."id", "Language"."name", "Language"."description"
+ """
+ params = ('973',)
+ assert False, "TODO"
+
+def test_select_count_having():
+ sql = """
+ SELECT "Language"."id", "Language"."name", "Language"."description",
+ COUNT("Material"."id") AS "material__count"
+ FROM "Language"
+ LEFT OUTER JOIN "Material"
+ ON ("Language"."id" = "Material"."language")
+ WHERE "Language"."id" IN (%(0)s)
+ GROUP BY "Language"."id", "Language"."name", "Language"."description"
+ HAVING COUNT("Material"."id") > %(1)s
+ """
+ params = ('973', 0)
+ assert False, "TODO"
+
+
+def test_select_count_having_multi_join():
+ sql = """
+ SELECT "Language"."id", "Language"."name", "Language"."description",
+ COUNT("Material"."id") AS "used"
+ FROM "Language"
+ LEFT OUTER JOIN "Material"
+ ON ("Language"."id" = "Material"."language")
+ INNER JOIN "Material" T4
+ ON ("Language"."id" = T4."language")
+ INNER JOIN "Material" T5
+ ON (T4."id" = T5."id")
+ WHERE ( T5."State" = %(0)s
+ AND "Language"."id" IN (%(1)s))
+ GROUP BY "Language"."id", "Language"."name", "Language"."description"
+ HAVING COUNT("Material"."id") > %(2)s
+ """
+ params = ('published', '973', 0)
+ assert False, "TODO"
+ count_result = "COUNT Material WHICH HAS A state=published AND REFERENCES 973"
+ result = "SELECT id, name, description, {COUNT_RESULT} FROM Language WITH id = 973"
+
+def test_field_not_null():
+ sql = """
+ SELECT "Language"."id", "Language"."name", "Language"."description"
+ FROM "Language"
+ INNER JOIN "Material"
+ ON ("Language"."id" = "Material"."language")
+ WHERE ("Material"."id" IS NOT NULL
+ AND "Language"."id" IN (%(0)s))
+ """
+ params = ('973',)
--
GitLab