diff --git a/djaosdb/models/fields.py b/djaosdb/models/fields.py
index 789fd2de974122b06634e9a45194ebc386e51788..ba2938afcb977773beac7dd4434393758ea8a83a 100644
--- a/djaosdb/models/fields.py
+++ b/djaosdb/models/fields.py
@@ -27,7 +27,9 @@ from django.db.models import (
ForeignKey, BigAutoField, ManyToManyField)
from django.utils import version
from django.db.models.fields.related import (
- RelatedField, lazy_related_operation)
+ RelatedField, lazy_related_operation, resolve_relation)
+from django.db.models.utils import make_model_tuple
+from django.db.models.deletion import CASCADE
from django.db.models.fields.related_descriptors import (
ManyToManyDescriptor)
from django.forms import modelform_factory
@@ -1194,7 +1196,7 @@ class ListOfReferencesField(ManyToManyField):
# clashes between multiple m2m fields with related_name == '+'.
self.remote_field.related_name = "_%s_%s_+" % (cls.__name__.lower(), name)
- super().contribute_to_class(cls, name, **kwargs)
+ RelatedField.contribute_to_class(self, cls, name, **kwargs)
# The intermediate m2m model is not auto created if:
# 1) There is a manually specified intermediate, or
diff --git a/djaosdb/sql2mongo/operators.py b/djaosdb/sql2mongo/operators.py
index 02f84e76f77f43653c9ac3066b076f7fce9c0754..05a0340580145208f91fee9772c82f675b7f50a4 100644
--- a/djaosdb/sql2mongo/operators.py
+++ b/djaosdb/sql2mongo/operators.py
@@ -73,8 +73,11 @@ class _BinaryOp(_Op):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
- identifier = SQLToken.token2sql(self.statement.prev_token, self.query)
- self._field = identifier.field
+ self._identifier = SQLToken.token2sql(self.statement.prev_token, self.query)
+
+ @property
+ def _field(self):
+ return self._identifier.field
def negate(self):
raise SQLDecodeError('Negating IN/NOT IN not supported')
@@ -195,6 +198,7 @@ class IsOp(_BinaryOp):
def __init__(self, *args, **kwargs):
super().__init__(name='IS', *args, **kwargs)
+
token = self.statement
key = token.next()
if key.match(tokens.Keyword, 'Null'):
@@ -208,11 +212,24 @@ class IsOp(_BinaryOp):
self.is_negated = True
def to_mongo(self):
+ ref = None
+ field = self._identifier.field
+ if self._identifier.query.left_table != self._identifier.table:
+ ref = self._identifier.table
+ field = self._identifier.column
+
is_null = not self._is_null if self.is_negated else self._is_null
- return {
- self._field: None if is_null else {'$ne': None}
+ result = {
+ "type": "pov",
+ "negation": False,
+ "p": field,
+ "o": " IS ",
+ "v": "NULL" if is_null else "NOT NULL",
}
+ if ref:
+ result["ref"] = ref
+ return result
class BetweenOp(_BinaryOp):
@@ -269,20 +286,21 @@ class NotOp(_UnaryOp):
self.lhs.rhs = self.rhs
-class NotNullOp(_UnaryOp):
+class IsTrueOp(_UnaryOp):
def __init__(self, *args, **kwargs):
- super().__init__(name="NOT", *args, **kwargs)
+ super().__init__(name="IS", *args, **kwargs)
self._identifier = SQLToken.token2sql(self.statement, self.query)
+ self._negation = False
def negate(self):
- raise SQLDecodeError
+ self._negation = True
def evaluate(self):
pass
def to_mongo(self):
return {"type": "pov", "p": self._identifier.field,
- "o": " IS ", "v": "NOT NULL", "negation": False}
+ "o": "=", "v": "TRUE", "negation": self._negation}
def simplify_and_or(oper, elems):
@@ -316,10 +334,6 @@ class _AndOrOp(_Op):
def evaluate(self):
if not (self.lhs and self.rhs):
- print(self)
- print(self.lhs)
- print(self.rhs)
- print(self.rhs.to_mongo())
raise SQLDecodeError
if isinstance(self.lhs, _AndOrOp):
@@ -458,7 +472,7 @@ class _StatementParser:
pass
elif isinstance(tok, Identifier):
- op = NotNullOp(tok, self.query)
+ op = IsTrueOp(tok, self.query)
elif tok.match(tokens.Whitespace, r"\s", True):
pass
else:
@@ -468,8 +482,6 @@ class _StatementParser:
def _statement2ops(self):
def link_op():
- print("0", prev_op)
- print("1", op)
if prev_op is not None:
prev_op.rhs = op
op.lhs = prev_op
@@ -480,10 +492,9 @@ class _StatementParser:
op = None
for tok in statement:
op = self._token2op(prev_op, tok, statement)
- print("###", tok, op)
if not op:
continue
- if isinstance(op, _InNotInOp) and prev_op is not None and isinstance(prev_op, NotNullOp):
+ if isinstance(op, (_InNotInOp, LikeOp, IsOp, NotOp, CmpOp)) and prev_op is not None and isinstance(prev_op, IsTrueOp):
self._ops.remove(prev_op)
prev_op = prev_op.lhs
link_op()
@@ -560,21 +571,25 @@ class CmpOp(_Op):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._identifier = SQLToken.token2sql(self.statement.left, self.query)
+ self._field_ext = None
+ self._operator = None
if isinstance(self.statement.right, Identifier):
- raise SQLDecodeError('Join using WHERE not supported')
-
- self._operator = OPERATOR_MAP[self.statement.token_next(0)[1].value]
-
- index = re_index(self.statement.right.value)
-
- self._constant = self.params[index] if index is not None else None
- if isinstance(self._constant, dict):
- self._field_ext, self._constant = next(iter(self._constant.items()))
+ right = SQLToken.token2sql(self.statement.right, self.query)
+ if right.column.lower() == "id":
+ self._filter_type = "back_reference"
+ self._constant = self._identifier.table
+ ## TODO reference
+ else:
+ raise SQLDecodeError('Join using WHERE not supported')
else:
- self._field_ext = None
-
+ self._filter_type = "pov"
+ self._operator = OPERATOR_MAP[self.statement.token_next(0)[1].value]
+ index = re_index(self.statement.right.value)
+ self._constant = self.params[index] if index is not None else None
+ if isinstance(self._constant, dict):
+ self._field_ext, self._constant = next(iter(self._constant.items()))
def negate(self):
self.is_negated = True
@@ -583,7 +598,6 @@ class CmpOp(_Op):
pass
def to_mongo(self):
-
ref = None
field = self._identifier.field
if self._field_ext:
@@ -592,11 +606,13 @@ class CmpOp(_Op):
ref = self._identifier.table
field = self._identifier.column
- result = {"type": "pov",
+ result = {"type": self._filter_type,
"negation": self.is_negated,
"p": field,
- "o": self._operator,
"v": self._constant}
+
+ if self._operator:
+ resule["o"] = self._operator
if ref:
result["ref"] = ref
diff --git a/djaosdb/sql2mongo/query.py b/djaosdb/sql2mongo/query.py
index 18841d628135abe72114473b4ba78c4ff9b5cbab..5c3d0408000a873af9b637f0b6579b78427d252f 100644
--- a/djaosdb/sql2mongo/query.py
+++ b/djaosdb/sql2mongo/query.py
@@ -67,14 +67,16 @@ def _merge_joins_and_where(joins, where):
if not joins:
return joins, where
- merged_joins = [join for join in joins
+ # split into two sets, one containing the reference and back_reference
+ # filters, one the remainder
+ remaining_joins = [join for join in joins
if join and not "type" in join]
refs = [join for join in joins
if "type" in join and join["type"] in ["reference",
"back_reference"]]
if not where:
- return merged_joins, _conjunction_filters(*refs)
+ return remaining_joins, _conjunction_filters(*refs)
for ref in refs:
# look for subqueries
@@ -86,19 +88,33 @@ def _merge_joins_and_where(joins, where):
ref["sub"]["p"] = "ID"
where = None
break
+ ## TODO else:
elif ref["type"] == "back_reference":
v = ref["v"]
- if ("ref" in where and where["ref"]):
+ if ("ref" in where and where["ref"] == v):
# no conjunction or disjuction
ref["sub"] = where
where = None
break
+ elif "elements" in where:
+ for i in range(len(where["elements"])):
+ elem = where["elements"][i]
+ if (elem is not None and "ref" in elem and elem["ref"] == v):
+ if "v" in elem and elem["v"] == "NOT NULL":
+ # this is unnecessary
+ pass
+ else:
+ ref["sub"] = elem
+ del elem["ref"]
+ where["elements"][i] = None
+ break
+ ## TODO deeper
else:
raise NotImplementedError("merging conjuction or disjuction "
"filters with joins")
merged_where = _conjunction_filters(where, *refs)
- return merged_joins, merged_where
+ return remaining_joins, merged_where
class ReturnDocument(enum.Enum):
AFTER = 1
@@ -848,7 +864,6 @@ class Query:
connection: 'caosdb_client.CaosDBClient',
sql: str,
params: Optional[Sequence]):
-
self._params = params
self.connection = connection
self._params_index_count = -1
diff --git a/tests/test_new.py b/tests/test_new.py
index 2bf1d376d6d4ca7c8d82c62d6bc9bebc911a1f96..3310a0b7664bdd7b9ee03e7eb2d9f0f814d4faf6 100644
--- a/tests/test_new.py
+++ b/tests/test_new.py
@@ -198,7 +198,6 @@ def test_merge_joins_and_where_backref_sub_query_id():
merged_joins, merged_where = _merge_joins_and_where(joins, where)
assert merged_joins == []
- print(merged_where)
assert merged_where == {
"type": "back_reference",
"p": "p1",
@@ -407,7 +406,6 @@ def test_inner_and_outer_join():
callback, _, pipeline = select_query._to_caosdb()
assert callback == connection.aggregate
assert "joins" not in pipeline
- print(pipeline["filter"])
assert pipeline["filter"] == {
"type": "reference",
'negation': False,
@@ -489,7 +487,6 @@ def test_parse_select_join_with_reverse_on_clause():
callback, _, pipeline = select_query._to_caosdb()
assert callback == connection.aggregate
assert "joins" not in pipeline
- print(pipeline["filter"])
assert pipeline["filter"] == {
"type": "back_reference",
'negation': False,
@@ -531,7 +528,6 @@ def test_inner_join_with_bad_names():
assert select_query.where is not None
assert select_query.where.to_mongo() == {
"type": "pov", 'negation': False,
- # TODO
'o': '=', 'p': 'material_id', 'ref': "B", 'v': '227'}
assert select_query._needs_aggregation() is True
@@ -544,27 +540,21 @@ def test_inner_join_with_bad_names():
assert inner_join.to_mongo() == {
"type": "back_reference",
- # TODO
'negation': False, 'p': 'language_id',
- # TODO
'v': 'B'}
callback, _, pipeline = select_query._to_caosdb()
assert callback == connection.aggregate
assert "joins" not in pipeline
- print(pipeline["filter"])
assert pipeline["filter"] == {
"type": "back_reference",
'negation': False,
- # TODO
'p': 'language_id',
- # TODO
'v': 'B',
"sub": {
"type": "pov",
'negation': False,
'ref': "B",
- # TODO
'p': 'material_id',
'o': '=',
'v': '227'
@@ -715,27 +705,26 @@ def test_user():
connection = _MockConnection(cached_record_types=cached_record_types)
q = Query(connection=connection, sql=sql, params=params)
caosdb_params = q._query._to_caosdb()[2]
- print(caosdb_params["filter"])
assert caosdb_params["filter"] == {
'type': 'and',
'elements': [{
'type': 'pov',
'negation': False,
'p': 'is_active',
- 'o': ' IS ',
- 'v': 'NOT NULL',
+ 'o': "=",
+ 'v': "TRUE",
},{
'type': 'pov',
'negation': False,
'p': 'is_staff',
- 'o': ' IS ',
- 'v': 'NOT NULL',
+ 'o': "=",
+ 'v': "TRUE",
}, {
'type': 'pov',
'negation': False,
'p': 'is_superuser',
- 'o': ' IS ',
- 'v': 'NOT NULL',
+ 'o': "=",
+ 'v': "TRUE",
}, {'type': 'pov',
'p': 'username',
'o': '=',
@@ -744,10 +733,32 @@ def test_user():
def test_join_where():
params = ('973',)
- sql = ('SELECT "Language"."id", "Language"."name", "Language"."description" FROM '
- '"Language" , "Material" WHERE (("Material"."Language"="Language"."id") AND '
- '"Language"."id" IN (%s))')
- assert False, "TODO"
+ sql = ('SELECT "A"."p1", "A"."p2", "A"."p3" FROM '
+ '"A" , "B" WHERE (("B"."A_id"="A"."id") AND '
+ '"A"."id" IN (%s))')
+ params = ('973',)
+ cached_record_types = [
+ "A", "B",
+ ]
+ connection = _MockConnection(cached_record_types=cached_record_types)
+ q = Query(connection=connection, sql=sql, params=params)
+ caosdb_params = q._query._to_caosdb()[2]
+ assert caosdb_params["projection"] == ["p1", "p2", "p3"]
+ print(caosdb_params["filter"]["elements"])
+ assert caosdb_params["filter"] == {
+ "type": "and",
+ "elements": [{
+ 'type': 'back_reference',
+ 'negation': False,
+ 'p': 'A_id',
+ 'v': 'B',
+ 'ref': 'B'
+ }, {
+ 'type': 'in',
+ 'p': 'id',
+ 'v': ['973']
+ }]
+ }
def test_select_count_group_by():
sql = """
@@ -776,7 +787,6 @@ def test_select_count_having():
params = ('973', 0)
assert False, "TODO"
-
def test_select_count_having_multi_join():
sql = """
SELECT "Language"."id", "Language"."name", "Language"."description",
@@ -800,11 +810,79 @@ def test_select_count_having_multi_join():
def test_field_not_null():
sql = """
- SELECT "Language"."id", "Language"."name", "Language"."description"
- FROM "Language"
- INNER JOIN "Material"
- ON ("Language"."id" = "Material"."language")
- WHERE ("Material"."id" IS NOT NULL
- AND "Language"."id" IN (%(0)s))
+ SELECT "A"."p1", "A"."p2", "A"."p3"
+ FROM "A"
+ INNER JOIN "B"
+ ON ("A"."id" = "B"."a_id")
+ WHERE ("B"."a_id" IS NOT NULL
+ AND "A"."id" IN (%(0)s))
"""
params = ('973',)
+ cached_record_types = [
+ "A", "B",
+ ]
+ connection = _MockConnection(cached_record_types=cached_record_types)
+ q = Query(connection=connection, sql=sql, params=params)
+ caosdb_params = q._query._to_caosdb()[2]
+ assert caosdb_params["projection"] == ["p1", "p2", "p3"]
+ assert caosdb_params["filter"] == {
+ "type": "and",
+ "elements": [{
+ "type": "in",
+ "p": "id",
+ "v": ["973"],
+ }, {
+ "type": "back_reference",
+ "p": "a_id",
+ "v": "B",
+ "negation": False,
+ }]
+ }
+
+def test_query():
+ sql = """
+ SELECT COUNT(*) AS "__count"
+ FROM "A"
+ WHERE ("A"."p1" = %(0)s
+ AND ("A"."p2" iLIKE %(1)s
+ OR "A"."p3" iLIKE %(2)s
+ OR "A"."p4" iLIKE %(3)s))
+ """
+ params = ('v1', 'v2', 'v3', 'v4')
+ cached_record_types = [
+ "A",
+ ]
+ connection = _MockConnection(cached_record_types=cached_record_types)
+ q = Query(connection=connection, sql=sql, params=params)
+ caosdb_params = q._query._to_caosdb()[2]
+ assert caosdb_params["count"] == "__count"
+ assert caosdb_params["filter"] == {
+ "type": "and",
+ "elements": [{
+ "type": "pov",
+ "negation": False,
+ "p": "p1",
+ "o": "=",
+ "v": "v1",
+ }, {
+ "type": "or",
+ "elements": [{
+ "type": "pov",
+ "negation": False,
+ "p": "p2",
+ "o": " LIKE ",
+ "v": "v2",
+ }, {
+ "type": "pov",
+ "negation": False,
+ "p": "p3",
+ "o": " LIKE ",
+ "v": "v3",
+ }, {
+ "type": "pov",
+ "negation": False,
+ "p": "p4",
+ "o": " LIKE ",
+ "v": "v4",
+ }],
+ }]}