Skip to content

Commit 36021bb

Browse files
MuhammadTahaNaveedjrgemignaniZainab-Saad
committed
Implement list comprehension (apache#1610)
* Add initial code for list comprehension - Additionally, fixed a bug in nested queries. * Resolve variable scope and ambigous variable issue - For scoping the variable, remove the pnsi from namespace list after being used by the list comprehension. - To resolve the ambigous variable issue, only scan the pnsi of list_comp subquery. * Fix non agg plan node issue in list comprehension - Resolved the issue 'aggref found in non-agg plan node' when the list comprehension is used as a property constraint in MATCH and other clauses or when used in WHERE clause. Intead of adding qual in the jointree, we are now adding the qual in having node of query. e.g. MATCH (n {list: [for i in [1,2,3]]}) MATCH (n) WHERE n.list=[for i in [1,2,3]] WITH n WHERE n.list=[for i in [1,2,3]] - Removed redundant call to function tranform_cypher_clause_with_where by tranform_match_clause. * Resolve variable reuse issue in list comprehension - Also, for scoping list comprehension variable, instead of removing the pnsi of list comp from namespace, now we just mark the cols of pnsi as not visible. * Modify grammer to match opencypher list comprehension grammer - Also added initial regression tests for list comprehension * Add regression tests for bugs resolved - Includes tests for nested cases, ambigous variable issue, list comprehension variable scoping and planner node aggref issue. - Should be added more. * Resolve invalid output on multiple list comprehensions - Fix the output of multiple list comprehensions in the RETURN clause and the WHERE clause. - Added regression tests for the above and some for the previous bugs resolved. * Fix aggref found where not expected issue (#189) - Resolved the 'aggref found where not expected' error thrown when using list comprehensions in UNWIND, or as a property update expression in SET or in the comparison expressions, typecasts or string matching in WITH - WHERE clause - Added an error to be thrown when using aggregation functions in UNWIND and property update expr in SET clause - Added regression tests * cleanup: Remove dead code and formatting changes --------- Co-authored-by: John Gemignani <[email protected]> Co-authored-by: Zainab Saad <[email protected]>
1 parent 59c9f75 commit 36021bb

File tree

15 files changed

+1345
-27
lines changed

15 files changed

+1345
-27
lines changed

‎Makefile

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,7 @@ REGRESS = scan \
110110
graph_generation \
111111
name_validation \
112112
jsonb_operators \
113+
list_comprehension \
113114
drop
114115

115116
srcdir=`pwd`

‎regress/expected/list_comprehension.out

Lines changed: 585 additions & 0 deletions
Large diffs are not rendered by default.

‎regress/sql/list_comprehension.sql

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
LOAD 'age';
21+
SET search_path TO ag_catalog;
22+
23+
SELECT create_graph('list_comprehension');
24+
25+
SELECT * from cypher('list_comprehension', $$ CREATE (u {list: [0, 2, 4, 6, 8, 10, 12]}) RETURN u $$) AS (result agtype);
26+
SELECT * from cypher('list_comprehension', $$ CREATE (u {list: [1, 3, 5, 7, 9, 11, 13]}) RETURN u $$) AS (result agtype);
27+
SELECT * from cypher('list_comprehension', $$ CREATE (u {list: []}) RETURN u $$) AS (result agtype);
28+
29+
SELECT * FROM cypher('list_comprehension', $$ RETURN [u IN range(1, 30, 2)] $$) AS (result agtype);
30+
SELECT * FROM cypher('list_comprehension', $$ RETURN [u IN range(1, 30, 2)][2] $$) AS (result agtype);
31+
SELECT * FROM cypher('list_comprehension', $$ RETURN [u IN range(1, 30, 2)][1..4] $$) AS (result agtype);
32+
33+
SELECT * FROM cypher('list_comprehension', $$ RETURN [u IN range(1, 30, 2) WHERE u % 3 = 0] $$) AS (result agtype);
34+
SELECT * FROM cypher('list_comprehension', $$ RETURN [u IN range(1, 30, 2) WHERE u % 3 = 0][2] $$) AS (result agtype);
35+
SELECT * FROM cypher('list_comprehension', $$ RETURN [u IN range(1, 30, 2) WHERE u % 3 = 0][0..4] $$) AS (result agtype);
36+
37+
SELECT * FROM cypher('list_comprehension', $$ RETURN [u IN range(1, 30, 2) WHERE u % 3 = 0 | u^2 ] $$) AS (result agtype);
38+
SELECT * FROM cypher('list_comprehension', $$ RETURN [u IN range(1, 30, 2) WHERE u % 3 = 0 | u^2 ][3] $$) AS (result agtype);
39+
SELECT * FROM cypher('list_comprehension', $$ RETURN [u IN range(1, 30, 2) WHERE u % 3 = 0 | u^2 ][1..5] $$) AS (result agtype);
40+
41+
SELECT * FROM cypher('list_comprehension', $$ RETURN [u IN range(1, 30, 2) | u^2 ] $$) AS (result agtype);
42+
SELECT * FROM cypher('list_comprehension', $$ RETURN [u IN range(1, 30, 2) | u^2 ][0] $$) AS (result agtype);
43+
SELECT * FROM cypher('list_comprehension', $$ RETURN [u IN range(1, 30, 2) | u^2 ][0..2] $$) AS (result agtype);
44+
45+
SELECT * FROM cypher('list_comprehension', $$ MATCH (u) RETURN (u) $$) AS (result agtype);
46+
SELECT * FROM cypher('list_comprehension', $$ MATCH () RETURN [i IN range(0, 20, 2) WHERE i % 3 = 0 ] $$) AS (result agtype);
47+
SELECT * FROM cypher('list_comprehension', $$ MATCH () RETURN [i IN range(0, 20, 2) WHERE i % 3 = 0 | i^2 ] $$) AS (result agtype);
48+
SELECT * FROM cypher('list_comprehension', $$ MATCH (u) RETURN [i IN range(0, 20, 2) WHERE i % 3 = 0 | i^2 ] $$) AS (result agtype);
49+
SELECT * FROM cypher('list_comprehension', $$ MATCH p=() RETURN [i IN range(0, 20, 2) WHERE i % 3 = 0 | i^2 ] $$) AS (result agtype);
50+
SELECT * FROM cypher('list_comprehension', $$ MATCH p=(u) RETURN [i IN range(0, 20, 2) WHERE i % 3 = 0 | i^2 ] $$) AS (result agtype);
51+
SELECT * FROM cypher('list_comprehension', $$ MATCH (u) RETURN [i IN u.list] $$) AS (result agtype);
52+
SELECT * FROM cypher('list_comprehension', $$ MATCH (u) RETURN [i IN u.list WHERE i % 3 = 0] $$) AS (result agtype);
53+
SELECT * FROM cypher('list_comprehension', $$ MATCH (u) RETURN [i IN u.list WHERE i % 3 = 0 | i/3] $$) AS (result agtype);
54+
SELECT * FROM cypher('list_comprehension', $$ MATCH (u) RETURN [i IN u.list WHERE i % 3 = 0 | i/3][1] $$) AS (result agtype);
55+
SELECT * FROM cypher('list_comprehension', $$ MATCH (u) RETURN [i IN u.list WHERE i % 3 = 0 | i/3][0..2] $$) AS (result agtype);
56+
SELECT * FROM cypher('list_comprehension', $$ MATCH (u) RETURN [i IN u.list WHERE i % 3 = 0 | i/3][0..2][1] $$) AS (result agtype);
57+
58+
-- Nested cases
59+
SELECT * FROM cypher('list_comprehension', $$ RETURN [i IN [i IN [1,2,3]]] $$) AS (result agtype);
60+
SELECT * FROM cypher('list_comprehension', $$ RETURN [i IN [i IN [i IN [1,2,3]]]] $$) AS (result agtype);
61+
62+
SELECT * FROM cypher('list_comprehension', $$ RETURN [i IN [i IN [1,2,3] WHERE i>1]] $$) AS (result agtype);
63+
SELECT * FROM cypher('list_comprehension', $$ RETURN [i IN [i IN [1,2,3]] WHERE i>1] $$) AS (result agtype);
64+
SELECT * FROM cypher('list_comprehension', $$ RETURN [i IN [i IN [1,2,3] WHERE i>1] WHERE i>2] $$) AS (result agtype);
65+
66+
SELECT * FROM cypher('list_comprehension', $$ RETURN [i IN [i IN [1,2,3] WHERE i>1 | i^2]] $$) AS (result agtype);
67+
SELECT * FROM cypher('list_comprehension', $$ RETURN [i IN [i IN [1,2,3]] WHERE i>1 | i^2] $$) AS (result agtype);
68+
SELECT * FROM cypher('list_comprehension', $$ RETURN [i IN [i IN [1,2,3] WHERE i>1] WHERE i>2 | i^2] $$) AS (result agtype);
69+
SELECT * FROM cypher('list_comprehension', $$ RETURN [i IN [i IN [1,2,3] WHERE i>1 | i^2] WHERE i>4] $$) AS (result agtype);
70+
SELECT * FROM cypher('list_comprehension', $$ RETURN [i IN [i IN [1,2,3] WHERE i>1 | i^2] WHERE i>4 | i^2] $$) AS (result agtype);
71+
72+
-- List comprehension inside where and property constraints
73+
SELECT * FROM cypher('list_comprehension', $$ MATCH (u) WHERE u.list=[i IN range(0,12,2)] RETURN u $$) AS (result agtype);
74+
SELECT * FROM cypher('list_comprehension', $$ MATCH (u) WHERE u.list=[i IN range(1,13,2)] RETURN u $$) AS (result agtype);
75+
SELECT * FROM cypher('list_comprehension', $$ MATCH (u) WHERE u.list=[i IN range(0,12,2) WHERE i>4] RETURN u $$) AS (result agtype);
76+
SELECT * FROM cypher('list_comprehension', $$ MATCH (u) WHERE u.list=[i IN range(1,13,2) WHERE i>4 | i^1] RETURN u $$) AS (result agtype);
77+
SELECT * FROM cypher('list_comprehension', $$ MATCH (u) WHERE u.list@>[i IN range(0,6,2)] RETURN u $$) AS (result agtype);
78+
SELECT * FROM cypher('list_comprehension', $$ MATCH (u) MATCH (v) WHERE v.list=[i IN u.list] RETURN v $$) AS (result agtype);
79+
80+
SELECT * FROM cypher('list_comprehension', $$ MATCH (u {list:[i IN range(0,12,2)]}) RETURN u $$) AS (result agtype);
81+
SELECT * FROM cypher('list_comprehension', $$ MATCH (u {list:[i IN range(1,13,2)]}) RETURN u $$) AS (result agtype);
82+
SELECT * FROM cypher('list_comprehension', $$ MATCH (u {list:[i IN range(0,12,2) WHERE i>4]}) RETURN u $$) AS (result agtype);
83+
SELECT * FROM cypher('list_comprehension', $$ MATCH (u {list:[i IN range(1,13,2) WHERE i>4 | i^1]}) RETURN u $$) AS (result agtype);
84+
SELECT * FROM cypher('list_comprehension', $$ MATCH (u) MATCH (v {list:[i IN u.list]}) RETURN v $$) AS (result agtype);
85+
86+
SELECT * FROM cypher('list_comprehension', $$ CREATE (u {list:[i IN range(12,24,2)]}) RETURN u $$) AS (result agtype);
87+
SELECT * FROM cypher('list_comprehension', $$ CREATE (u {list:[i IN range(0,12,2) WHERE i>4]}) RETURN u $$) AS (result agtype);
88+
SELECT * FROM cypher('list_comprehension', $$ CREATE (u {list:[i IN range(1,13,2) WHERE i>4 | i^2]}) RETURN u $$) AS (result agtype);
89+
SELECT * FROM cypher('list_comprehension', $$ WITH [u IN [1,2,3]] AS a CREATE (u {list:a}) RETURN u $$) AS (result agtype);
90+
91+
-- List comprehension in the WITH WHERE clause
92+
SELECT * FROM cypher('list_comprehension', $$ MATCH(u) WITH u WHERE u.list = [u IN [0, 2, 4, 6, 8, 10, 12]] RETURN u $$) AS (u agtype);
93+
SELECT * FROM cypher('list_comprehension', $$ MATCH(u) WITH u WHERE u.list = [u IN [0, 2, 4, 6, 8, 10, 12]] OR size(u.list) = 0 RETURN u $$) AS (u agtype);
94+
SELECT * FROM cypher('list_comprehension', $$ MATCH(u) WITH u WHERE u.list = [u IN [0, 2, 4, 6, 8, 10, 12]] OR size(u.list)::bool RETURN u $$) AS (u agtype);
95+
SELECT * FROM cypher('list_comprehension', $$ MATCH(u) WITH u WHERE u.list = [u IN [0, 2, 4, 6, 8, 10, 12]] OR NOT size(u.list)::bool RETURN u $$) AS (u agtype);
96+
97+
SELECT * FROM cypher('list_comprehension', $$ CREATE(u:csm_match {list: ['abc', 'def', 'ghi']}) $$) AS (u agtype);
98+
SELECT * FROM cypher('list_comprehension', $$ MATCH(u: csm_match) WITH u WHERE [u IN u.list][0] STARTS WITH "ab" RETURN u $$) AS (u agtype);
99+
100+
SELECT * FROM cypher('list_comprehension', $$ WITH [1, 2, 3] AS u WHERE u = [u IN [1, 2, 3]] RETURN u $$) AS (u agtype);
101+
SELECT * FROM cypher('list_comprehension', $$ WITH 1 AS u WHERE u = [u IN [1, 2, 3]][0] RETURN u $$) AS (u agtype);
102+
SELECT * FROM cypher('list_comprehension', $$ WITH ['abc', 'defgh'] AS u WHERE [v In u][1] STARTS WITH 'de' RETURN u $$) AS (u agtype);
103+
104+
-- List comprehension in UNWIND
105+
SELECT * FROM cypher('list_comprehension', $$ UNWIND [u IN [1, 2, 3]] AS u RETURN u $$) AS (u agtype);
106+
SELECT * FROM cypher('list_comprehension', $$ UNWIND [u IN [1, 2, 3] WHERE u > 1 | u*2] AS u RETURN u $$) AS (u agtype);
107+
108+
-- invalid use of aggregation in UNWIND
109+
SELECT * FROM cypher('list_comprehension', $$ WITH [1, 2, 3] AS u UNWIND collect(u) AS v RETURN v $$) AS (u agtype);
110+
111+
-- List comprehension in SET
112+
SELECT * FROM cypher('list_comprehension', $$ MATCH(u {list: [0, 2, 4, 6, 8, 10, 12]}) SET u.a = [u IN range(0, 5)] RETURN u $$) AS (u agtype);
113+
SELECT * FROM cypher('list_comprehension', $$ MATCH(u {list: [0, 2, 4, 6, 8, 10, 12]}) SET u.a = [u IN []] RETURN u $$) AS (u agtype);
114+
SELECT * FROM cypher('list_comprehension', $$ MATCH(u {list: [0, 2, 4, 6, 8, 10, 12]}) SET u += {b: [u IN range(0, 5)]} RETURN u $$) AS (u agtype);
115+
SELECT * FROM cypher('list_comprehension', $$ MATCH(u {list: [0, 2, 4, 6, 8, 10, 12]}) WITh u, collect(u.list) AS v SET u += {b: [u IN range(0, 5)]} SET u.c = [u IN v[0]] RETURN u $$) AS (u agtype);
116+
117+
-- invalid use of aggregation in SET
118+
SELECT * FROM cypher('list_comprehension', $$ MATCH(u {list: [0, 2, 4, 6, 8, 10, 12]}) SET u.c = collect(u.list) RETURN u $$) AS (u agtype);
119+
120+
-- Known issue
121+
SELECT * FROM cypher('list_comprehension', $$ MERGE (u {list:[i IN [1,2,3]]}) RETURN u $$) AS (result agtype);
122+
123+
-- List comprehension variable scoping
124+
SELECT * FROM cypher('list_comprehension', $$ WITH 1 AS m, [m IN [1, 2, 3]] AS n RETURN [m IN [1, 2, 3]] $$) AS (result agtype);
125+
SELECT * FROM cypher('list_comprehension', $$ WITH 1 AS m RETURN [m IN [1, 2, 3]], m $$) AS (result agtype, result2 agtype);
126+
SELECT * FROM cypher('list_comprehension', $$ WITH [m IN [1,2,3]] AS m RETURN [m IN [1, 2, 3]], m $$) AS (result agtype, result2 agtype);
127+
SELECT * FROM cypher('list_comprehension', $$ CREATE n=()-[:edge]->() RETURN [n IN nodes(n)] $$) AS (u agtype);
128+
129+
-- Multiple list comprehensions in RETURN and WITH clause
130+
SELECT * FROM cypher('list_comprehension', $$ RETURN [u IN [1,2,3]], [u IN [1,2,3]] $$) AS (result agtype, result2 agtype);
131+
SELECT * FROM cypher('list_comprehension', $$ RETURN [u IN [1,2,3] WHERE u>1], [u IN [1,2,3] WHERE u>2] $$) AS (result agtype, result2 agtype);
132+
SELECT * FROM cypher('list_comprehension', $$ RETURN [u IN [1,2,3] WHERE u>1 | u^3], [u IN [1,2,3] WHERE u>2 | u^3] $$) AS (result agtype, result2 agtype);
133+
134+
SELECT * FROM cypher('list_comprehension', $$ RETURN [u IN [1,2,3]] AS u, [u IN [1,2,3]] AS i $$) AS (result agtype, result2 agtype);
135+
SELECT * FROM cypher('list_comprehension', $$ RETURN [u IN [1,2,3] WHERE u>1] AS u, [u IN [1,2,3] WHERE u>2] AS i $$) AS (result agtype, result2 agtype);
136+
SELECT * FROM cypher('list_comprehension', $$ RETURN [u IN [1,2,3] WHERE u>1 | u^3] AS u, [u IN [1,2,3] WHERE u>2 | u^3] AS i $$) AS (result agtype, result2 agtype);
137+
138+
SELECT * FROM cypher('list_comprehension', $$ RETURN [u IN [u IN [1,2,3]]], [u IN [u IN [1,2,3]]] $$) AS (result agtype, result2 agtype);
139+
SELECT * FROM cypher('list_comprehension', $$ RETURN [u IN [u IN [1,2,3] WHERE u>1] WHERE u>2], [u IN [u IN [1,2,3] WHERE u>1] WHERE u>2] $$) AS (result agtype, result2 agtype);
140+
SELECT * FROM cypher('list_comprehension', $$ RETURN [u IN [u IN [1,2,3] WHERE u>1] WHERE u>2 | u^3], [u IN [u IN [1,2,3] WHERE u>1] WHERE u>2 | u^3] $$) AS (result agtype, result2 agtype);
141+
142+
SELECT * FROM cypher('list_comprehension', $$ WITH [u IN [1,2,3]] AS u, [u IN [1,2,3]] AS i RETURN u, i $$) AS (result agtype, result2 agtype);
143+
144+
-- Should error out
145+
SELECT * FROM cypher('list_comprehension', $$ RETURN [i IN range(0, 10, 2)],i $$) AS (result agtype, i agtype);
146+
SELECT * FROM cypher('list_comprehension', $$ RETURN [i IN range(0, 10, 2) WHERE i>5 | i^2], i $$) AS (result agtype, i agtype);
147+
148+
SELECT * FROM drop_graph('list_comprehension', true);

‎sql/agtype_typecast.sql

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -181,7 +181,8 @@ CREATE FUNCTION ag_catalog.age_range(variadic "any")
181181
PARALLEL SAFE
182182
AS 'MODULE_PATHNAME';
183183

184-
CREATE FUNCTION ag_catalog.age_unnest(agtype)
184+
CREATE FUNCTION ag_catalog.age_unnest(agtype,
185+
list_comprehension boolean = false)
185186
RETURNS SETOF agtype
186187
LANGUAGE c
187188
IMMUTABLE

‎src/backend/nodes/cypher_outfuncs.c

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,7 @@ void out_cypher_unwind(StringInfo str, const ExtensibleNode *node)
174174
DEFINE_AG_NODE(cypher_unwind);
175175

176176
WRITE_NODE_FIELD(target);
177+
WRITE_NODE_FIELD(collect);
177178
}
178179

179180
// serialization function for the cypher_delete ExtensibleNode.

0 commit comments

Comments
 (0)