Skip to content

Commit 849bd2e

Browse files
gruuyagruuya
committed
Add cases for more node types in deparse walker
This allows Multicorn to correctly classify the quals it supports into remote and local conditions. In turn, this allows us to decide whether pushdown is achievable or not. Additionally, inquire Python FDW instance about the qual operators it supports.
1 parent cbe45ed commit 849bd2e

5 files changed

Lines changed: 262 additions & 21 deletions

File tree

‎python/multicorn/__init__.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -228,12 +228,17 @@ def can_pushdown_upperrel(self):
228228
<PG_agg_func_name>: <foreign_agg_func_name>,
229229
...
230230
},
231+
"supported_operators": [">", "<", "=", ...]
231232
}
232233
233234
Each entry in `agg_functions` dict corresponds to a maping between
234235
the name of a aggregation function in PostgreSQL, and the equivalent
235236
foreign function. If no mapping exists for an aggregate function any
236237
queries containing it won't be pushed down.
238+
239+
The `supported_operators` entry lists all operators that can be used
240+
in qual (WHERE) clauses so that the aggregation pushdown will still
241+
be supported.
237242
"""
238243
return None
239244

‎src/deparse.c‎

Lines changed: 198 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,36 @@ typedef struct foreign_loc_cxt
7777
static Value *multicorn_deparse_function_name(Oid funcid);
7878

7979

80+
/*
81+
* Examine each qual clause in input_conds, and classify them into two groups,
82+
* which are returned as two lists:
83+
* - remote_conds contains expressions that can be evaluated remotely
84+
* - local_conds contains expressions that can't be evaluated remotely
85+
*/
86+
void
87+
multicorn_classify_conditions(PlannerInfo *root,
88+
RelOptInfo *baserel,
89+
List *input_conds,
90+
List **remote_conds,
91+
List **local_conds)
92+
{
93+
ListCell *lc;
94+
95+
*remote_conds = NIL;
96+
*local_conds = NIL;
97+
98+
foreach(lc, input_conds)
99+
{
100+
RestrictInfo *ri = lfirst_node(RestrictInfo, lc);
101+
102+
if (multicorn_is_foreign_expr(root, baserel, ri->clause))
103+
*remote_conds = lappend(*remote_conds, ri);
104+
else
105+
*local_conds = lappend(*local_conds, ri);
106+
}
107+
}
108+
109+
80110
/*
81111
* Return true if given object is one of PostgreSQL's built-in objects.
82112
*
@@ -190,6 +220,174 @@ multicorn_foreign_expr_walker(Node *node,
190220
}
191221
}
192222
break;
223+
case T_Const:
224+
{
225+
Const *c = (Const *) node;
226+
227+
/* TODO: see if Python FDW instance can handle interval type */
228+
if (c->consttype == INTERVALOID)
229+
return false;
230+
231+
/*
232+
* If the constant has nondefault collation, either it's of a
233+
* non-builtin type, or it reflects folding of a CollateExpr;
234+
* either way, it's unsafe to send to the remote.
235+
*/
236+
if (c->constcollid != InvalidOid &&
237+
c->constcollid != DEFAULT_COLLATION_OID)
238+
return false;
239+
240+
/* Otherwise, we can consider that it doesn't set collation */
241+
collation = InvalidOid;
242+
state = FDW_COLLATE_NONE;
243+
}
244+
break;
245+
case T_OpExpr:
246+
{
247+
OpExpr *oe = (OpExpr *) node;
248+
char *operatorName = get_opname(oe->opno);
249+
250+
/*
251+
* Consult Python FDW instance on portability of the operator.
252+
*/
253+
if (!list_member(fpinfo->operators_supported, makeString(operatorName)))
254+
return false;
255+
256+
/*
257+
* Similarly, only built-in operators can be sent to remote.
258+
* (If the operator is, surely its underlying function is
259+
* too.)
260+
*/
261+
if (!multicorn_is_builtin(oe->opno))
262+
return false;
263+
264+
/*
265+
* Recurse to input subexpressions.
266+
*/
267+
if (!multicorn_foreign_expr_walker((Node *) oe->args,
268+
glob_cxt, &inner_cxt))
269+
return false;
270+
271+
/*
272+
* If operator's input collation is not derived from a foreign
273+
* Var, it can't be sent to remote.
274+
*/
275+
if (oe->inputcollid == InvalidOid)
276+
/* OK, inputs are all noncollatable */ ;
277+
else if (inner_cxt.state != FDW_COLLATE_SAFE ||
278+
oe->inputcollid != inner_cxt.collation)
279+
return false;
280+
281+
/* Result-collation handling is same as for functions */
282+
collation = oe->opcollid;
283+
if (collation == InvalidOid)
284+
state = FDW_COLLATE_NONE;
285+
else if (inner_cxt.state == FDW_COLLATE_SAFE &&
286+
collation == inner_cxt.collation)
287+
state = FDW_COLLATE_SAFE;
288+
else
289+
state = FDW_COLLATE_UNSAFE;
290+
}
291+
break;
292+
case T_ScalarArrayOpExpr:
293+
{
294+
ScalarArrayOpExpr *oe = (ScalarArrayOpExpr *) node;
295+
296+
/*
297+
* Again, only built-in operators can be sent to remote.
298+
*/
299+
if (!multicorn_is_builtin(oe->opno))
300+
return false;
301+
302+
/*
303+
* Recurse to input subexpressions.
304+
*/
305+
if (!multicorn_foreign_expr_walker((Node *) oe->args,
306+
glob_cxt, &inner_cxt))
307+
return false;
308+
309+
/*
310+
* If operator's input collation is not derived from a foreign
311+
* Var, it can't be sent to remote.
312+
*/
313+
if (oe->inputcollid == InvalidOid)
314+
/* OK, inputs are all noncollatable */ ;
315+
else if (inner_cxt.state != FDW_COLLATE_SAFE ||
316+
oe->inputcollid != inner_cxt.collation)
317+
return false;
318+
319+
/* Output is always boolean and so noncollatable. */
320+
collation = InvalidOid;
321+
state = FDW_COLLATE_NONE;
322+
}
323+
break;
324+
case T_RelabelType:
325+
{
326+
RelabelType *r = (RelabelType *) node;
327+
328+
/*
329+
* Recurse to input subexpression.
330+
*/
331+
if (!multicorn_foreign_expr_walker((Node *) r->arg,
332+
glob_cxt, &inner_cxt))
333+
return false;
334+
335+
/*
336+
* RelabelType must not introduce a collation not derived from
337+
* an input foreign Var.
338+
*/
339+
collation = r->resultcollid;
340+
if (collation == InvalidOid)
341+
state = FDW_COLLATE_NONE;
342+
else if (inner_cxt.state == FDW_COLLATE_SAFE &&
343+
collation == inner_cxt.collation)
344+
state = FDW_COLLATE_SAFE;
345+
else
346+
state = FDW_COLLATE_UNSAFE;
347+
}
348+
break;
349+
case T_NullTest:
350+
{
351+
NullTest *nt = (NullTest *) node;
352+
353+
/*
354+
* Recurse to input subexpressions.
355+
*/
356+
if (!multicorn_foreign_expr_walker((Node *) nt->arg,
357+
glob_cxt, &inner_cxt))
358+
return false;
359+
360+
/* Output is always boolean and so noncollatable. */
361+
collation = InvalidOid;
362+
state = FDW_COLLATE_NONE;
363+
}
364+
break;
365+
case T_List:
366+
{
367+
List *l = (List *) node;
368+
ListCell *lc;
369+
370+
/*
371+
* Recurse to component subexpressions.
372+
*/
373+
foreach(lc, l)
374+
{
375+
if (!multicorn_foreign_expr_walker((Node *) lfirst(lc),
376+
glob_cxt, &inner_cxt))
377+
return false;
378+
}
379+
380+
/*
381+
* When processing a list, collation state just bubbles up
382+
* from the list elements.
383+
*/
384+
collation = inner_cxt.collation;
385+
state = inner_cxt.state;
386+
387+
/* Don't apply exprType() to the list. */
388+
check_type = false;
389+
}
390+
break;
193391
case T_Aggref:
194392
{
195393
Aggref *agg = (Aggref *) node;

‎src/multicorn.c‎

Lines changed: 16 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -308,12 +308,12 @@ multicornGetForeignRelSize(PlannerInfo *root,
308308
/* Base foreign tables need to be push down always */
309309
planstate->pushdown_safe = true;
310310

311-
/* Initialize upperrel pushdown info */
312-
planstate->groupby_supported = false;
313-
planstate->agg_functions = NULL;
314-
315311
planstate->fdw_instance = getInstance(foreigntableid);
316312
planstate->foreigntableid = foreigntableid;
313+
314+
/* Initialize upperrel pushdown info */
315+
initUpperrelPushdownInfo(planstate);
316+
317317
/* Initialize the conversion info array */
318318
{
319319
Relation rel = RelationIdGetRelation(ftable->relid);
@@ -402,7 +402,12 @@ multicornGetForeignRelSize(PlannerInfo *root,
402402

403403
}
404404

405-
planstate->baserestrictinfo = baserel->baserestrictinfo;
405+
/*
406+
* Identify which baserestrictinfo clauses can be sent to the remote server
407+
* and which can't.
408+
*/
409+
multicorn_classify_conditions(root, baserel, baserel->baserestrictinfo,
410+
&planstate->remote_conds, &planstate->local_conds);
406411

407412
/* Inject the "rows" and "width" attribute into the baserel */
408413
#if PG_VERSION_NUM >= 90600
@@ -589,7 +594,7 @@ multicornGetForeignPlan(PlannerInfo *root,
589594
* reason. We pass the clauses from the base relation obtained in MulticornGetForeignRelSize.
590595
*/
591596
ofpinfo = (MulticornPlanState *) planstate->outerrel->fdw_private;
592-
planstate->baserestrictinfo = extract_actual_clauses(ofpinfo->baserestrictinfo, false);
597+
planstate->remote_conds = extract_actual_clauses(ofpinfo->remote_conds, false);
593598
}
594599

595600
return make_foreignscan(tlist,
@@ -683,7 +688,7 @@ multicornBeginForeignScan(ForeignScanState *node, int eflags)
683688
* NB: This may not work well in case of joins, keep an eye out for it.
684689
*/
685690
rtindex = bms_next_member(fscan->fs_relids, -1);
686-
clauses = execstate->baserestrictinfo;
691+
clauses = execstate->remote_conds;
687692
}
688693

689694
execstate->values = palloc(sizeof(Datum) * execstate->tupdesc->natts);
@@ -1882,12 +1887,12 @@ multicornGetForeignUpperPaths(PlannerInfo *root, UpperRelationKind stage,
18821887

18831888
/*
18841889
* Check with the Python FDW instance whether it supports pushdown at all
1885-
* NB: Here we deviate from other FDWs, in that we don't know whether the
1890+
* NB: Here we deviate from other FDWs, in that we don't know whether
18861891
* something can be pushed down without consulting the corresponding Python
18871892
* FDW instance.
18881893
*/
18891894

1890-
if (!canPushdownUpperrel((MulticornPlanState *) input_rel->fdw_private))
1895+
if (!((MulticornPlanState *) input_rel->fdw_private)->upperrel_pushdown_supported)
18911896
{
18921897
return;
18931898
}
@@ -2010,7 +2015,7 @@ serializePlanState(MulticornPlanState * state)
20102015

20112016
result = lappend(result, state->group_clauses);
20122017

2013-
result = lappend(result, state->baserestrictinfo);
2018+
result = lappend(result, state->remote_conds);
20142019

20152020
return result;
20162021
}
@@ -2053,6 +2058,6 @@ initializeExecState(void *internalstate)
20532058
execstate->upper_rel_targets = list_nth(values, 4);
20542059
execstate->aggs = list_nth(values, 5);
20552060
execstate->group_clauses = list_nth(values, 6);
2056-
execstate->baserestrictinfo = list_nth(values, 7);
2061+
execstate->remote_conds = list_nth(values, 7);
20572062
return execstate;
20582063
}

‎src/multicorn.h‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -101,8 +101,10 @@ typedef struct MulticornPlanState
101101
int width;
102102

103103
/* Details about upperrel pushdown fetched from the Python FDW instance */
104+
bool upperrel_pushdown_supported;
104105
bool groupby_supported;
105106
List *agg_functions;
107+
List *operators_supported;
106108

107109
/*
108110
* Aggregation and grouping data to be passed to the execution phase.
@@ -118,9 +120,9 @@ typedef struct MulticornPlanState
118120
*/
119121
bool pushdown_safe;
120122

121-
/* qual clauses */
122-
List *baserestrictinfo;
123-
List *local_conds;
123+
/* baserestrictinfo clauses, broken down into safe and unsafe subsets. */
124+
List *remote_conds;
125+
List *local_conds;
124126

125127
/* Actual remote restriction clauses for scan (sans RestrictInfos) */
126128
List *final_remote_exprs;
@@ -194,9 +196,9 @@ typedef struct MulticornExecState
194196
*/
195197
List *group_clauses;
196198
/*
197-
* Qual conditions parsed in the MulticornGetForeignRelSize
199+
* Remote conditions parsed in the MulticornGetForeignRelSize
198200
*/
199-
List *baserestrictinfo;
201+
List *remote_conds;
200202

201203
/* Common buffer to avoid repeated allocations */
202204
StringInfo buffer;
@@ -270,6 +272,11 @@ typedef struct MulticornDeparsedSortGroup
270272
} MulticornDeparsedSortGroup;
271273

272274
/* deparse.c */
275+
extern void multicorn_classify_conditions(PlannerInfo *root,
276+
RelOptInfo *baserel,
277+
List *input_conds,
278+
List **remote_conds,
279+
List **local_conds);
273280
extern bool multicorn_is_foreign_expr(PlannerInfo *root,
274281
RelOptInfo *baserel,
275282
Expr *expr);
@@ -298,7 +305,7 @@ PyObject *tupleTableSlotToPyObject(TupleTableSlot *slot, ConversionInfo ** cin
298305
char *getRowIdColumn(PyObject *fdw_instance);
299306
PyObject *optionsListToPyDict(List *options);
300307
const char *getPythonEncodingName(void);
301-
bool canPushdownUpperrel(MulticornPlanState * state);
308+
void initUpperrelPushdownInfo(MulticornPlanState * state);
302309

303310
void getRelSize(MulticornPlanState * state,
304311
PlannerInfo *root,

0 commit comments

Comments
 (0)