in asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SqlppExpressionToPlanTranslator.java [1211:1637]
public Pair<ILogicalOperator, LogicalVariable> visit(WindowExpression winExpr, Mutable<ILogicalOperator> tupSource)
throws CompilationException {
SourceLocation sourceLoc = winExpr.getSourceLocation();
List<Expression> fargs = winExpr.getExprList();
FunctionSignature fs = winExpr.getFunctionSignature();
IFunctionInfo finfo = BuiltinFunctions.getBuiltinFunctionInfo(fs.createFunctionIdentifier());
if (finfo == null) {
throw new CompilationException(ErrorCode.COMPILATION_EXPECTED_WINDOW_FUNCTION, winExpr.getSourceLocation(),
fs.getName());
}
FunctionIdentifier fi = finfo.getFunctionIdentifier();
boolean isWin = BuiltinFunctions.isWindowFunction(fi);
boolean isWinAgg = isWin && BuiltinFunctions.builtinFunctionHasProperty(fi,
BuiltinFunctions.WindowFunctionProperty.HAS_LIST_ARG);
boolean prohibitOrderClause = (isWin && BuiltinFunctions.builtinFunctionHasProperty(fi,
BuiltinFunctions.WindowFunctionProperty.NO_ORDER_CLAUSE))
|| (!isWin && BuiltinFunctions.builtinFunctionHasProperty(fi,
BuiltinFunctions.AggregateFunctionProperty.NO_ORDER_CLAUSE));
boolean prohibitFrameClause = (isWin && BuiltinFunctions.builtinFunctionHasProperty(fi,
BuiltinFunctions.WindowFunctionProperty.NO_FRAME_CLAUSE))
|| (!isWin && BuiltinFunctions.builtinFunctionHasProperty(fi,
BuiltinFunctions.AggregateFunctionProperty.NO_FRAME_CLAUSE));
boolean allowRespectIgnoreNulls = isWin && BuiltinFunctions.builtinFunctionHasProperty(fi,
BuiltinFunctions.WindowFunctionProperty.ALLOW_RESPECT_IGNORE_NULLS);
boolean allowFromFirstLast = isWin && BuiltinFunctions.builtinFunctionHasProperty(fi,
BuiltinFunctions.WindowFunctionProperty.ALLOW_FROM_FIRST_LAST);
if (winExpr.hasAggregateFilterExpr()) {
throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_USE_OF_FILTER_CLAUSE, sourceLoc);
}
Mutable<ILogicalOperator> currentOpRef = tupSource;
List<Mutable<ILogicalExpression>> partExprListOut = Collections.emptyList();
if (winExpr.hasPartitionList()) {
List<Expression> partExprList = winExpr.getPartitionList();
partExprListOut = new ArrayList<>(partExprList.size());
for (Expression partExpr : partExprList) {
Pair<ILogicalOperator, LogicalVariable> partExprResult = partExpr.accept(this, currentOpRef);
VariableReferenceExpression partExprOut = new VariableReferenceExpression(partExprResult.second);
partExprOut.setSourceLocation(partExpr.getSourceLocation());
partExprListOut.add(new MutableObject<>(partExprOut));
currentOpRef = new MutableObject<>(partExprResult.first);
}
}
int orderExprCount = 0;
List<Pair<OrderOperator.IOrder, Mutable<ILogicalExpression>>> orderExprListOut = Collections.emptyList();
List<Pair<OrderOperator.IOrder, Mutable<ILogicalExpression>>> frameValueExprRefs = null;
List<Mutable<ILogicalExpression>> frameStartExprRefs = null;
List<Mutable<ILogicalExpression>> frameStartValidationExprRefs = null;
List<Mutable<ILogicalExpression>> frameEndExprRefs = null;
List<Mutable<ILogicalExpression>> frameEndValidationExprRefs = null;
List<Mutable<ILogicalExpression>> frameExcludeExprRefs = null;
int frameExcludeNotStartIdx = -1;
AbstractLogicalExpression frameExcludeUnaryExpr = null;
AbstractLogicalExpression frameOffsetExpr = null;
if (winExpr.hasOrderByList()) {
if (prohibitOrderClause) {
throw new CompilationException(ErrorCode.COMPILATION_UNEXPECTED_WINDOW_ORDERBY, sourceLoc);
}
List<Expression> orderExprList = winExpr.getOrderbyList();
List<OrderbyClause.OrderModifier> orderModifierList = winExpr.getOrderbyModifierList();
List<OrderbyClause.NullOrderModifier> nullOrderModifierList = winExpr.getOrderbyNullModifierList();
orderExprCount = orderExprList.size();
orderExprListOut = new ArrayList<>(orderExprCount);
for (int i = 0; i < orderExprCount; i++) {
Expression orderExpr = orderExprList.get(i);
OrderbyClause.OrderModifier orderModifier = orderModifierList.get(i);
OrderbyClause.NullOrderModifier nullOrderModifier = nullOrderModifierList.get(i);
Pair<ILogicalOperator, LogicalVariable> orderExprResult = orderExpr.accept(this, currentOpRef);
VariableReferenceExpression orderExprOut = new VariableReferenceExpression(orderExprResult.second);
addOrderByExpression(orderExprListOut, orderExprOut, orderModifier, nullOrderModifier);
currentOpRef = new MutableObject<>(orderExprResult.first);
}
} else if (winExpr.hasFrameDefinition()) {
// frame definition without ORDER BY is not allowed by the grammar
throw new CompilationException(ErrorCode.COMPILATION_UNEXPECTED_WINDOW_FRAME, sourceLoc);
}
WindowExpression.FrameMode winFrameMode = null;
WindowExpression.FrameExclusionKind winFrameExclusionKind = null;
WindowExpression.FrameBoundaryKind winFrameStartKind = null, winFrameEndKind = null;
Expression winFrameStartExpr = null, winFrameEndExpr = null;
AbstractExpression winFrameExcludeUnaryExpr = null, winFrameOffsetExpr = null;
int winFrameMaxOjbects = WindowOperator.FRAME_MAX_OBJECTS_UNLIMITED;
if (winExpr.hasFrameDefinition()) {
if (prohibitFrameClause) {
throw new CompilationException(ErrorCode.COMPILATION_UNEXPECTED_WINDOW_FRAME, sourceLoc);
}
winFrameMode = winExpr.getFrameMode();
winFrameStartKind = winExpr.getFrameStartKind();
winFrameStartExpr = winExpr.getFrameStartExpr();
winFrameEndKind = winExpr.getFrameEndKind();
winFrameEndExpr = winExpr.getFrameEndExpr();
winFrameExclusionKind = winExpr.getFrameExclusionKind();
if (!isValidWindowFrameDefinition(winFrameMode, winFrameStartKind, winFrameEndKind, orderExprCount)) {
throw new CompilationException(ErrorCode.COMPILATION_INVALID_WINDOW_FRAME, sourceLoc);
}
} else if (!prohibitFrameClause) {
winFrameMode = WindowExpression.FrameMode.RANGE;
winFrameStartKind = WindowExpression.FrameBoundaryKind.UNBOUNDED_PRECEDING;
winFrameEndKind = WindowExpression.FrameBoundaryKind.CURRENT_ROW;
winFrameExclusionKind = WindowExpression.FrameExclusionKind.NO_OTHERS;
}
boolean respectNulls = !getBooleanModifier(winExpr.getIgnoreNulls(), false, allowRespectIgnoreNulls, sourceLoc,
"RESPECT/IGNORE NULLS", fs.getName());
boolean fromLast = getBooleanModifier(winExpr.getFromLast(), false, allowFromFirstLast, sourceLoc,
"FROM FIRST/LAST", fs.getName());
boolean makeRunningAgg = false, makeNestedAgg = false;
FunctionIdentifier runningAggFunc = null, nestedAggFunc = null, winResultFunc = null, postWinResultFunc = null;
Expression postWinExpr = null;
List<Expression> nestedAggArgs = null;
boolean postWinResultArgsReverse = false;
if (isWinAgg) {
makeNestedAgg = true;
nestedAggArgs = new ArrayList<>(fargs.size());
nestedAggArgs.add(fargs.get(0));
boolean isLead;
if ((isLead = BuiltinFunctions.LEAD_IMPL.equals(fi)) || BuiltinFunctions.LAG_IMPL.equals(fi)) {
boolean isLag = !isLead;
int argCount = fargs.size();
// LEAD_IMPL() and LAG_IMPL() have one extra (last) argument introduced by SqlppWindowRewriteVisitor
// which is a copy of the original first argument
if (argCount < 2 || argCount > 4) {
throw new CompilationException(ErrorCode.COMPILATION_INVALID_NUM_OF_ARGS, sourceLoc, fi.getName());
}
winFrameMode = WindowExpression.FrameMode.ROWS;
winFrameExclusionKind = WindowExpression.FrameExclusionKind.NO_OTHERS;
if (respectNulls) {
winFrameStartKind = winFrameEndKind = isLead ? WindowExpression.FrameBoundaryKind.BOUNDED_FOLLOWING
: WindowExpression.FrameBoundaryKind.BOUNDED_PRECEDING;
winFrameStartExpr = argCount == 2 ? new LiteralExpr(new IntegerLiteral(1)) : fargs.get(1);
winFrameEndExpr = (Expression) SqlppRewriteUtil.deepCopy(winFrameStartExpr);
winFrameMaxOjbects = 1;
} else {
// IGNORE NULLS
if (isLag) {
// reverse order for LAG()
reverseOrder(orderExprListOut);
}
winFrameStartKind = WindowExpression.FrameBoundaryKind.BOUNDED_FOLLOWING;
winFrameStartExpr = new LiteralExpr(new IntegerLiteral(1));
winFrameEndKind = WindowExpression.FrameBoundaryKind.UNBOUNDED_FOLLOWING;
Expression fargLast = fargs.get(fargs.size() - 1);
winFrameExcludeUnaryExpr = createCallExpr(BuiltinFunctions.IS_UNKNOWN, fargLast, sourceLoc);
if (argCount > 2) {
winFrameOffsetExpr =
createOperatorExpr(fargs.get(1), OperatorType.MINUS, new IntegerLiteral(1), sourceLoc);
}
}
if (argCount < 4) {
nestedAggFunc = BuiltinFunctions.SCALAR_FIRST_ELEMENT;
} else {
// return SYSTEM_NULL if required offset is not reached
nestedAggFunc = BuiltinFunctions.SCALAR_LOCAL_FIRST_ELEMENT;
postWinResultFunc = BuiltinFunctions.IF_SYSTEM_NULL;
postWinExpr = fargs.get(2);
}
} else if (BuiltinFunctions.FIRST_VALUE_IMPL.equals(fi)) {
nestedAggFunc = BuiltinFunctions.SCALAR_FIRST_ELEMENT;
if (respectNulls) {
winFrameMaxOjbects = 1;
} else {
Expression fargLast = fargs.get(fargs.size() - 1);
winFrameExcludeUnaryExpr = createCallExpr(BuiltinFunctions.IS_UNKNOWN, fargLast, sourceLoc);
}
} else if (BuiltinFunctions.LAST_VALUE_IMPL.equals(fi)) {
nestedAggFunc = BuiltinFunctions.SCALAR_LAST_ELEMENT;
if (!respectNulls) {
Expression fargLast = fargs.get(fargs.size() - 1);
winFrameExcludeUnaryExpr = createCallExpr(BuiltinFunctions.IS_UNKNOWN, fargLast, sourceLoc);
}
} else if (BuiltinFunctions.NTH_VALUE_IMPL.equals(fi)) {
nestedAggFunc = BuiltinFunctions.SCALAR_FIRST_ELEMENT;
if (fromLast) {
// if FROM LAST modifier is present
// 1. reverse order
reverseOrder(orderExprListOut);
// 2. reverse frame specification
WindowExpression.FrameBoundaryKind tmpFrameStartKind = winFrameStartKind;
Expression tmpFrameStartExpr = winFrameStartExpr;
winFrameStartKind = reverseFrameBoundaryKind(winFrameEndKind);
winFrameStartExpr = winFrameEndExpr;
winFrameEndKind = reverseFrameBoundaryKind(tmpFrameStartKind);
winFrameEndExpr = tmpFrameStartExpr;
}
if (respectNulls) {
winFrameMaxOjbects = 1;
} else {
Expression fargLast = fargs.get(fargs.size() - 1);
winFrameExcludeUnaryExpr = createCallExpr(BuiltinFunctions.IS_UNKNOWN, fargLast, sourceLoc);
}
winFrameOffsetExpr =
createOperatorExpr(fargs.get(1), OperatorType.MINUS, new IntegerLiteral(1), sourceLoc);
} else if (BuiltinFunctions.RATIO_TO_REPORT_IMPL.equals(fi)) {
// ratio_to_report(x) over (...) --> x / sum(x) over (...)
nestedAggFunc = BuiltinFunctions.SCALAR_SQL_SUM;
postWinResultFunc = BuiltinFunctions.NUMERIC_DIVIDE;
postWinExpr = fargs.get(1);
postWinResultArgsReverse = true;
} else {
throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, sourceLoc, fi.getName());
}
} else if (isWin) {
makeRunningAgg = true;
if (BuiltinFunctions.CUME_DIST_IMPL.equals(fi)) {
winFrameMode = WindowExpression.FrameMode.RANGE;
winFrameStartKind = WindowExpression.FrameBoundaryKind.UNBOUNDED_PRECEDING;
winFrameEndKind = WindowExpression.FrameBoundaryKind.CURRENT_ROW;
winFrameExclusionKind = WindowExpression.FrameExclusionKind.NO_OTHERS;
makeNestedAgg = true;
runningAggFunc = BuiltinFunctions.WIN_PARTITION_LENGTH_IMPL;
nestedAggFunc = BuiltinFunctions.SCALAR_COUNT;
nestedAggArgs = mkSingletonArrayList((Expression) SqlppRewriteUtil.deepCopy(winExpr.getWindowVar()));
winResultFunc = BuiltinFunctions.NUMERIC_DIVIDE;
} else {
runningAggFunc = fi;
}
} else { // regular aggregate
makeNestedAgg = true;
nestedAggFunc = fi;
nestedAggArgs = fargs;
}
if (winFrameMode != null) {
LogicalVariable rowNumVar = context.newVar();
LogicalVariable denseRankVar = context.newVar();
ListSet<LogicalVariable> usedVars = new ListSet<>();
frameValueExprRefs = translateWindowFrameMode(winFrameMode, winFrameStartKind, winFrameEndKind,
orderExprListOut, rowNumVar, denseRankVar, usedVars, sourceLoc);
Pair<List<Mutable<ILogicalExpression>>, Integer> frameExclusionResult =
translateWindowExclusion(winFrameExclusionKind, rowNumVar, denseRankVar, usedVars, sourceLoc);
if (frameExclusionResult != null) {
frameExcludeExprRefs = frameExclusionResult.first;
frameExcludeNotStartIdx = frameExclusionResult.second;
}
if (!usedVars.isEmpty()) {
List<Mutable<ILogicalExpression>> partExprListClone =
OperatorManipulationUtil.cloneExpressions(partExprListOut);
List<Pair<OrderOperator.IOrder, Mutable<ILogicalExpression>>> orderExprListClone =
OperatorManipulationUtil.cloneOrderExpressions(orderExprListOut);
WindowOperator helperWinOp = createHelperWindowOperator(partExprListClone, orderExprListClone,
rowNumVar, denseRankVar, usedVars, sourceLoc);
helperWinOp.getInputs().add(currentOpRef);
currentOpRef = new MutableObject<>(helperWinOp);
}
Triple<ILogicalOperator, List<Mutable<ILogicalExpression>>, List<Mutable<ILogicalExpression>>> frameStartResult =
translateWindowBoundary(winFrameStartKind, winFrameStartExpr, frameValueExprRefs, orderExprListOut,
currentOpRef);
if (frameStartResult != null) {
frameStartExprRefs = frameStartResult.second;
frameStartValidationExprRefs = frameStartResult.third;
if (frameStartResult.first != null) {
currentOpRef = new MutableObject<>(frameStartResult.first);
}
}
Triple<ILogicalOperator, List<Mutable<ILogicalExpression>>, List<Mutable<ILogicalExpression>>> frameEndResult =
translateWindowBoundary(winFrameEndKind, winFrameEndExpr, frameValueExprRefs, orderExprListOut,
currentOpRef);
if (frameEndResult != null) {
frameEndExprRefs = frameEndResult.second;
frameEndValidationExprRefs = frameEndResult.third;
if (frameEndResult.first != null) {
currentOpRef = new MutableObject<>(frameEndResult.first);
}
}
if (winFrameExcludeUnaryExpr != null) {
Pair<ILogicalOperator, LogicalVariable> frameExcludeUnaryResult =
winFrameExcludeUnaryExpr.accept(this, currentOpRef);
frameExcludeUnaryExpr = new VariableReferenceExpression(frameExcludeUnaryResult.second);
frameExcludeUnaryExpr.setSourceLocation(sourceLoc);
currentOpRef = new MutableObject<>(frameExcludeUnaryResult.first);
}
if (winFrameOffsetExpr != null) {
Pair<ILogicalOperator, LogicalVariable> frameOffsetResult =
winFrameOffsetExpr.accept(this, currentOpRef);
frameOffsetExpr = new VariableReferenceExpression(frameOffsetResult.second);
frameOffsetExpr.setSourceLocation(sourceLoc);
currentOpRef = new MutableObject<>(frameOffsetResult.first);
}
}
WindowOperator winOp = new WindowOperator(partExprListOut, orderExprListOut, frameValueExprRefs,
frameStartExprRefs, frameStartValidationExprRefs, frameEndExprRefs, frameEndValidationExprRefs,
frameExcludeExprRefs, frameExcludeNotStartIdx, frameExcludeUnaryExpr, frameOffsetExpr,
winFrameMaxOjbects);
winOp.setSourceLocation(sourceLoc);
LogicalVariable runningAggResultVar = null, nestedAggResultVar = null;
if (makeNestedAgg) {
LogicalVariable windowRecordVar = context.newVar();
ILogicalExpression windowRecordConstr =
createRecordConstructor(winExpr.getWindowFieldList(), currentOpRef, sourceLoc);
AssignOperator assignOp = new AssignOperator(windowRecordVar, new MutableObject<>(windowRecordConstr));
assignOp.getInputs().add(currentOpRef);
assignOp.setSourceLocation(sourceLoc);
NestedTupleSourceOperator ntsOp = new NestedTupleSourceOperator(new MutableObject<>(winOp));
ntsOp.setSourceLocation(sourceLoc);
VariableReferenceExpression frameRecordVarRef = new VariableReferenceExpression(windowRecordVar);
frameRecordVarRef.setSourceLocation(sourceLoc);
AggregateFunctionCallExpression listifyCall = BuiltinFunctions.makeAggregateFunctionExpression(
BuiltinFunctions.LISTIFY, mkSingletonArrayList(new MutableObject<>(frameRecordVarRef)));
listifyCall.setSourceLocation(sourceLoc);
LogicalVariable windowVar = context.newVar();
AggregateOperator aggOp = new AggregateOperator(mkSingletonArrayList(windowVar),
mkSingletonArrayList(new MutableObject<>(listifyCall)));
aggOp.getInputs().add(new MutableObject<>(ntsOp));
aggOp.setSourceLocation(sourceLoc);
context.setVar(winExpr.getWindowVar(), windowVar);
CallExpr callExpr = new CallExpr(new FunctionSignature(nestedAggFunc), nestedAggArgs);
Pair<ILogicalOperator, LogicalVariable> exprResult = callExpr.accept(this, new MutableObject<>(aggOp));
winOp.getNestedPlans().add(new ALogicalPlanImpl(new MutableObject<>(exprResult.first)));
currentOpRef = new MutableObject<>(assignOp);
nestedAggResultVar = exprResult.second;
}
if (makeRunningAgg) {
CallExpr callExpr = new CallExpr(new FunctionSignature(runningAggFunc), fargs);
Pair<ILogicalOperator, LogicalVariable> callExprResult = callExpr.accept(this, currentOpRef);
ILogicalOperator op = callExprResult.first;
if (op.getOperatorTag() != LogicalOperatorTag.ASSIGN) {
throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, sourceLoc, "");
}
AssignOperator assignOp = (AssignOperator) op;
List<LogicalVariable> assignVars = assignOp.getVariables();
if (assignVars.size() != 1) {
throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, sourceLoc, "");
}
List<Mutable<ILogicalExpression>> assignExprs = assignOp.getExpressions();
if (assignExprs.size() != 1) {
throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, sourceLoc, "");
}
ILogicalExpression assignExpr = assignExprs.get(0).getValue();
if (assignExpr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, sourceLoc, "");
}
AbstractFunctionCallExpression fcallExpr = (AbstractFunctionCallExpression) assignExpr;
if (fcallExpr.getKind() != AbstractFunctionCallExpression.FunctionKind.STATEFUL) {
throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, sourceLoc, fcallExpr.getKind());
}
if (BuiltinFunctions.builtinFunctionHasProperty(fi,
BuiltinFunctions.WindowFunctionProperty.INJECT_ORDER_ARGS)) {
for (Pair<OrderOperator.IOrder, Mutable<ILogicalExpression>> p : orderExprListOut) {
fcallExpr.getArguments().add(new MutableObject<>(p.second.getValue().cloneExpression()));
}
}
winOp.getVariables().addAll(assignVars);
winOp.getExpressions().addAll(assignExprs);
currentOpRef = new MutableObject<>(assignOp.getInputs().get(0).getValue());
runningAggResultVar = assignVars.get(0);
}
winOp.getInputs().add(currentOpRef);
currentOpRef = new MutableObject<>(winOp);
AbstractLogicalExpression resultExpr;
if (makeRunningAgg && makeNestedAgg) {
VariableReferenceExpression runningAggResultVarRef = new VariableReferenceExpression(runningAggResultVar);
runningAggResultVarRef.setSourceLocation(sourceLoc);
VariableReferenceExpression nestedAggResultVarRef = new VariableReferenceExpression(nestedAggResultVar);
nestedAggResultVarRef.setSourceLocation(sourceLoc);
AbstractFunctionCallExpression resultCallExpr = createFunctionCallExpression(winResultFunc, sourceLoc);
resultCallExpr.getArguments().add(new MutableObject<>(nestedAggResultVarRef));
resultCallExpr.getArguments().add(new MutableObject<>(runningAggResultVarRef));
resultExpr = resultCallExpr;
} else if (makeRunningAgg) {
resultExpr = new VariableReferenceExpression(runningAggResultVar);
resultExpr.setSourceLocation(sourceLoc);
} else if (makeNestedAgg) {
resultExpr = new VariableReferenceExpression(nestedAggResultVar);
resultExpr.setSourceLocation(sourceLoc);
} else {
throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, sourceLoc, "");
}
if (postWinExpr != null) {
Pair<ILogicalOperator, LogicalVariable> postWinExprResult = postWinExpr.accept(this, currentOpRef);
currentOpRef = new MutableObject<>(postWinExprResult.first);
VariableReferenceExpression postWinVarRef = new VariableReferenceExpression(postWinExprResult.second);
postWinVarRef.setSourceLocation(sourceLoc);
AbstractFunctionCallExpression postWinResultCallExpr =
createFunctionCallExpression(postWinResultFunc, sourceLoc);
List<Mutable<ILogicalExpression>> postWinResultCallArgs = postWinResultCallExpr.getArguments();
if (!postWinResultArgsReverse) {
postWinResultCallArgs.add(new MutableObject<>(resultExpr));
postWinResultCallArgs.add(new MutableObject<>(postWinVarRef));
} else {
postWinResultCallArgs.add(new MutableObject<>(postWinVarRef));
postWinResultCallArgs.add(new MutableObject<>(resultExpr));
}
resultExpr = postWinResultCallExpr;
}
// must return ASSIGN
LogicalVariable resultVar = context.newVar();
AssignOperator resultOp = new AssignOperator(resultVar, new MutableObject<>(resultExpr));
resultOp.setSourceLocation(sourceLoc);
resultOp.getInputs().add(currentOpRef);
return new Pair<>(resultOp, resultVar);
}