in fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java [552:901]
private void turnOffPreAgg(AggregateInfo aggInfo, SelectStmt selectStmt, Analyzer analyzer, PlanNode root) {
String turnOffReason = null;
do {
if (!(root instanceof OlapScanNode)) {
turnOffReason = "left-deep Node is not OlapScanNode";
break;
}
if (((OlapScanNode) root).getForceOpenPreAgg()) {
((OlapScanNode) root).setIsPreAggregation(true, "");
return;
}
if (null == aggInfo) {
turnOffReason = "No AggregateInfo";
break;
}
ArrayList<FunctionCallExpr> aggExprs = aggInfo.getAggregateExprs();
// multi table join
boolean aggTableValidate = true;
if (selectStmt.getTableRefs().size() > 1) {
for (int i = 1; i < selectStmt.getTableRefs().size(); ++i) {
final JoinOperator joinOperator = selectStmt.getTableRefs().get(i).getJoinOp();
// TODO chenhao , right out join ?
if (joinOperator.isRightOuterJoin() || joinOperator.isFullOuterJoin()) {
turnOffReason = selectStmt.getTableRefs().get(i)
+ " joinOp is full outer join or right outer join.";
aggTableValidate = false;
break;
}
}
if (!aggTableValidate) {
break;
}
for (FunctionCallExpr aggExpr : aggExprs) {
TableRef olapTableRef = selectStmt.getTableRefs().get(0);
if (Expr.isBound(Lists.newArrayList(aggExpr), Lists.newArrayList(olapTableRef.getId()))) {
// do nothing
if (LOG.isDebugEnabled()) {
LOG.debug("All agg exprs is bound to olapTable: {}" + olapTableRef.getTable().getName());
}
} else {
List<TupleId> tupleIds = Lists.newArrayList();
List<SlotId> slotIds = Lists.newArrayList();
aggExpr.getIds(tupleIds, slotIds);
for (TupleId tupleId : tupleIds) {
// if tupleid is agg's result tuple, there is no tableref
// for only scanNode has the tableref
if (analyzer.getTupleDesc(tupleId).getRef() == null) {
aggTableValidate = false;
break;
}
if (analyzer.getTupleDesc(tupleId).getRef() != olapTableRef) {
if (analyzer.getTupleDesc(tupleId).getTable() != null
&& analyzer.getTupleDesc(tupleId).getTable().getType()
== Table.TableType.OLAP) {
turnOffReason = "agg expr [" + aggExpr.debugString() + "] is not bound ["
+ selectStmt.getTableRefs().get(0).toSql() + "]";
aggTableValidate = false;
} else {
if (LOG.isDebugEnabled()) {
LOG.debug("The table which agg expr [{}] is bound to, is not OLAP table [{}]",
aggExpr.debugString(),
analyzer.getTupleDesc(tupleId).getTable() == null ? "inline view" :
analyzer.getTupleDesc(tupleId).getTable().getName());
}
}
}
}
}
}
if (!aggTableValidate) {
break;
}
}
boolean valueColumnValidate = true;
List<Expr> allConjuncts = analyzer.getAllConjuncts(selectStmt.getTableRefs().get(0).getId());
List<SlotId> conjunctSlotIds = Lists.newArrayList();
if (allConjuncts != null) {
for (Expr conjunct : allConjuncts) {
if (!conjunct.isAuxExpr()) {
conjunct.getIds(null, conjunctSlotIds);
}
}
for (SlotDescriptor slot : selectStmt.getTableRefs().get(0).getDesc().getSlots()) {
if (!slot.getColumn().isKey()) {
if (conjunctSlotIds.contains(slot.getId())) {
turnOffReason = "conjunct on `" + slot.getColumn().getName()
+ "` which is StorageEngine value column";
valueColumnValidate = false;
break;
}
}
}
}
if (!valueColumnValidate) {
break;
}
boolean aggExprValidate = true;
for (FunctionCallExpr aggExpr : aggExprs) {
if (aggExpr.getChildren().size() != 1) {
turnOffReason = "aggExpr has more than one child";
aggExprValidate = false;
break;
}
List<Column> returnColumns = Lists.newArrayList();
List<Column> conditionColumns = Lists.newArrayList();
if (!(aggExpr.getChild(0) instanceof SlotRef)) {
Expr child = aggExpr.getChild(0);
// ignore cast
boolean castReturnExprValidate = true;
while (child instanceof CastExpr) {
if (child.getChild(0) instanceof SlotRef) {
if (child.getType().isNumericType() && child.getChild(0).getType().isNumericType()) {
returnColumns.add(((SlotRef) child.getChild(0)).getDesc().getColumn());
} else {
turnOffReason = "aggExpr.getChild(0)["
+ aggExpr.getChild(0).toSql()
+ "] is not Numeric CastExpr";
castReturnExprValidate = false;
break;
}
}
child = child.getChild(0);
}
if (!castReturnExprValidate) {
aggExprValidate = false;
break;
}
// convert IF to CASE WHEN.
// For example:
// IF(a > 1, 1, 0) -> CASE WHEN a > 1 THEN 1 ELSE 0 END
if (child instanceof FunctionCallExpr && ((FunctionCallExpr) child)
.getFnName().getFunction().equalsIgnoreCase("IF")) {
Preconditions.checkArgument(child.getChildren().size() == 3);
CaseWhenClause caseWhenClause = new CaseWhenClause(child.getChild(0), child.getChild(1));
child = new CaseExpr(ImmutableList.of(caseWhenClause), child.getChild(2));
}
if (child instanceof CaseExpr) {
CaseExpr caseExpr = (CaseExpr) child;
List<Expr> conditionExprs = caseExpr.getConditionExprs();
for (Expr conditionExpr : conditionExprs) {
List<TupleId> conditionTupleIds = Lists.newArrayList();
List<SlotId> conditionSlotIds = Lists.newArrayList();
conditionExpr.getIds(conditionTupleIds, conditionSlotIds);
for (SlotId conditionSlotId : conditionSlotIds) {
conditionColumns.add(analyzer.getDescTbl().getSlotDesc(conditionSlotId).getColumn());
}
}
boolean caseReturnExprValidate = true;
List<Expr> returnExprs = caseExpr.getReturnExprs();
for (Expr returnExpr : returnExprs) {
// ignore cast in return expr
while (returnExpr instanceof CastExpr) {
returnExpr = returnExpr.getChild(0);
}
if (returnExpr instanceof SlotRef) {
returnColumns.add(((SlotRef) returnExpr).getDesc().getColumn());
} else if (returnExpr.isNullLiteral() || returnExpr.isZeroLiteral()) {
// If then expr is NULL or Zero, open the preaggregation
} else {
turnOffReason = "aggExpr.getChild(0)[" + aggExpr.getChild(0).toSql()
+ "] is not SlotExpr";
caseReturnExprValidate = false;
break;
}
}
if (!caseReturnExprValidate) {
aggExprValidate = false;
break;
}
} else {
turnOffReason = "aggExpr.getChild(0)[" + aggExpr.getChild(0).debugString()
+ "] is not SlotRef or CastExpr|CaseExpr";
aggExprValidate = false;
break;
}
} else {
returnColumns.add(((SlotRef) aggExpr.getChild(0)).getDesc().getColumn());
}
// check condition columns
boolean conditionColumnValidate = true;
for (Column col : conditionColumns) {
// TODO(zc): Here column is null is too bad
// Only column of Inline-view will be null
if (col == null) {
continue;
}
if (!col.isKey()) {
turnOffReason = "the condition column [" + col.getName() + "] is not key type in aggr expr ["
+ aggExpr.toSql() + "].";
conditionColumnValidate = false;
break;
}
}
if (!conditionColumnValidate) {
aggExprValidate = false;
break;
}
// check return columns
boolean returnColumnValidate = true;
for (Column col : returnColumns) {
// TODO(zc): Here column is null is too bad
// Only column of Inline-view will be null
if (col == null) {
continue;
}
String functionName = aggExpr.getFnName().getFunction();
if (col.isKey()) {
if ((!functionName.equalsIgnoreCase("MAX"))
&& (!aggExpr.getFnName().getFunction().equalsIgnoreCase("MIN"))) {
returnColumnValidate = false;
turnOffReason = "the type of agg on StorageEngine's Key column should only be MAX or MIN."
+ "agg expr: " + aggExpr.toSql();
break;
}
}
if (functionName.equalsIgnoreCase("SUM")) {
if (col.getAggregationType() != AggregateType.SUM) {
turnOffReason = "Aggregate Operator not match: SUM <--> " + col.getAggregationType();
returnColumnValidate = false;
break;
}
} else if (functionName.equalsIgnoreCase("MAX")) {
if ((!col.isKey()) && col.getAggregationType() != AggregateType.MAX) {
turnOffReason = "Aggregate Operator not match: MAX <--> " + col.getAggregationType();
returnColumnValidate = false;
break;
}
} else if (functionName.equalsIgnoreCase("MIN")) {
if ((!col.isKey()) && col.getAggregationType() != AggregateType.MIN) {
turnOffReason = "Aggregate Operator not match: MIN <--> " + col.getAggregationType();
returnColumnValidate = false;
break;
}
} else if (functionName.equalsIgnoreCase(FunctionSet.HLL_UNION_AGG)
|| functionName.equalsIgnoreCase(FunctionSet.HLL_RAW_AGG)
|| functionName.equalsIgnoreCase(FunctionSet.HLL_UNION)) {
if (col.getAggregationType() != AggregateType.HLL_UNION) {
turnOffReason =
"Aggregate Operator not match: HLL_UNION <--> " + col.getAggregationType();
returnColumnValidate = false;
break;
}
} else if (functionName.equalsIgnoreCase("NDV")) {
if ((!col.isKey())) {
turnOffReason = "NDV function with non-key column: " + col.getName();
returnColumnValidate = false;
break;
}
} else if (functionName.equalsIgnoreCase(FunctionSet.BITMAP_UNION_INT)) {
if ((!col.isKey())) {
turnOffReason = "BITMAP_UNION_INT function with non-key column: " + col.getName();
returnColumnValidate = false;
break;
}
} else if (functionName.equalsIgnoreCase(FunctionSet.BITMAP_UNION)
|| functionName.equalsIgnoreCase(FunctionSet.BITMAP_UNION_COUNT)
|| functionName.equalsIgnoreCase(FunctionSet.ORTHOGONAL_BITMAP_UNION_COUNT)) {
if (col.getAggregationType() != AggregateType.BITMAP_UNION) {
turnOffReason =
"Aggregate Operator not match: BITMAP_UNION <--> " + col.getAggregationType();
returnColumnValidate = false;
break;
}
} else if (functionName.equalsIgnoreCase(FunctionSet.QUANTILE_UNION)) {
if (col.getAggregationType() != AggregateType.QUANTILE_UNION) {
turnOffReason =
"Aggregate Operator not match: QUANTILE_UNION <---> " + col.getAggregationType();
returnColumnValidate = false;
break;
}
} else if (functionName.equalsIgnoreCase("multi_distinct_count")) {
// count(distinct k1), count(distinct k2) / count(distinct k1,k2) can turn on pre aggregation
if ((!col.isKey())) {
turnOffReason = "Multi count or sum distinct with non-key column: " + col.getName();
returnColumnValidate = false;
break;
}
} else {
turnOffReason = "Invalid Aggregate Operator: " + functionName;
returnColumnValidate = false;
break;
}
}
if (!returnColumnValidate) {
aggExprValidate = false;
break;
}
}
if (!aggExprValidate) {
break;
}
boolean groupExprValidate = true;
ArrayList<Expr> groupExprs = aggInfo.getGroupingExprs();
for (Expr groupExpr : groupExprs) {
List<SlotId> groupSlotIds = Lists.newArrayList();
groupExpr.getIds(null, groupSlotIds);
for (SlotDescriptor slot : selectStmt.getTableRefs().get(0).getDesc().getSlots()) {
if (!slot.getColumn().isKey()) {
if (groupSlotIds.contains(slot.getId())) {
turnOffReason = "groupExpr contains StorageEngine's Value";
groupExprValidate = false;
break;
}
}
}
if (!groupExprValidate) {
break;
}
}
if (!groupExprValidate) {
break;
}
OlapScanNode olapNode = (OlapScanNode) root;
if (!olapNode.getCanTurnOnPreAggr()) {
turnOffReason = "this olap scan node[" + olapNode.debugString()
+ "] has already been turned off pre-aggregation.";
break;
}
olapNode.setIsPreAggregation(true, null);
} while (false);
if ((root instanceof OlapScanNode) && turnOffReason != null) {
((OlapScanNode) root).setIsPreAggregation(false, turnOffReason);
}
}