in fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/mv/SelectMaterializedIndexWithAggregate.java [113:580]
public List<Rule> buildRules() {
return ImmutableList.of(
// only agg above scan
// Aggregate(Scan)
logicalAggregate(logicalOlapScan().when(this::shouldSelectIndexWithAgg)).thenApplyNoThrow(ctx -> {
if (ctx.connectContext.getSessionVariable().isEnableSyncMvCostBasedRewrite()) {
return ctx.root;
}
LogicalAggregate<LogicalOlapScan> agg = ctx.root;
LogicalOlapScan scan = agg.child();
SelectResult result = select(
scan,
agg.getInputSlots(),
ImmutableSet.of(),
extractAggFunctionAndReplaceSlot(agg, Optional.empty()),
agg.getGroupByExpressions(),
new HashSet<>(agg.getExpressions()));
LogicalOlapScan mvPlan = createLogicalOlapScan(scan, result);
SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan);
return new LogicalProject<>(
generateProjectsAlias(agg.getOutputs(), slotContext),
new ReplaceExpressions(slotContext).replace(
new LogicalAggregate<>(
agg.getGroupByExpressions(),
replaceAggOutput(
agg, Optional.empty(), Optional.empty(), result.exprRewriteMap),
agg.isNormalized(),
agg.getSourceRepeat(),
mvPlan
), mvPlan));
}).toRule(RuleType.MATERIALIZED_INDEX_AGG_SCAN),
// filter could push down scan.
// Aggregate(Filter(Scan))
logicalAggregate(logicalFilter(logicalOlapScan().when(this::shouldSelectIndexWithAgg)))
.thenApplyNoThrow(ctx -> {
if (ctx.connectContext.getSessionVariable().isEnableSyncMvCostBasedRewrite()) {
return ctx.root;
}
LogicalAggregate<LogicalFilter<LogicalOlapScan>> agg = ctx.root;
LogicalFilter<LogicalOlapScan> filter = agg.child();
LogicalOlapScan scan = filter.child();
ImmutableSet<Slot> requiredSlots = ImmutableSet.<Slot>builder()
.addAll(agg.getInputSlots())
.addAll(filter.getInputSlots())
.build();
ImmutableSet<Expression> requiredExpr = ImmutableSet.<Expression>builder()
.addAll(agg.getExpressions())
.addAll(filter.getExpressions())
.build();
SelectResult result = select(
scan,
requiredSlots,
filter.getConjuncts(),
extractAggFunctionAndReplaceSlot(agg, Optional.empty()),
agg.getGroupByExpressions(),
requiredExpr
);
LogicalOlapScan mvPlan = createLogicalOlapScan(scan, result);
SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan, requiredExpr.stream()
.map(e -> result.exprRewriteMap.replaceAgg(e)).collect(Collectors.toSet()),
filter.getConjuncts());
return new LogicalProject<>(
generateProjectsAlias(agg.getOutputs(), slotContext),
new ReplaceExpressions(slotContext).replace(
new LogicalAggregate<>(
agg.getGroupByExpressions(),
replaceAggOutput(agg, Optional.empty(), Optional.empty(),
result.exprRewriteMap),
agg.isNormalized(),
agg.getSourceRepeat(),
// Note that no need to replace slots in the filter,
// because the slots to
// replace are value columns, which shouldn't appear in filters.
filter.withChildren(mvPlan)
), mvPlan));
}).toRule(RuleType.MATERIALIZED_INDEX_AGG_FILTER_SCAN),
// column pruning or other projections such as alias, etc.
// Aggregate(Project(Scan))
logicalAggregate(logicalProject(logicalOlapScan().when(this::shouldSelectIndexWithAgg)))
.thenApplyNoThrow(ctx -> {
if (ctx.connectContext.getSessionVariable().isEnableSyncMvCostBasedRewrite()) {
return ctx.root;
}
LogicalAggregate<LogicalProject<LogicalOlapScan>> agg = ctx.root;
LogicalProject<LogicalOlapScan> project = agg.child();
LogicalOlapScan scan = project.child();
SelectResult result = select(
scan,
project.getInputSlots(),
ImmutableSet.of(),
extractAggFunctionAndReplaceSlot(agg,
Optional.of(project)),
ExpressionUtils.replace(agg.getGroupByExpressions(),
project.getAliasToProducer()),
collectRequireExprWithAggAndProject(agg.getExpressions(), Optional.of(project))
);
LogicalOlapScan mvPlan = createLogicalOlapScan(scan, result);
SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan);
List<NamedExpression> newProjectList = replaceOutput(project.getProjects(),
result.exprRewriteMap.projectExprMap);
LogicalProject<LogicalOlapScan> newProject = new LogicalProject<>(
generateNewOutputsWithMvOutputs(mvPlan, newProjectList),
scan.withMaterializedIndexSelected(result.indexId));
return new LogicalProject<>(generateProjectsAlias(agg.getOutputs(), slotContext),
new ReplaceExpressions(slotContext)
.replace(
new LogicalAggregate<>(agg.getGroupByExpressions(),
replaceAggOutput(agg, Optional.of(project),
Optional.of(newProject), result.exprRewriteMap),
agg.isNormalized(), agg.getSourceRepeat(), newProject),
mvPlan));
}).toRule(RuleType.MATERIALIZED_INDEX_AGG_PROJECT_SCAN),
// filter could push down and project.
// Aggregate(Project(Filter(Scan)))
logicalAggregate(logicalProject(logicalFilter(logicalOlapScan()
.when(this::shouldSelectIndexWithAgg)))).thenApplyNoThrow(ctx -> {
if (ctx.connectContext.getSessionVariable().isEnableSyncMvCostBasedRewrite()) {
return ctx.root;
}
LogicalAggregate<LogicalProject<LogicalFilter<LogicalOlapScan>>> agg = ctx.root;
LogicalProject<LogicalFilter<LogicalOlapScan>> project = agg.child();
LogicalFilter<LogicalOlapScan> filter = project.child();
LogicalOlapScan scan = filter.child();
Set<Slot> requiredSlots = Stream.concat(
project.getInputSlots().stream(), filter.getInputSlots().stream())
.collect(Collectors.toSet());
ImmutableSet<Expression> requiredExpr = ImmutableSet.<Expression>builder()
.addAll(collectRequireExprWithAggAndProject(
agg.getExpressions(), Optional.of(project)))
.addAll(filter.getExpressions())
.build();
SelectResult result = select(
scan,
requiredSlots,
filter.getConjuncts(),
extractAggFunctionAndReplaceSlot(agg, Optional.of(project)),
ExpressionUtils.replace(agg.getGroupByExpressions(),
project.getAliasToProducer()),
requiredExpr
);
LogicalOlapScan mvPlan = createLogicalOlapScan(scan, result);
SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan, requiredExpr.stream()
.map(e -> result.exprRewriteMap.replaceAgg(e)).collect(Collectors.toSet()),
filter.getConjuncts());
if (result.indexId == scan.getTable().getBaseIndexId()) {
LogicalOlapScan mvPlanWithoutAgg = SelectMaterializedIndexWithoutAggregate.select(scan,
project::getInputSlots, filter::getConjuncts,
Suppliers.memoize(() -> Utils.concatToSet(
filter.getExpressions(), project.getExpressions()
))
);
SlotContext slotContextWithoutAgg = generateBaseScanExprToMvExpr(mvPlanWithoutAgg);
return agg.withChildren(new LogicalProject(
generateProjectsAlias(project.getOutput(), slotContextWithoutAgg),
new ReplaceExpressions(slotContextWithoutAgg).replace(
project.withChildren(filter.withChildren(mvPlanWithoutAgg)),
mvPlanWithoutAgg)));
}
List<NamedExpression> newProjectList = replaceOutput(project.getProjects(),
result.exprRewriteMap.projectExprMap);
LogicalProject<Plan> newProject = new LogicalProject<>(
generateNewOutputsWithMvOutputs(mvPlan, newProjectList),
filter.withChildren(mvPlan));
return new LogicalProject<>(
generateProjectsAlias(agg.getOutputs(), slotContext),
new ReplaceExpressions(slotContext).replace(
new LogicalAggregate<>(
agg.getGroupByExpressions(),
replaceAggOutput(agg, Optional.of(project), Optional.of(newProject),
result.exprRewriteMap),
agg.isNormalized(),
agg.getSourceRepeat(),
newProject
), mvPlan));
}).toRule(RuleType.MATERIALIZED_INDEX_AGG_PROJECT_FILTER_SCAN),
// filter can't push down
// Aggregate(Filter(Project(Scan)))
logicalAggregate(logicalFilter(logicalProject(logicalOlapScan()
.when(this::shouldSelectIndexWithAgg)))).thenApplyNoThrow(ctx -> {
if (ctx.connectContext.getSessionVariable().isEnableSyncMvCostBasedRewrite()) {
return ctx.root;
}
LogicalAggregate<LogicalFilter<LogicalProject<LogicalOlapScan>>> agg = ctx.root;
LogicalFilter<LogicalProject<LogicalOlapScan>> filter = agg.child();
LogicalProject<LogicalOlapScan> project = filter.child();
LogicalOlapScan scan = project.child();
ImmutableSet<Expression> requiredExpr = ImmutableSet.<Expression>builder()
.addAll(collectRequireExprWithAggAndProject(
agg.getExpressions(), Optional.of(project)))
.addAll(collectRequireExprWithAggAndProject(
filter.getExpressions(), Optional.of(project)))
.build();
SelectResult result = select(
scan,
project.getInputSlots(),
filter.getConjuncts(),
extractAggFunctionAndReplaceSlot(agg, Optional.of(project)),
ExpressionUtils.replace(agg.getGroupByExpressions(),
project.getAliasToProducer()),
requiredExpr
);
LogicalOlapScan mvPlan = createLogicalOlapScan(scan, result);
SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan, requiredExpr.stream()
.map(e -> result.exprRewriteMap.replaceAgg(e)).collect(Collectors.toSet()),
filter.getConjuncts());
List<NamedExpression> newProjectList = replaceOutput(project.getProjects(),
result.exprRewriteMap.projectExprMap);
LogicalProject<Plan> newProject = new LogicalProject<>(
generateNewOutputsWithMvOutputs(mvPlan, newProjectList), mvPlan);
return new LogicalProject<>(
generateProjectsAlias(agg.getOutputs(), slotContext),
new ReplaceExpressions(slotContext).replace(
new LogicalAggregate<>(
agg.getGroupByExpressions(),
replaceAggOutput(agg, Optional.of(project), Optional.of(newProject),
result.exprRewriteMap),
agg.isNormalized(),
agg.getSourceRepeat(),
filter.withChildren(newProject)
), mvPlan));
}).toRule(RuleType.MATERIALIZED_INDEX_AGG_FILTER_PROJECT_SCAN),
// only agg above scan
// Aggregate(Repeat(Scan))
logicalAggregate(logicalRepeat(logicalOlapScan().when(this::shouldSelectIndexWithAgg)))
.thenApplyNoThrow(ctx -> {
if (ctx.connectContext.getSessionVariable().isEnableSyncMvCostBasedRewrite()) {
return ctx.root;
}
LogicalAggregate<LogicalRepeat<LogicalOlapScan>> agg = ctx.root;
LogicalRepeat<LogicalOlapScan> repeat = agg.child();
LogicalOlapScan scan = repeat.child();
SelectResult result = select(scan, agg.getInputSlots(), ImmutableSet.of(),
extractAggFunctionAndReplaceSlot(agg, Optional.empty()),
nonVirtualGroupByExprs(agg), new HashSet<>(agg.getExpressions()));
LogicalOlapScan mvPlan = createLogicalOlapScan(scan, result);
SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan);
return new LogicalProject<>(generateProjectsAlias(agg.getOutputs(), slotContext),
new ReplaceExpressions(slotContext)
.replace(
new LogicalAggregate<>(agg.getGroupByExpressions(),
replaceAggOutput(agg, Optional.empty(), Optional.empty(),
result.exprRewriteMap),
agg.isNormalized(), agg.getSourceRepeat(),
repeat.withAggOutputAndChild(
replaceOutput(repeat.getOutputs(),
result.exprRewriteMap.projectExprMap),
mvPlan)),
mvPlan));
}).toRule(RuleType.MATERIALIZED_INDEX_AGG_REPEAT_SCAN),
// filter could push down scan.
// Aggregate(Repeat(Filter(Scan)))
logicalAggregate(logicalRepeat(logicalFilter(logicalOlapScan().when(this::shouldSelectIndexWithAgg))))
.thenApplyNoThrow(ctx -> {
if (ctx.connectContext.getSessionVariable().isEnableSyncMvCostBasedRewrite()) {
return ctx.root;
}
LogicalAggregate<LogicalRepeat<LogicalFilter<LogicalOlapScan>>> agg = ctx.root;
LogicalRepeat<LogicalFilter<LogicalOlapScan>> repeat = agg.child();
LogicalFilter<LogicalOlapScan> filter = repeat.child();
LogicalOlapScan scan = filter.child();
ImmutableSet<Slot> requiredSlots = ImmutableSet.<Slot>builder()
.addAll(agg.getInputSlots())
.addAll(filter.getInputSlots())
.build();
ImmutableSet<Expression> requiredExpr = ImmutableSet.<Expression>builder()
.addAll(agg.getExpressions())
.addAll(filter.getExpressions())
.build();
SelectResult result = select(
scan,
requiredSlots,
filter.getConjuncts(),
extractAggFunctionAndReplaceSlot(agg, Optional.empty()),
nonVirtualGroupByExprs(agg),
requiredExpr
);
LogicalOlapScan mvPlan = createLogicalOlapScan(scan, result);
SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan, requiredExpr.stream()
.map(e -> result.exprRewriteMap.replaceAgg(e)).collect(Collectors.toSet()),
filter.getConjuncts());
return new LogicalProject<>(generateProjectsAlias(agg.getOutputs(), slotContext),
new ReplaceExpressions(slotContext).replace(new LogicalAggregate<>(
agg.getGroupByExpressions(),
replaceAggOutput(agg, Optional.empty(), Optional.empty(),
result.exprRewriteMap),
agg.isNormalized(), agg.getSourceRepeat(),
// Not that no need to replace slots in the filter,
// because the slots to replace
// are value columns, which shouldn't appear in filters.
repeat.withAggOutputAndChild(
replaceOutput(repeat.getOutputs(),
result.exprRewriteMap.projectExprMap),
filter.withChildren(mvPlan))),
mvPlan));
}).toRule(RuleType.MATERIALIZED_INDEX_AGG_REPEAT_FILTER_SCAN),
// column pruning or other projections such as alias, etc.
// Aggregate(Repeat(Project(Scan)))
logicalAggregate(logicalRepeat(logicalProject(logicalOlapScan().when(this::shouldSelectIndexWithAgg))))
.thenApplyNoThrow(ctx -> {
if (ctx.connectContext.getSessionVariable().isEnableSyncMvCostBasedRewrite()) {
return ctx.root;
}
LogicalAggregate<LogicalRepeat<LogicalProject<LogicalOlapScan>>> agg = ctx.root;
LogicalRepeat<LogicalProject<LogicalOlapScan>> repeat = agg.child();
LogicalProject<LogicalOlapScan> project = repeat.child();
LogicalOlapScan scan = project.child();
SelectResult result = select(
scan,
project.getInputSlots(),
ImmutableSet.of(),
extractAggFunctionAndReplaceSlot(agg,
Optional.of(project)),
ExpressionUtils.replace(nonVirtualGroupByExprs(agg),
project.getAliasToProducer()),
collectRequireExprWithAggAndProject(agg.getExpressions(), Optional.of(project))
);
LogicalOlapScan mvPlan = createLogicalOlapScan(scan, result);
SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan);
List<NamedExpression> newProjectList = replaceOutput(project.getProjects(),
result.exprRewriteMap.projectExprMap);
LogicalProject<LogicalOlapScan> newProject = new LogicalProject<>(
generateNewOutputsWithMvOutputs(mvPlan, newProjectList), mvPlan);
return new LogicalProject<>(generateProjectsAlias(agg.getOutputs(), slotContext),
new ReplaceExpressions(slotContext).replace(
new LogicalAggregate<>(agg.getGroupByExpressions(),
replaceAggOutput(agg, Optional.of(project), Optional.of(newProject),
result.exprRewriteMap),
agg.isNormalized(), agg.getSourceRepeat(),
repeat.withAggOutputAndChild(replaceOutput(repeat.getOutputs(),
result.exprRewriteMap.projectExprMap), newProject)),
mvPlan));
}).toRule(RuleType.MATERIALIZED_INDEX_AGG_REPEAT_PROJECT_SCAN),
// filter could push down and project.
// Aggregate(Repeat(Project(Filter(Scan))))
logicalAggregate(logicalRepeat(logicalProject(logicalFilter(logicalOlapScan()
.when(this::shouldSelectIndexWithAgg))))).thenApplyNoThrow(ctx -> {
if (ctx.connectContext.getSessionVariable().isEnableSyncMvCostBasedRewrite()) {
return ctx.root;
}
LogicalAggregate<LogicalRepeat<LogicalProject
<LogicalFilter<LogicalOlapScan>>>> agg = ctx.root;
LogicalRepeat<LogicalProject<LogicalFilter<LogicalOlapScan>>> repeat = agg.child();
LogicalProject<LogicalFilter<LogicalOlapScan>> project = repeat.child();
LogicalFilter<LogicalOlapScan> filter = project.child();
LogicalOlapScan scan = filter.child();
Set<Slot> requiredSlots = Stream.concat(
project.getInputSlots().stream(), filter.getInputSlots().stream())
.collect(Collectors.toSet());
ImmutableSet<Expression> requiredExpr = ImmutableSet.<Expression>builder()
.addAll(collectRequireExprWithAggAndProject(
agg.getExpressions(), Optional.of(project)))
.addAll(filter.getExpressions())
.build();
SelectResult result = select(
scan,
requiredSlots,
filter.getConjuncts(),
extractAggFunctionAndReplaceSlot(agg, Optional.of(project)),
ExpressionUtils.replace(nonVirtualGroupByExprs(agg),
project.getAliasToProducer()),
requiredExpr
);
LogicalOlapScan mvPlan = createLogicalOlapScan(scan, result);
SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan, requiredExpr.stream()
.map(e -> result.exprRewriteMap.replaceAgg(e)).collect(Collectors.toSet()),
filter.getConjuncts());
List<NamedExpression> newProjectList = replaceOutput(project.getProjects(),
result.exprRewriteMap.projectExprMap);
LogicalProject<Plan> newProject = new LogicalProject<>(
generateNewOutputsWithMvOutputs(mvPlan, newProjectList),
filter.withChildren(mvPlan));
return new LogicalProject<>(generateProjectsAlias(agg.getOutputs(), slotContext),
new ReplaceExpressions(slotContext).replace(
new LogicalAggregate<>(agg.getGroupByExpressions(),
replaceAggOutput(agg, Optional.of(project), Optional.of(newProject),
result.exprRewriteMap),
agg.isNormalized(), agg.getSourceRepeat(),
repeat.withAggOutputAndChild(replaceOutput(repeat.getOutputs(),
result.exprRewriteMap.projectExprMap), newProject)),
mvPlan));
}).toRule(RuleType.MATERIALIZED_INDEX_AGG_REPEAT_PROJECT_FILTER_SCAN),
// filter can't push down
// Aggregate(Repeat(Filter(Project(Scan))))
logicalAggregate(logicalRepeat(logicalFilter(logicalProject(logicalOlapScan()
.when(this::shouldSelectIndexWithAgg))))).thenApplyNoThrow(ctx -> {
if (ctx.connectContext.getSessionVariable().isEnableSyncMvCostBasedRewrite()) {
return ctx.root;
}
LogicalAggregate<LogicalRepeat<LogicalFilter
<LogicalProject<LogicalOlapScan>>>> agg = ctx.root;
LogicalRepeat<LogicalFilter<LogicalProject<LogicalOlapScan>>> repeat = agg.child();
LogicalFilter<LogicalProject<LogicalOlapScan>> filter = repeat.child();
LogicalProject<LogicalOlapScan> project = filter.child();
LogicalOlapScan scan = project.child();
ImmutableSet<Expression> requiredExpr = ImmutableSet.<Expression>builder()
.addAll(collectRequireExprWithAggAndProject(
agg.getExpressions(), Optional.of(project)))
.addAll(collectRequireExprWithAggAndProject(
filter.getExpressions(), Optional.of(project)))
.build();
SelectResult result = select(
scan,
project.getInputSlots(),
filter.getConjuncts(),
extractAggFunctionAndReplaceSlot(agg, Optional.of(project)),
ExpressionUtils.replace(nonVirtualGroupByExprs(agg),
project.getAliasToProducer()),
requiredExpr
);
LogicalOlapScan mvPlan = createLogicalOlapScan(scan, result);
SlotContext slotContext = generateBaseScanExprToMvExpr(mvPlan, requiredExpr.stream()
.map(e -> result.exprRewriteMap.replaceAgg(e)).collect(Collectors.toSet()),
filter.getConjuncts());
List<NamedExpression> newProjectList = replaceOutput(project.getProjects(),
result.exprRewriteMap.projectExprMap);
LogicalProject<Plan> newProject = new LogicalProject<>(
generateNewOutputsWithMvOutputs(mvPlan, newProjectList),
scan.withMaterializedIndexSelected(result.indexId));
return new LogicalProject<>(generateProjectsAlias(agg.getOutputs(), slotContext),
new ReplaceExpressions(slotContext).replace(new LogicalAggregate<>(
agg.getGroupByExpressions(), replaceAggOutput(agg, Optional.of(project),
Optional.of(newProject), result.exprRewriteMap),
agg.isNormalized(), agg.getSourceRepeat(),
repeat.withAggOutputAndChild(
replaceOutput(repeat.getOutputs(),
result.exprRewriteMap.projectExprMap),
filter.withChildren(newProject))),
mvPlan));
}).toRule(RuleType.MATERIALIZED_INDEX_AGG_REPEAT_FILTER_PROJECT_SCAN)
);
}