in exec/java-exec/src/main/java/org/apache/drill/exec/planner/index/rules/DbScanToIndexScanPrule.java [304:531]
private void processWithIndexSelection(
IndexLogicalPlanCallContext indexContext,
PlannerSettings settings,
RexNode condition,
IndexCollection collection,
RexBuilder builder) {
double totalRows = 0;
double filterRows = totalRows;
DrillScanRel scan = indexContext.scan;
if (! (indexContext.scan.getGroupScan() instanceof DbGroupScan) ) {
return;
}
IndexConditionInfo.Builder infoBuilder = IndexConditionInfo.newBuilder(condition, collection, builder, indexContext.scan);
IndexConditionInfo cInfo = infoBuilder.getCollectiveInfo(indexContext);
boolean isValidIndexHint = infoBuilder.isValidIndexHint(indexContext);
if (!cInfo.hasIndexCol) {
logger.info("index_plan_info: No index columns are projected from the scan..continue.");
return;
}
if (cInfo.indexCondition == null) {
logger.info("index_plan_info: No conditions were found eligible for applying index lookup.");
return;
}
if (!indexContext.indexHint.equals("") && !isValidIndexHint) {
logger.warn("index_plan_info: Index Hint {} is not useful as index with that name is not available", indexContext.indexHint);
}
RexNode indexCondition = cInfo.indexCondition;
RexNode remainderCondition = cInfo.remainderCondition;
if (remainderCondition.isAlwaysTrue()) {
remainderCondition = null;
}
logger.debug("index_plan_info: condition split into indexcondition: {} and remaindercondition: {}", indexCondition, remainderCondition);
IndexableExprMarker indexableExprMarker = new IndexableExprMarker(indexContext.scan);
indexCondition.accept(indexableExprMarker);
indexContext.origMarker = indexableExprMarker;
if (scan.getGroupScan() instanceof DbGroupScan) {
// Initialize statistics
DbGroupScan dbScan = ((DbGroupScan) scan.getGroupScan());
if (settings.isStatisticsEnabled()) {
dbScan.getStatistics().initialize(condition, scan, indexContext);
}
totalRows = dbScan.getRowCount(null, scan);
filterRows = dbScan.getRowCount(condition, scan);
double sel = filterRows/totalRows;
if (totalRows != Statistics.ROWCOUNT_UNKNOWN &&
filterRows != Statistics.ROWCOUNT_UNKNOWN &&
!settings.isDisableFullTableScan() && !isValidIndexHint &&
sel > Math.max(settings.getIndexCoveringSelThreshold(),
settings.getIndexNonCoveringSelThreshold() )) {
// If full table scan is not disabled, generate full table scan only plans if selectivity
// is greater than covering and non-covering selectivity thresholds
logger.info("index_plan_info: Skip index planning because filter selectivity: {} is greater than thresholds {}, {}",
sel, settings.getIndexCoveringSelThreshold(), settings.getIndexNonCoveringSelThreshold());
return;
}
}
if (totalRows == Statistics.ROWCOUNT_UNKNOWN ||
totalRows == 0 || filterRows == Statistics.ROWCOUNT_UNKNOWN ) {
logger.warn("index_plan_info: Total row count is UNKNOWN or 0, or filterRows UNKNOWN; skip index planning");
return;
}
List<IndexGroup> coveringIndexes = Lists.newArrayList();
List<IndexGroup> nonCoveringIndexes = Lists.newArrayList();
List<IndexGroup> intersectIndexes = Lists.newArrayList();
//update sort expressions in context, it is needed for computing collation, so do it before IndexSelector
IndexPlanUtils.updateSortExpression(indexContext, indexContext.sort != null ?
indexContext.sort.collation.getFieldCollations() : null);
IndexSelector selector = new IndexSelector(indexCondition,
remainderCondition,
indexContext,
collection,
builder,
totalRows);
for (IndexDescriptor indexDesc : collection) {
logger.info("index_plan_info indexDescriptor: {}", indexDesc.toString());
// check if any of the indexed fields of the index are present in the filter condition
if (IndexPlanUtils.conditionIndexed(indexableExprMarker, indexDesc) != IndexPlanUtils.ConditionIndexed.NONE) {
if (isValidIndexHint && !indexContext.indexHint.equals(indexDesc.getIndexName())) {
logger.info("index_plan_info: Index {} is being discarded due to index Hint", indexDesc.getIndexName());
continue;
}
FunctionalIndexInfo functionInfo = indexDesc.getFunctionalInfo();
selector.addIndex(indexDesc, IndexPlanUtils.isCoveringIndex(indexContext, functionInfo),
indexContext.lowerProject != null ? indexContext.lowerProject.getRowType().getFieldCount() :
scan.getRowType().getFieldCount());
}
}
// get the candidate indexes based on selection
selector.getCandidateIndexes(infoBuilder, coveringIndexes, nonCoveringIndexes, intersectIndexes);
if (logger.isDebugEnabled()) {
StringBuilder strb = new StringBuilder();
if (coveringIndexes.size() > 0) {
strb.append("Covering indexes:");
for (IndexGroup index : coveringIndexes) {
strb.append(index.getIndexProps().get(0).getIndexDesc().getIndexName()).append(", ");
}
}
if(nonCoveringIndexes.size() > 0) {
strb.append("Non-covering indexes:");
for (IndexGroup index : nonCoveringIndexes) {
strb.append(index.getIndexProps().get(0).getIndexDesc().getIndexName()).append(", ");
}
}
logger.debug("index_plan_info: IndexSelector return: {}",strb.toString());
}
GroupScan primaryTableScan = indexContext.scan.getGroupScan();
// Only non-covering indexes can be intersected. Check if
// (a) there are no covering indexes. Intersect plans will almost always be more
// expensive than a covering index, so no need to generate one if there is covering.
// (b) there is more than 1 non-covering indexes that can be intersected
// TODO: this logic for intersect should eventually be migrated to the IndexSelector
if (coveringIndexes.size() == 0 && nonCoveringIndexes.size() > 1) {
List<IndexDescriptor> indexList = Lists.newArrayList();
for (IndexGroup index : nonCoveringIndexes) {
IndexDescriptor indexDesc = index.getIndexProps().get(0).getIndexDesc();
IndexGroupScan idxScan = indexDesc.getIndexGroupScan();
//Copy primary table statistics to index table
idxScan.setStatistics(((DbGroupScan) primaryTableScan).getStatistics());
indexList.add(index.getIndexProps().get(0).getIndexDesc());
}
Map<IndexDescriptor, IndexConditionInfo> indexInfoMap = infoBuilder.getIndexConditionMap(indexList);
//no usable index
if (indexInfoMap == null || indexInfoMap.size() == 0) {
logger.info("index_plan_info: skipping intersect plan generation as there is no usable index");
return;
}
//if there is only one index found, no need to do intersect, but just a regular non-covering plan
//some part of filter condition needs to apply on primary table.
if(indexInfoMap.size() > 1) {
logger.info("index_plan_info: intersect plan is generated");
if (logger.isDebugEnabled()) {
List<String> indices = new ArrayList<>(nonCoveringIndexes.size());
for (IndexGroup index : nonCoveringIndexes) {
indices.add(index.getIndexProps().get(0).getIndexDesc().getIndexName());
}
logger.debug("index_plan_info: intersect plan is generated on index list {}", indices);
}
boolean intersectPlanGenerated = false;
//multiple indexes, let us try to intersect results from multiple index tables
//TODO: make sure the smallest selectivity of these indexes times rowcount smaller than broadcast threshold
for (IndexGroup index : intersectIndexes) {
List<IndexDescriptor> candidateDesc = Lists.newArrayList();
for (IndexProperties candProp : index.getIndexProps()) {
candidateDesc.add(candProp.getIndexDesc());
}
Map<IndexDescriptor, IndexConditionInfo> intersectIdxInfoMap = infoBuilder.getIndexConditionMap(candidateDesc);
IndexIntersectPlanGenerator planGen = new IndexIntersectPlanGenerator(
indexContext, intersectIdxInfoMap, builder, settings);
try {
planGen.go();
intersectPlanGenerated = true;
} catch (Exception e) {
// If error while generating intersect plans, continue onto generating non-covering plans
logger.warn("index_plan_info: Exception while trying to generate intersect index plan", e);
}
}
// If intersect plans are forced do not generate further non-covering plans
if (intersectPlanGenerated && settings.isIndexIntersectPlanPreferred()) {
return;
}
}
}
try {
for (IndexGroup index : coveringIndexes) {
IndexProperties indexProps = index.getIndexProps().get(0);
IndexDescriptor indexDesc = indexProps.getIndexDesc();
IndexGroupScan idxScan = indexDesc.getIndexGroupScan();
FunctionalIndexInfo indexInfo = indexDesc.getFunctionalInfo();
indexCondition = indexProps.getLeadingColumnsFilter();
remainderCondition = indexProps.getTotalRemainderFilter();
//Copy primary table statistics to index table
idxScan.setStatistics(((DbGroupScan) scan.getGroupScan()).getStatistics());
logger.info("index_plan_info: Generating covering index plan for index: {}, query condition {}", indexDesc.getIndexName(), indexCondition.toString());
CoveringIndexPlanGenerator planGen = new CoveringIndexPlanGenerator(indexContext, indexInfo, idxScan,
indexCondition, remainderCondition, builder, settings);
planGen.go();
}
} catch (Exception e) {
logger.warn("Exception while trying to generate covering index plan", e);
}
// Create non-covering index plans.
//First, check if the primary table scan supports creating a restricted scan
if (primaryTableScan instanceof DbGroupScan &&
(((DbGroupScan) primaryTableScan).supportsRestrictedScan())) {
try {
for (IndexGroup index : nonCoveringIndexes) {
IndexProperties indexProps = index.getIndexProps().get(0);
IndexDescriptor indexDesc = indexProps.getIndexDesc();
IndexGroupScan idxScan = indexDesc.getIndexGroupScan();
indexCondition = indexProps.getLeadingColumnsFilter();
remainderCondition = indexProps.getTotalRemainderFilter();
//Copy primary table statistics to index table
idxScan.setStatistics(((DbGroupScan) primaryTableScan).getStatistics());
logger.info("index_plan_info: Generating non-covering index plan for index: {}, query condition {}", indexDesc.getIndexName(), indexCondition.toString());
NonCoveringIndexPlanGenerator planGen = new NonCoveringIndexPlanGenerator(indexContext, indexDesc,
idxScan, indexCondition, remainderCondition, builder, settings);
planGen.go();
}
} catch (Exception e) {
logger.warn("Exception while trying to generate non-covering index access plan", e);
}
}
}