in pinot-core/src/main/java/org/apache/pinot/core/plan/FilterPlanNode.java [198:331]
private BaseFilterOperator constructPhysicalOperator(FilterContext filter, int numDocs) {
switch (filter.getType()) {
case AND:
List<FilterContext> childFilters = filter.getChildren();
List<BaseFilterOperator> childFilterOperators = new ArrayList<>(childFilters.size());
for (FilterContext childFilter : childFilters) {
BaseFilterOperator childFilterOperator = constructPhysicalOperator(childFilter, numDocs);
if (childFilterOperator.isResultEmpty()) {
// Return empty filter operator if any of the child filter operator's result is empty
return EmptyFilterOperator.getInstance();
} else if (!childFilterOperator.isResultMatchingAll()) {
// Remove child filter operators that match all records
childFilterOperators.add(childFilterOperator);
}
}
return FilterOperatorUtils.getAndFilterOperator(_queryContext, childFilterOperators, numDocs);
case OR:
childFilters = filter.getChildren();
childFilterOperators = new ArrayList<>(childFilters.size());
for (FilterContext childFilter : childFilters) {
BaseFilterOperator childFilterOperator = constructPhysicalOperator(childFilter, numDocs);
if (childFilterOperator.isResultMatchingAll()) {
// Return match all filter operator if any of the child filter operator matches all records
return new MatchAllFilterOperator(numDocs);
} else if (!childFilterOperator.isResultEmpty()) {
// Remove child filter operators whose result is empty
childFilterOperators.add(childFilterOperator);
}
}
return FilterOperatorUtils.getOrFilterOperator(_queryContext, childFilterOperators, numDocs);
case NOT:
childFilters = filter.getChildren();
assert childFilters.size() == 1;
BaseFilterOperator childFilterOperator = constructPhysicalOperator(childFilters.get(0), numDocs);
return FilterOperatorUtils.getNotFilterOperator(_queryContext, childFilterOperator, numDocs);
case PREDICATE:
Predicate predicate = filter.getPredicate();
ExpressionContext lhs = predicate.getLhs();
if (lhs.getType() == ExpressionContext.Type.FUNCTION) {
if (canApplyH3IndexForDistanceCheck(predicate, lhs.getFunction())) {
return new H3IndexFilterOperator(_indexSegment, _queryContext, predicate, numDocs);
} else if (canApplyH3IndexForInclusionCheck(predicate, lhs.getFunction())) {
return new H3InclusionIndexFilterOperator(_indexSegment, _queryContext, predicate, numDocs);
} else {
// TODO: ExpressionFilterOperator does not support predicate types without PredicateEvaluator (TEXT_MATCH)
return new ExpressionFilterOperator(_indexSegment, _queryContext, predicate, numDocs);
}
} else {
String column = lhs.getIdentifier();
DataSource dataSource = _indexSegment.getDataSource(column);
PredicateEvaluator predicateEvaluator;
switch (predicate.getType()) {
case TEXT_CONTAINS:
TextIndexReader textIndexReader = dataSource.getTextIndex();
if (!(textIndexReader instanceof NativeTextIndexReader)
&& !(textIndexReader instanceof NativeMutableTextIndex)) {
throw new UnsupportedOperationException("TEXT_CONTAINS is supported only on native text index");
}
return new TextContainsFilterOperator(textIndexReader, (TextContainsPredicate) predicate, numDocs);
case TEXT_MATCH:
textIndexReader = dataSource.getTextIndex();
Preconditions.checkState(textIndexReader != null,
"Cannot apply TEXT_MATCH on column: %s without text index", column);
// We could check for real time and segment Lucene reader, but easier to check the other way round
if (textIndexReader instanceof NativeTextIndexReader
|| textIndexReader instanceof NativeMutableTextIndex) {
throw new UnsupportedOperationException("TEXT_MATCH is not supported on native text index");
}
return new TextMatchFilterOperator(textIndexReader, (TextMatchPredicate) predicate, numDocs);
case REGEXP_LIKE:
// FST Index is available only for rolled out segments. So, we use different evaluator for rolled out and
// consuming segments.
//
// Rolled out segments (immutable): FST Index reader is available use FSTBasedEvaluator
// else use regular flow of getting predicate evaluator.
//
// Consuming segments: When FST is enabled, use AutomatonBasedEvaluator so that regexp matching logic is
// similar to that of FSTBasedEvaluator, else use regular flow of getting predicate evaluator.
if (dataSource.getFSTIndex() != null) {
predicateEvaluator =
FSTBasedRegexpPredicateEvaluatorFactory.newFSTBasedEvaluator((RegexpLikePredicate) predicate,
dataSource.getFSTIndex(), dataSource.getDictionary());
} else {
predicateEvaluator =
PredicateEvaluatorProvider.getPredicateEvaluator(predicate, dataSource.getDictionary(),
dataSource.getDataSourceMetadata().getDataType());
}
_predicateEvaluators.add(Pair.of(predicate, predicateEvaluator));
return FilterOperatorUtils.getLeafFilterOperator(_queryContext, predicateEvaluator, dataSource, numDocs);
case JSON_MATCH:
JsonIndexReader jsonIndex = dataSource.getJsonIndex();
if (jsonIndex == null) { //TODO: rework
Optional<IndexType<?, ?, ?>> compositeIndex =
IndexService.getInstance().getOptional("composite_json_index");
if (compositeIndex.isPresent()) {
jsonIndex =
(JsonIndexReader) dataSource.getIndex(compositeIndex.get());
}
}
Preconditions.checkState(jsonIndex != null, "Cannot apply JSON_MATCH on column: %s without json index",
column);
return new JsonMatchFilterOperator(jsonIndex, (JsonMatchPredicate) predicate, numDocs);
case VECTOR_SIMILARITY:
VectorIndexReader vectorIndex = dataSource.getVectorIndex();
Preconditions.checkState(vectorIndex != null,
"Cannot apply VECTOR_SIMILARITY on column: %s without vector index", column);
return new VectorSimilarityFilterOperator(vectorIndex, (VectorSimilarityPredicate) predicate, numDocs);
case IS_NULL:
NullValueVectorReader nullValueVector = dataSource.getNullValueVector();
if (nullValueVector != null) {
return new BitmapBasedFilterOperator(nullValueVector.getNullBitmap(), false, numDocs);
} else {
return EmptyFilterOperator.getInstance();
}
case IS_NOT_NULL:
nullValueVector = dataSource.getNullValueVector();
if (nullValueVector != null) {
return new BitmapBasedFilterOperator(nullValueVector.getNullBitmap(), true, numDocs);
} else {
return new MatchAllFilterOperator(numDocs);
}
default:
predicateEvaluator =
PredicateEvaluatorProvider.getPredicateEvaluator(predicate, dataSource, _queryContext);
_predicateEvaluators.add(Pair.of(predicate, predicateEvaluator));
return FilterOperatorUtils.getLeafFilterOperator(_queryContext, predicateEvaluator, dataSource, numDocs);
}
}
case CONSTANT:
return filter.isConstantTrue() ? new MatchAllFilterOperator(numDocs) : EmptyFilterOperator.getInstance();
default:
throw new IllegalStateException();
}
}