in solr/core/src/java/org/apache/solr/handler/component/ExpandComponent.java [131:452]
public void process(ResponseBuilder rb) throws IOException {
if (!rb.doExpand) {
return;
}
SolrQueryRequest req = rb.req;
SolrParams params = req.getParams();
String field = params.get(ExpandParams.EXPAND_FIELD);
String hint = null;
if (field == null) {
List<Query> filters = rb.getFilters();
if (filters != null) {
int cost = Integer.MAX_VALUE;
for (Query q : filters) {
if (q instanceof CollapsingQParserPlugin.CollapsingPostFilter cp) {
// if there are multiple collapse pick the low cost one
// if cost are equal then first one is picked
if (cp.getCost() < cost) {
cost = cp.getCost();
field = cp.getField();
hint = cp.hint;
}
}
}
}
}
if (field == null) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "missing expand field");
}
String sortParam = params.get(ExpandParams.EXPAND_SORT);
String[] fqs = params.getParams(ExpandParams.EXPAND_FQ);
String qs = params.get(ExpandParams.EXPAND_Q);
int limit = params.getInt(ExpandParams.EXPAND_ROWS, 5);
Sort sort = null;
if (sortParam != null) {
sort = SortSpecParsing.parseSortSpec(sortParam, rb.req).getSort();
}
final Query query;
List<Query> newFilters = new ArrayList<>();
try {
if (qs == null) {
query = rb.getQuery();
} else {
QParser parser = QParser.getParser(qs, req);
query = parser.getQuery();
}
if (fqs == null) {
List<Query> filters = rb.getFilters();
if (filters != null) {
for (Query q : filters) {
if (!(q instanceof CollapsingQParserPlugin.CollapsingPostFilter)) {
newFilters.add(q);
}
}
}
} else {
for (String fq : fqs) {
if (StrUtils.isNotBlank(fq) && !fq.equals("*:*")) {
QParser fqp = QParser.getParser(fq, req);
newFilters.add(fqp.getQuery());
}
}
}
} catch (SyntaxError e) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
}
SolrIndexSearcher searcher = req.getSearcher();
LeafReader reader = searcher.getSlowAtomicReader();
SchemaField schemaField = searcher.getSchema().getField(field);
FieldType fieldType = schemaField.getType();
SortedDocValues values = null;
if (fieldType instanceof StrField) {
// Get The Top Level SortedDocValues
if (CollapsingQParserPlugin.HINT_TOP_FC.equals(hint)) {
LeafReader uninvertingReader =
CollapsingQParserPlugin.getTopFieldCacheReader(searcher, field);
values = uninvertingReader.getSortedDocValues(field);
} else {
values = DocValues.getSorted(reader, field);
}
} else if (fieldType.getNumberType() == null) {
// possible if directly expand.field is specified
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST,
"Expand not supported for fieldType:'" + fieldType.getTypeName() + "'");
}
FixedBitSet groupBits = null;
LongHashSet groupSet = null;
DocList docList = rb.getResults().docList;
IntHashSet collapsedSet = new IntHashSet(docList.size() * 2);
// Gather the groups for the current page of documents
DocIterator idit = docList.iterator();
int[] globalDocs = new int[docList.size()];
int docsIndex = -1;
while (idit.hasNext()) {
globalDocs[++docsIndex] = idit.nextDoc();
}
Arrays.sort(globalDocs);
Query groupQuery = null;
/*
* This code gathers the group information for the current page.
*/
List<LeafReaderContext> contexts = searcher.getTopReaderContext().leaves();
if (contexts.size() == 0) {
// When no context is available we can skip the expanding
return;
}
QueryLimits queryLimits = QueryLimits.getCurrentLimits();
if (queryLimits.maybeExitWithPartialResults("Expand process")) {
return;
}
boolean nullGroupOnCurrentPage = false;
int currentContext = 0;
int currentDocBase = contexts.get(currentContext).docBase;
int nextDocBase =
(currentContext + 1) < contexts.size()
? contexts.get(currentContext + 1).docBase
: Integer.MAX_VALUE;
IntObjectHashMap<BytesRef> ordBytes = null;
if (values != null) {
groupBits = new FixedBitSet(values.getValueCount());
OrdinalMap ordinalMap = null;
SortedDocValues[] sortedDocValues = null;
LongValues segmentOrdinalMap = null;
SortedDocValues currentValues = null;
if (values instanceof MultiDocValues.MultiSortedDocValues) {
ordinalMap = ((MultiDocValues.MultiSortedDocValues) values).mapping;
sortedDocValues = ((MultiDocValues.MultiSortedDocValues) values).values;
currentValues = sortedDocValues[currentContext];
segmentOrdinalMap = ordinalMap.getGlobalOrds(currentContext);
}
ordBytes = new IntObjectHashMap<>();
for (int i = 0; i < globalDocs.length; i++) {
int globalDoc = globalDocs[i];
while (globalDoc >= nextDocBase) {
currentContext++;
currentDocBase = contexts.get(currentContext).docBase;
nextDocBase =
(currentContext + 1) < contexts.size()
? contexts.get(currentContext + 1).docBase
: Integer.MAX_VALUE;
if (ordinalMap != null) {
currentValues = sortedDocValues[currentContext];
segmentOrdinalMap = ordinalMap.getGlobalOrds(currentContext);
}
}
collapsedSet.add(globalDoc);
int contextDoc = globalDoc - currentDocBase;
if (ordinalMap != null) {
if (contextDoc > currentValues.docID()) {
currentValues.advance(contextDoc);
}
if (contextDoc == currentValues.docID()) {
int contextOrd = currentValues.ordValue();
int ord = (int) segmentOrdinalMap.get(contextOrd);
if (!groupBits.getAndSet(ord)) {
BytesRef ref = currentValues.lookupOrd(contextOrd);
ordBytes.put(ord, BytesRef.deepCopyOf(ref));
}
} else {
nullGroupOnCurrentPage = true;
}
} else {
if (globalDoc > values.docID()) {
values.advance(globalDoc);
}
if (globalDoc == values.docID()) {
int ord = values.ordValue();
if (!groupBits.getAndSet(ord)) {
BytesRef ref = values.lookupOrd(ord);
ordBytes.put(ord, BytesRef.deepCopyOf(ref));
}
} else {
nullGroupOnCurrentPage = true;
}
}
}
int count = ordBytes.size();
if (count > 0 && count < 200) {
groupQuery = getGroupQuery(field, count, ordBytes);
}
} else {
groupSet = new LongHashSet(docList.size());
NumericDocValues collapseValues =
contexts.get(currentContext).reader().getNumericDocValues(field);
for (int i = 0; i < globalDocs.length; i++) {
int globalDoc = globalDocs[i];
while (globalDoc >= nextDocBase) {
currentContext++;
currentDocBase = contexts.get(currentContext).docBase;
nextDocBase =
currentContext + 1 < contexts.size()
? contexts.get(currentContext + 1).docBase
: Integer.MAX_VALUE;
collapseValues = contexts.get(currentContext).reader().getNumericDocValues(field);
}
collapsedSet.add(globalDoc);
int contextDoc = globalDoc - currentDocBase;
int valueDocID = collapseValues.docID();
if (valueDocID < contextDoc) {
valueDocID = collapseValues.advance(contextDoc);
}
if (valueDocID == contextDoc) {
final long value = collapseValues.longValue();
groupSet.add(value);
} else {
nullGroupOnCurrentPage = true;
}
}
int count = groupSet.size();
if (count > 0 && count < 200) {
if (fieldType.isPointField()) {
groupQuery = getPointGroupQuery(schemaField, count, groupSet);
} else {
groupQuery = getGroupQuery(field, fieldType, count, groupSet);
}
}
}
final boolean expandNullGroup =
params.getBool(ExpandParams.EXPAND_NULL, false)
&&
// Our GroupCollector can typically ignore nulls (and the user's nullGroup param) unless
// the current page had any - but if expand.q was specified, current page doesn't mater:
// We need look for nulls if the user asked us to because we don't know what the
// expand.q will match
(nullGroupOnCurrentPage || (null != query));
if (expandNullGroup && null != groupQuery) {
// we need to also consider docs w/o a field value
final BooleanQuery.Builder inner = new BooleanQuery.Builder();
inner.add(fieldType.getExistenceQuery(null, schemaField), BooleanClause.Occur.MUST_NOT);
inner.add(new MatchAllDocsQuery(), BooleanClause.Occur.MUST);
final BooleanQuery.Builder outer = new BooleanQuery.Builder();
outer.add(inner.build(), BooleanClause.Occur.SHOULD);
outer.add(groupQuery, BooleanClause.Occur.SHOULD);
groupQuery = outer.build();
}
Collector collector;
if (sort != null) sort = sort.rewrite(searcher);
GroupCollector groupExpandCollector = null;
if (values != null) {
// Get The Top Level SortedDocValues again so we can re-iterate:
if (CollapsingQParserPlugin.HINT_TOP_FC.equals(hint)) {
LeafReader uninvertingReader =
CollapsingQParserPlugin.getTopFieldCacheReader(searcher, field);
values = uninvertingReader.getSortedDocValues(field);
} else {
values = DocValues.getSorted(reader, field);
}
groupExpandCollector =
new GroupExpandCollector(
limit,
sort,
query,
expandNullGroup,
fieldType,
ordBytes,
values,
groupBits,
collapsedSet);
} else {
groupExpandCollector =
new NumericGroupExpandCollector(
limit,
sort,
query,
expandNullGroup,
fieldType,
ordBytes,
field,
groupSet,
collapsedSet);
}
if (groupQuery != null) {
// Limits the results to documents that are in the same group as the documents in the page.
newFilters.add(groupQuery);
}
SolrIndexSearcher.ProcessedFilter pfilter = searcher.getProcessedFilter(newFilters);
if (pfilter.postFilter != null) {
pfilter.postFilter.setLastDelegate(groupExpandCollector);
collector = pfilter.postFilter;
} else {
collector = groupExpandCollector;
}
searcher.search(QueryUtils.combineQueryAndFilter(query, pfilter.filter), collector);
if (queryLimits.maybeExitWithPartialResults("Expand expand")) {
return;
}
rb.rsp.add("expanded", groupExpandCollector.getGroups(searcher, rb.rsp.getReturnFields()));
}