in core/src/main/java/org/apache/carbondata/core/scan/filter/executer/RowLevelFilterExecutorImpl.java [196:353]
public BitSetGroup applyFilter(RawBlockletColumnChunks rawBlockletColumnChunks,
boolean useBitsetPipeLine) throws FilterUnsupportedException, IOException {
if (exp instanceof MatchExpression) {
BitSetGroup bitSetGroup = rawBlockletColumnChunks.getBitSetGroup();
if (bitSetGroup == null) {
// It means there are no index created on this table
throw new FilterUnsupportedException(String.format("%s is not supported on table %s",
exp.getFilterExpressionType().name(), tableIdentifier.getTableName()));
}
return bitSetGroup;
}
readColumnChunks(rawBlockletColumnChunks);
// CHECKSTYLE:ON
int[] numberOfRows = null;
int pageNumbers = 0;
if (dimColEvaluatorInfoList.size() > 0) {
if (isDimensionPresentInCurrentBlock[0]) {
pageNumbers = rawBlockletColumnChunks.getDimensionRawColumnChunks()[dimensionChunkIndex[0]]
.getPagesCount();
numberOfRows = rawBlockletColumnChunks.getDimensionRawColumnChunks()[dimensionChunkIndex[0]]
.getRowCount();
} else {
// specific for restructure case where default values need to be filled
pageNumbers = rawBlockletColumnChunks.getDataBlock().numberOfPages();
numberOfRows = new int[pageNumbers];
for (int i = 0; i < pageNumbers; i++) {
numberOfRows[i] = rawBlockletColumnChunks.getDataBlock().getPageRowCount(i);
}
}
}
if (msrColEvalutorInfoList.size() > 0) {
if (isMeasurePresentInCurrentBlock[0]) {
pageNumbers =
rawBlockletColumnChunks.getMeasureRawColumnChunks()[msrColEvalutorInfoList.get(0)
.getColumnIndex()].getPagesCount();
numberOfRows =
rawBlockletColumnChunks.getMeasureRawColumnChunks()[msrColEvalutorInfoList.get(0)
.getColumnIndex()].getRowCount();
} else {
// specific for restructure case where default values need to be filled
pageNumbers = rawBlockletColumnChunks.getDataBlock().numberOfPages();
numberOfRows = new int[pageNumbers];
for (int i = 0; i < pageNumbers; i++) {
numberOfRows[i] = rawBlockletColumnChunks.getDataBlock().getPageRowCount(i);
}
}
}
BitSetGroup bitSetGroup = new BitSetGroup(pageNumbers);
if (isDimensionPresentInCurrentBlock.length == 1 && isDimensionPresentInCurrentBlock[0]
&& dimColEvaluatorInfoList.get(0).getDimension().getDataType().isComplexType()
&& exp instanceof EqualToExpression) {
LiteralExpression literalExp = (LiteralExpression) (((EqualToExpression) exp).getRight());
// convert filter value to byte[] to compare with byte[] data from columnPage
Object literalExpValue = literalExp.getLiteralExpValue();
DataType literalExpDataType = literalExp.getLiteralExpDataType();
if (literalExpDataType == DataTypes.TIMESTAMP) {
if ((long) literalExpValue == 0) {
literalExpValue = null;
} else {
literalExpValue =
(long) literalExpValue / TimeStampGranularityTypeValue.MILLIS_SECONDS.getValue();
}
} else if (literalExpDataType == DataTypes.DATE) {
// change data type to int to get the byte[] filter value as it is direct dictionary
literalExpDataType = DataTypes.INT;
if (literalExpValue == null) {
literalExpValue = CarbonCommonConstants.DIRECT_DICT_VALUE_NULL;
} else {
literalExpValue =
(int) literalExpValue + DateDirectDictionaryGenerator.cutOffDate;
}
}
byte[] filterValueInBytes = DataTypeUtil.getBytesDataDataTypeForNoDictionaryColumn(
literalExpValue,
literalExpDataType);
ArrayQueryType complexType =
(ArrayQueryType) complexDimensionInfoMap.get(dimensionChunkIndex[0]);
int totalCount = 0;
// check all the pages
for (int i = 0; i < pageNumbers; i++) {
if (limit != -1 && totalCount >= limit) {
break;
}
BitSet set = new BitSet(numberOfRows[i]);
int[][] numberOfChild = complexType
.getNumberOfChild(rawBlockletColumnChunks.getDimensionRawColumnChunks(), null,
numberOfRows[i], i);
DimensionColumnPage page = complexType
.parseBlockAndReturnChildData(rawBlockletColumnChunks.getDimensionRawColumnChunks(),
null, i);
// check every row
for (int index = 0; index < numberOfRows[i]; index++) {
if (limit != -1 && totalCount >= limit) {
break;
}
int dataOffset = numberOfChild[index][1];
// loop the children
for (int j = 0; j < numberOfChild[index][0]; j++) {
byte[] obj = page.getChunkData(dataOffset++);
if (ByteUtil.UnsafeComparer.INSTANCE.compareTo(obj, filterValueInBytes) == 0) {
set.set(index);
totalCount++;
break;
}
}
}
bitSetGroup.setBitSet(set, i);
}
} else {
for (int i = 0; i < pageNumbers; i++) {
BitSet set = new BitSet(numberOfRows[i]);
RowIntf row = new RowImpl();
BitSet prvBitset = null;
// if bitset pipe line is enabled then use row id from previous bitset
// otherwise use older flow
if (!useBitsetPipeLine ||
null == rawBlockletColumnChunks.getBitSetGroup() ||
null == bitSetGroup.getBitSet(i) ||
rawBlockletColumnChunks.getBitSetGroup().getBitSet(i).isEmpty()) {
for (int index = 0; index < numberOfRows[i]; index++) {
createRow(rawBlockletColumnChunks, row, i, index);
Boolean result = false;
try {
result = exp.evaluate(row).getBoolean();
}
// Any invalid member while evaluation shall be ignored, system will log the
// error only once since all rows the evaluation happens so inorder to avoid
// too much log inforation only once the log will be printed.
catch (FilterIllegalMemberException e) {
FilterUtil.logError(e, false);
}
if (null != result && result) {
set.set(index);
}
}
} else {
prvBitset = rawBlockletColumnChunks.getBitSetGroup().getBitSet(i);
for (int index = prvBitset.nextSetBit(0);
index >= 0; index = prvBitset.nextSetBit(index + 1)) {
createRow(rawBlockletColumnChunks, row, i, index);
Boolean rslt = false;
try {
rslt = exp.evaluate(row).getBoolean();
} catch (FilterIllegalMemberException e) {
FilterUtil.logError(e, false);
}
if (null != rslt && rslt) {
set.set(index);
}
}
}
bitSetGroup.setBitSet(set, i);
}
}
return bitSetGroup;
}