in core/src/main/java/org/apache/carbondata/core/datastore/page/encoding/compress/DirectCompressCodec.java [328:544]
private void fillPrimitiveType(byte[] pageData, CarbonColumnVector vector,
DataType vectorDataType, DataType pageDataType, int pageSize, ColumnVectorInfo vectorInfo,
BitSet nullBits) {
int intSizeInBytes = DataTypes.INT.getSizeInBytes();
int shortSizeInBytes = DataTypes.SHORT.getSizeInBytes();
int lengthStoredInBytes;
// check if local dictionary is enabled for complex primitve type
if (!vectorInfo.vectorStack.isEmpty()) {
CarbonColumnVectorImpl tempVector =
(CarbonColumnVectorImpl) (vectorInfo.vectorStack.peek().getColumnVector());
if (tempVector.getLocalDictionary() != null) {
DimensionChunkStoreFactory.DimensionStoreType dimStoreType =
DimensionChunkStoreFactory.DimensionStoreType.LOCAL_DICT;
// This will call fillVector() for local-dict eventually
new VariableLengthDimensionColumnPage(pageData, new int[0], new int[0], pageSize,
dimStoreType, tempVector.getLocalDictionary(), vectorInfo, pageSize);
return;
}
}
if (vectorInfo.encodings != null && vectorInfo.encodings.size() > 0 && CarbonUtil
.hasEncoding(vectorInfo.encodings, Encoding.INT_LENGTH_COMPLEX_CHILD_BYTE_ARRAY)) {
lengthStoredInBytes = intSizeInBytes;
} else {
// Before to carbon 2.0, complex child length is stored as SHORT
// for string, varchar, binary, date, decimal types
lengthStoredInBytes = shortSizeInBytes;
}
int rowId = 0;
if (pageDataType == DataTypes.BOOLEAN || pageDataType == DataTypes.BYTE) {
if (vectorDataType == DataTypes.SHORT) {
for (int i = 0; i < pageSize; i++) {
vector.putShort(i, pageData[i]);
}
} else if (vectorDataType == DataTypes.INT) {
for (int i = 0; i < pageSize; i++) {
vector.putInt(i, pageData[i]);
}
} else if (vectorDataType == DataTypes.LONG) {
for (int i = 0; i < pageSize; i++) {
vector.putLong(i, pageData[i]);
}
} else if (vectorDataType == DataTypes.TIMESTAMP) {
for (int i = 0; i < pageSize; i++) {
vector.putLong(i, (long) pageData[i] * 1000);
}
} else if (vectorDataType == DataTypes.BOOLEAN || vectorDataType == DataTypes.BYTE) {
vector.putBytes(0, pageSize, pageData, 0);
} else if (DataTypes.isDecimal(vectorDataType)) {
DecimalConverterFactory.DecimalConverter decimalConverter = vectorInfo.decimalConverter;
decimalConverter.fillVector(pageData, pageSize, vectorInfo, nullBits, pageDataType);
} else {
for (int i = 0; i < pageSize; i++) {
vector.putDouble(i, pageData[i]);
}
}
} else if (pageDataType == DataTypes.SHORT) {
int size = pageSize * shortSizeInBytes;
if (vectorDataType == DataTypes.SHORT) {
for (int i = 0; i < size; i += shortSizeInBytes) {
vector.putShort(rowId++, (ByteUtil.toShortLittleEndian(pageData, i)));
}
} else if (vectorDataType == DataTypes.INT) {
for (int i = 0; i < size; i += shortSizeInBytes) {
vector.putInt(rowId++, ByteUtil.toShortLittleEndian(pageData, i));
}
} else if (vectorDataType == DataTypes.LONG) {
for (int i = 0; i < size; i += shortSizeInBytes) {
vector.putLong(rowId++, ByteUtil.toShortLittleEndian(pageData, i));
}
} else if (vectorDataType == DataTypes.TIMESTAMP) {
for (int i = 0; i < size; i += shortSizeInBytes) {
vector.putLong(rowId++, (long) ByteUtil.toShortLittleEndian(pageData, i) * 1000);
}
} else if (DataTypes.isDecimal(vectorDataType)) {
DecimalConverterFactory.DecimalConverter decimalConverter = vectorInfo.decimalConverter;
decimalConverter.fillVector(pageData, pageSize, vectorInfo, nullBits, pageDataType);
} else {
for (int i = 0; i < size; i += shortSizeInBytes) {
vector.putDouble(rowId++, ByteUtil.toShortLittleEndian(pageData, i));
}
}
} else if (pageDataType == DataTypes.SHORT_INT) {
if (vectorDataType == DataTypes.INT) {
for (int i = 0; i < pageSize; i++) {
int shortInt = ByteUtil.valueOf3Bytes(pageData, i * 3);
vector.putInt(i, shortInt);
}
} else if (vectorDataType == DataTypes.LONG) {
for (int i = 0; i < pageSize; i++) {
int shortInt = ByteUtil.valueOf3Bytes(pageData, i * 3);
vector.putLong(i, shortInt);
}
} else if (vectorDataType == DataTypes.TIMESTAMP) {
for (int i = 0; i < pageSize; i++) {
int shortInt = ByteUtil.valueOf3Bytes(pageData, i * 3);
vector.putLong(i, (long) shortInt * 1000);
}
} else if (DataTypes.isDecimal(vectorDataType)) {
DecimalConverterFactory.DecimalConverter decimalConverter = vectorInfo.decimalConverter;
decimalConverter.fillVector(pageData, pageSize, vectorInfo, nullBits, pageDataType);
} else {
for (int i = 0; i < pageSize; i++) {
int shortInt = ByteUtil.valueOf3Bytes(pageData, i * 3);
vector.putDouble(i, shortInt);
}
}
} else {
if (pageDataType == DataTypes.INT) {
int size = pageSize * intSizeInBytes;
if (vectorDataType == DataTypes.INT) {
for (int i = 0; i < size; i += intSizeInBytes) {
vector.putInt(rowId++, ByteUtil.toIntLittleEndian(pageData, i));
}
} else if (vectorDataType == DataTypes.LONG) {
for (int i = 0; i < size; i += intSizeInBytes) {
vector.putLong(rowId++, ByteUtil.toIntLittleEndian(pageData, i));
}
} else if (vectorDataType == DataTypes.TIMESTAMP) {
for (int i = 0; i < size; i += intSizeInBytes) {
vector.putLong(rowId++, (long) ByteUtil.toIntLittleEndian(pageData, i) * 1000);
}
} else if (DataTypes.isDecimal(vectorDataType)) {
DecimalConverterFactory.DecimalConverter decimalConverter = vectorInfo.decimalConverter;
decimalConverter.fillVector(pageData, pageSize, vectorInfo, nullBits, pageDataType);
} else {
for (int i = 0; i < size; i += intSizeInBytes) {
vector.putDouble(rowId++, ByteUtil.toIntLittleEndian(pageData, i));
}
}
} else if (pageDataType == DataTypes.LONG) {
int longSizeInBytes = DataTypes.LONG.getSizeInBytes();
int size = pageSize * longSizeInBytes;
if (vectorDataType == DataTypes.LONG) {
for (int i = 0; i < size; i += longSizeInBytes) {
vector.putLong(rowId++, ByteUtil.toLongLittleEndian(pageData, i));
}
} else if (vectorDataType == DataTypes.TIMESTAMP) {
for (int i = 0; i < size; i += longSizeInBytes) {
vector.putLong(rowId++, ByteUtil.toLongLittleEndian(pageData, i) * 1000);
}
} else if (DataTypes.isDecimal(vectorDataType)) {
DecimalConverterFactory.DecimalConverter decimalConverter = vectorInfo.decimalConverter;
decimalConverter.fillVector(pageData, pageSize, vectorInfo, nullBits, pageDataType);
}
} else if (pageDataType == DataTypes.BYTE_ARRAY) {
// for complex primitive types
if (vectorDataType == DataTypes.STRING || vectorDataType == DataTypes.BINARY
|| vectorDataType == DataTypes.VARCHAR) {
// for complex primitive string, binary, varchar type
int offset = 0;
int length;
for (int i = 0; i < pageSize; i++) {
if (lengthStoredInBytes == intSizeInBytes) {
length = ByteBuffer.wrap(pageData, offset, lengthStoredInBytes).getInt();
} else {
length = ByteBuffer.wrap(pageData, offset, lengthStoredInBytes).getShort();
}
offset += lengthStoredInBytes;
if (vectorDataType == DataTypes.BINARY && length == 0) {
vector.putNull(i);
continue;
}
byte[] row = new byte[length];
System.arraycopy(pageData, offset, row, 0, length);
if (Arrays.equals(row, CarbonCommonConstants.MEMBER_DEFAULT_VAL_ARRAY)) {
vector.putNull(i);
} else {
vector.putObject(i, row);
}
offset += length;
}
} else if (vectorDataType == DataTypes.DATE) {
// for complex primitive date type
int offset = 0;
int length;
for (int i = 0; i < pageSize; i++) {
if (lengthStoredInBytes == intSizeInBytes) {
length = ByteBuffer.wrap(pageData, offset, lengthStoredInBytes).getInt();
} else {
length = ByteBuffer.wrap(pageData, offset, lengthStoredInBytes).getShort();
}
offset += lengthStoredInBytes;
if (length == 0) {
vector.putObject(0, null);
} else {
// Calculating surrogate only in case of non-null values
int surrogateInternal = ByteUtil.toXorInt(pageData, offset, intSizeInBytes);
vector.putObject(0, surrogateInternal - DateDirectDictionaryGenerator.cutOffDate);
}
offset += length;
}
} else if (DataTypes.isDecimal(vectorDataType)) {
// for complex primitive decimal type
DecimalConverterFactory.DecimalConverter decimalConverter = vectorInfo.decimalConverter;
if (decimalConverter == null) {
decimalConverter = DecimalConverterFactory.INSTANCE
.getDecimalConverter(((DecimalType) vectorDataType).getPrecision(),
((DecimalType) vectorDataType).getScale());
}
decimalConverter.fillVector(pageData, pageSize, vectorInfo, nullBits, pageDataType);
}
} else if (vectorDataType == DataTypes.FLOAT) {
int floatSizeInBytes = DataTypes.FLOAT.getSizeInBytes();
int size = pageSize * floatSizeInBytes;
for (int i = 0; i < size; i += floatSizeInBytes) {
vector.putFloat(rowId++, ByteUtil.toFloatLittleEndian(pageData, i));
}
} else {
int doubleSizeInBytes = DataTypes.DOUBLE.getSizeInBytes();
int size = pageSize * doubleSizeInBytes;
for (int i = 0; i < size; i += doubleSizeInBytes) {
vector.putDouble(rowId++, ByteUtil.toDoubleLittleEndian(pageData, i));
}
}
}
}