in spark-connector/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala [461:757]
def unwrapperFor(objectInspector: ObjectInspector): Any => Any =
objectInspector match {
case coi: ConstantObjectInspector if coi.getWritableConstantValue == null =>
_ => null
case poi: WritableConstantStringObjectInspector =>
val constant = UTF8String.fromString(poi.getWritableConstantValue.toString)
_ => constant
case poi: WritableConstantHiveVarcharObjectInspector =>
val constant = UTF8String.fromString(poi.getWritableConstantValue.getHiveVarchar.getValue)
_ => constant
case poi: WritableConstantHiveCharObjectInspector =>
val constant = UTF8String.fromString(poi.getWritableConstantValue.getHiveChar.getValue)
_ => constant
case poi: WritableConstantHiveDecimalObjectInspector =>
val constant = HiveShim.toCatalystDecimal(
PrimitiveObjectInspectorFactory.javaHiveDecimalObjectInspector,
poi.getWritableConstantValue.getHiveDecimal)
_ => constant
case poi: WritableConstantTimestampObjectInspector =>
val t = poi.getWritableConstantValue
val constant = DateTimeUtils.fromJavaTimestamp(t.getTimestamp)
_ => constant
case poi: WritableConstantIntObjectInspector =>
val constant = poi.getWritableConstantValue.get()
_ => constant
case poi: WritableConstantDoubleObjectInspector =>
val constant = poi.getWritableConstantValue.get()
_ => constant
case poi: WritableConstantBooleanObjectInspector =>
val constant = poi.getWritableConstantValue.get()
_ => constant
case poi: WritableConstantLongObjectInspector =>
val constant = poi.getWritableConstantValue.get()
_ => constant
case poi: WritableConstantFloatObjectInspector =>
val constant = poi.getWritableConstantValue.get()
_ => constant
case poi: WritableConstantShortObjectInspector =>
val constant = poi.getWritableConstantValue.get()
_ => constant
case poi: WritableConstantByteObjectInspector =>
val constant = poi.getWritableConstantValue.get()
_ => constant
case poi: WritableConstantBinaryObjectInspector =>
val writable = poi.getWritableConstantValue
val constant = new Array[Byte](writable.getLength)
System.arraycopy(writable.getBytes, 0, constant, 0, constant.length)
_ => constant
case poi: WritableConstantDateObjectInspector =>
val constant = DateTimeUtils.fromJavaDate(poi.getWritableConstantValue.get())
_ => constant
case mi: StandardConstantMapObjectInspector =>
val keyUnwrapper = unwrapperFor(mi.getMapKeyObjectInspector)
val valueUnwrapper = unwrapperFor(mi.getMapValueObjectInspector)
val keyValues = mi.getWritableConstantValue
val constant = ArrayBasedMapData(keyValues, keyUnwrapper, valueUnwrapper)
_ => constant
case li: StandardConstantListObjectInspector =>
val unwrapper = unwrapperFor(li.getListElementObjectInspector)
val values = li.getWritableConstantValue.asScala
.map(unwrapper)
.toArray
val constant = new GenericArrayData(values)
_ => constant
case poi: VoidObjectInspector =>
_ => null // always be null for void object inspector
case dt: WritableConstantHiveIntervalDayTimeObjectInspector =>
val constant = dt.getWritableConstantValue.asInstanceOf[HiveIntervalDayTime]
_ => IntervalUtils.durationToMicros(
Duration.ofSeconds(constant.getTotalSeconds).plusNanos(constant.getNanos.toLong))
case ym: WritableConstantHiveIntervalYearMonthObjectInspector =>
val constant = ym.getWritableConstantValue.asInstanceOf[HiveIntervalYearMonth]
_ => constant.getTotalMonths
case pi: PrimitiveObjectInspector => pi match {
// We think HiveVarchar/HiveChar is also a String
case hvoi: HiveVarcharObjectInspector if hvoi.preferWritable() =>
data: Any => {
if (data != null) {
UTF8String.fromString(hvoi.getPrimitiveWritableObject(data).getHiveVarchar.getValue)
} else {
null
}
}
case hvoi: HiveVarcharObjectInspector =>
data: Any => {
if (data != null) {
UTF8String.fromString(hvoi.getPrimitiveJavaObject(data).getValue)
} else {
null
}
}
case hvoi: HiveCharObjectInspector if hvoi.preferWritable() =>
data: Any => {
if (data != null) {
UTF8String.fromString(hvoi.getPrimitiveWritableObject(data).getHiveChar.getValue)
} else {
null
}
}
case hvoi: HiveCharObjectInspector =>
data: Any => {
if (data != null) {
UTF8String.fromString(hvoi.getPrimitiveJavaObject(data).getValue)
} else {
null
}
}
case x: StringObjectInspector if x.preferWritable() =>
data: Any => {
if (data != null) {
// Text is in UTF-8 already. No need to convert again via fromString. Copy bytes
val wObj = x.getPrimitiveWritableObject(data)
val result = wObj.copyBytes()
UTF8String.fromBytes(result, 0, result.length)
} else {
null
}
}
case x: StringObjectInspector =>
data: Any => {
if (data != null) {
UTF8String.fromString(x.getPrimitiveJavaObject(data))
} else {
null
}
}
case x: IntObjectInspector if x.preferWritable() =>
data: Any => {
if (data != null) x.get(data) else null
}
case x: BooleanObjectInspector if x.preferWritable() =>
data: Any => {
if (data != null) x.get(data) else null
}
case x: FloatObjectInspector if x.preferWritable() =>
data: Any => {
if (data != null) x.get(data) else null
}
case x: DoubleObjectInspector if x.preferWritable() =>
data: Any => {
if (data != null) x.get(data) else null
}
case x: LongObjectInspector if x.preferWritable() =>
data: Any => {
if (data != null) x.get(data) else null
}
case x: ShortObjectInspector if x.preferWritable() =>
data: Any => {
if (data != null) x.get(data) else null
}
case x: ByteObjectInspector if x.preferWritable() =>
data: Any => {
if (data != null) x.get(data) else null
}
case x: HiveDecimalObjectInspector =>
data: Any => {
if (data != null) {
HiveShim.toCatalystDecimal(x, data)
} else {
null
}
}
case x: BinaryObjectInspector if x.preferWritable() =>
data: Any => {
if (data != null) {
// BytesWritable.copyBytes() only available since Hadoop2
// In order to keep backward-compatible, we have to copy the
// bytes with old apis
val bw = x.getPrimitiveWritableObject(data)
val result = new Array[Byte](bw.getLength())
System.arraycopy(bw.getBytes(), 0, result, 0, bw.getLength())
result
} else {
null
}
}
case x: DateObjectInspector if x.preferWritable() =>
data: Any => {
if (data != null) {
new DaysWritable(x.getPrimitiveWritableObject(data)).gregorianDays
} else {
null
}
}
case x: DateObjectInspector =>
data: Any => {
if (data != null) {
DateTimeUtils.fromJavaDate(x.getPrimitiveJavaObject(data))
} else {
null
}
}
case x: TimestampObjectInspector if x.preferWritable() =>
data: Any => {
if (data != null) {
DateTimeUtils.fromJavaTimestamp(x.getPrimitiveWritableObject(data).getTimestamp)
} else {
null
}
}
case ti: TimestampObjectInspector =>
data: Any => {
if (data != null) {
DateTimeUtils.fromJavaTimestamp(ti.getPrimitiveJavaObject(data))
} else {
null
}
}
case dt: HiveIntervalDayTimeObjectInspector if dt.preferWritable() =>
data: Any => {
if (data != null) {
val dayTime = dt.getPrimitiveWritableObject(data).getHiveIntervalDayTime
IntervalUtils.durationToMicros(
Duration.ofSeconds(dayTime.getTotalSeconds).plusNanos(dayTime.getNanos.toLong))
} else {
null
}
}
case dt: HiveIntervalDayTimeObjectInspector =>
data: Any => {
if (data != null) {
val dayTime = dt.getPrimitiveJavaObject(data)
IntervalUtils.durationToMicros(
Duration.ofSeconds(dayTime.getTotalSeconds).plusNanos(dayTime.getNanos.toLong))
} else {
null
}
}
case ym: HiveIntervalYearMonthObjectInspector if ym.preferWritable() =>
data: Any => {
if (data != null) {
ym.getPrimitiveWritableObject(data).getHiveIntervalYearMonth.getTotalMonths
} else {
null
}
}
case ym: HiveIntervalYearMonthObjectInspector =>
data: Any => {
if (data != null) {
ym.getPrimitiveJavaObject(data).getTotalMonths
} else {
null
}
}
case _ =>
data: Any => {
if (data != null) {
pi.getPrimitiveJavaObject(data)
} else {
null
}
}
}
case li: ListObjectInspector =>
val unwrapper = unwrapperFor(li.getListElementObjectInspector)
data: Any => {
if (data != null) {
Option(li.getList(data))
.map { l =>
val values = l.asScala.map(unwrapper).toArray
new GenericArrayData(values)
}
.orNull
} else {
null
}
}
case mi: MapObjectInspector =>
val keyUnwrapper = unwrapperFor(mi.getMapKeyObjectInspector)
val valueUnwrapper = unwrapperFor(mi.getMapValueObjectInspector)
data: Any => {
if (data != null) {
val map = mi.getMap(data)
if (map == null) {
null
} else {
ArrayBasedMapData(map, keyUnwrapper, valueUnwrapper)
}
} else {
null
}
}
// currently, hive doesn't provide the ConstantStructObjectInspector
case si: StructObjectInspector =>
val fields = si.getAllStructFieldRefs.asScala
val unwrappers = fields.map { field =>
val unwrapper = unwrapperFor(field.getFieldObjectInspector)
data: Any => unwrapper(si.getStructFieldData(data, field))
}
data: Any => {
if (data != null) {
InternalRow.fromSeq(unwrappers.map(_(data)).toSeq)
} else {
null
}
}
}