in spark/src/main/scala/org/apache/comet/parquet/SourceFilterSerde.scala [69:141]
def createValueExpr(value: Any, dataType: DataType): Option[ExprOuterClass.Expr] = {
val exprBuilder = ExprOuterClass.Literal.newBuilder()
var valueIsSet = true
if (value == null) {
exprBuilder.setIsNull(true)
} else {
exprBuilder.setIsNull(false)
// value is a scala value, not a catalyst value
// refer to org.apache.spark.sql.catalyst.CatalystTypeConverters.CatalystTypeConverter#toScala
dataType match {
case _: BooleanType => exprBuilder.setBoolVal(value.asInstanceOf[Boolean])
case _: ByteType => exprBuilder.setByteVal(value.asInstanceOf[Byte])
case _: ShortType => exprBuilder.setShortVal(value.asInstanceOf[Short])
case _: IntegerType => exprBuilder.setIntVal(value.asInstanceOf[Int])
case _: LongType => exprBuilder.setLongVal(value.asInstanceOf[Long])
case _: FloatType => exprBuilder.setFloatVal(value.asInstanceOf[Float])
case _: DoubleType => exprBuilder.setDoubleVal(value.asInstanceOf[Double])
case _: StringType => exprBuilder.setStringVal(value.asInstanceOf[String])
case _: TimestampType =>
value match {
case v: Timestamp => exprBuilder.setLongVal(DateTimeUtils.fromJavaTimestamp(v))
case v: Instant => exprBuilder.setLongVal(DateTimeUtils.instantToMicros(v))
case v: Long => exprBuilder.setLongVal(v)
case _ =>
valueIsSet = false
logWarning(s"Unexpected timestamp type '${value.getClass}' for value '$value'")
}
case _: TimestampNTZType =>
value match {
case v: LocalDateTime =>
exprBuilder.setLongVal(DateTimeUtils.localDateTimeToMicros(v))
case v: Long => exprBuilder.setLongVal(v)
case _ =>
valueIsSet = false
logWarning(s"Unexpected timestamp type '${value.getClass}' for value' $value'")
}
case _: DecimalType =>
// Pass decimal literal as bytes.
val unscaled = value.asInstanceOf[JavaBigDecimal].unscaledValue
exprBuilder.setDecimalVal(com.google.protobuf.ByteString.copyFrom(unscaled.toByteArray))
case _: BinaryType =>
val byteStr =
com.google.protobuf.ByteString.copyFrom(value.asInstanceOf[Array[Byte]])
exprBuilder.setBytesVal(byteStr)
case _: DateType =>
value match {
case v: LocalDate => exprBuilder.setIntVal(DateTimeUtils.localDateToDays(v))
case v: Date => exprBuilder.setIntVal(DateTimeUtils.fromJavaDate(v))
case v: Int => exprBuilder.setIntVal(v)
case _ =>
valueIsSet = false
logWarning(s"Unexpected date type '${value.getClass}' for value '$value'")
}
case dt =>
valueIsSet = false
logWarning(s"Unexpected data type '$dt' for literal value '$value'")
}
}
val dt = serializeDataType(dataType)
if (valueIsSet && dt.isDefined) {
exprBuilder.setDatatype(dt.get)
Some(
ExprOuterClass.Expr
.newBuilder()
.setLiteral(exprBuilder)
.build())
} else {
None
}
}