in extensions/spark/kyuubi-spark-connector-hive/src/main/scala/org/apache/kyuubi/spark/connector/hive/HiveConnectorUtils.scala [78:168]
def splitFiles(
sparkSession: SparkSession,
file: AnyRef,
filePath: Path,
isSplitable: JBoolean,
maxSplitBytes: JLong,
partitionValues: InternalRow): Seq[PartitionedFile] =
Try { // SPARK-42821: 4.0.0-preview2
val fileStatusWithMetadataClz = DynClasses.builder()
.impl("org.apache.spark.sql.execution.datasources.FileStatusWithMetadata")
.buildChecked()
DynMethods
.builder("splitFiles")
.impl(
"org.apache.spark.sql.execution.PartitionedFileUtil",
fileStatusWithMetadataClz,
classOf[Boolean],
classOf[Long],
classOf[InternalRow])
.buildChecked()
.invokeChecked[Seq[PartitionedFile]](
null,
file,
isSplitable,
maxSplitBytes,
partitionValues)
}.recover { case _: Exception => // SPARK-51185: Spark 3.5.5
val fileStatusWithMetadataClz = DynClasses.builder()
.impl("org.apache.spark.sql.execution.datasources.FileStatusWithMetadata")
.buildChecked()
DynMethods
.builder("splitFiles")
.impl(
"org.apache.spark.sql.execution.PartitionedFileUtil",
classOf[SparkSession],
fileStatusWithMetadataClz,
classOf[Path],
classOf[Boolean],
classOf[Long],
classOf[InternalRow])
.buildChecked()
.invokeChecked[Seq[PartitionedFile]](
null,
sparkSession,
file,
filePath,
isSplitable,
maxSplitBytes,
partitionValues)
}.recover { case _: Exception => // SPARK-43039: 3.5.0
val fileStatusWithMetadataClz = DynClasses.builder()
.impl("org.apache.spark.sql.execution.datasources.FileStatusWithMetadata")
.buildChecked()
DynMethods
.builder("splitFiles")
.impl(
"org.apache.spark.sql.execution.PartitionedFileUtil",
classOf[SparkSession],
fileStatusWithMetadataClz,
classOf[Boolean],
classOf[Long],
classOf[InternalRow])
.buildChecked()
.invokeChecked[Seq[PartitionedFile]](
null,
sparkSession,
file,
isSplitable,
maxSplitBytes,
partitionValues)
}.recover { case _: Exception =>
DynMethods
.builder("splitFiles")
.impl(
"org.apache.spark.sql.execution.PartitionedFileUtil",
classOf[SparkSession],
classOf[FileStatus],
classOf[Path],
classOf[Boolean],
classOf[Long],
classOf[InternalRow])
.buildChecked()
.invokeChecked[Seq[PartitionedFile]](
null,
sparkSession,
file,
filePath,
isSplitable,
maxSplitBytes,
partitionValues)
}.get