in shims/common/src/main/scala/org/apache/gluten/config/GlutenConfig.scala [48:248]
def enableGluten: Boolean = getConf(GLUTEN_ENABLED)
def glutenUiEnabled: Boolean = getConf(GLUTEN_UI_ENABLED)
// FIXME the option currently controls both JVM and native validation against a Substrait plan.
def enableNativeValidation: Boolean = getConf(NATIVE_VALIDATION_ENABLED)
def enableColumnarBatchScan: Boolean = getConf(COLUMNAR_BATCHSCAN_ENABLED)
def enableColumnarFileScan: Boolean = getConf(COLUMNAR_FILESCAN_ENABLED)
def enableColumnarHiveTableScan: Boolean = getConf(COLUMNAR_HIVETABLESCAN_ENABLED)
def enableColumnarHiveTableScanNestedColumnPruning: Boolean =
getConf(COLUMNAR_HIVETABLESCAN_NESTED_COLUMN_PRUNING_ENABLED)
def enableColumnarHashAgg: Boolean = getConf(COLUMNAR_HASHAGG_ENABLED)
def forceToUseHashAgg: Boolean = getConf(COLUMNAR_FORCE_HASHAGG_ENABLED)
def mergeTwoPhasesAggEnabled: Boolean = getConf(MERGE_TWO_PHASES_ENABLED)
def enableColumnarProject: Boolean = getConf(COLUMNAR_PROJECT_ENABLED)
def enableColumnarFilter: Boolean = getConf(COLUMNAR_FILTER_ENABLED)
def enableColumnarSort: Boolean = getConf(COLUMNAR_SORT_ENABLED)
def enableColumnarWindow: Boolean = getConf(COLUMNAR_WINDOW_ENABLED)
def enableColumnarWindowGroupLimit: Boolean = getConf(COLUMNAR_WINDOW_GROUP_LIMIT_ENABLED)
def enableColumnarShuffledHashJoin: Boolean = getConf(COLUMNAR_SHUFFLED_HASH_JOIN_ENABLED)
def shuffledHashJoinOptimizeBuildSide: Boolean =
getConf(COLUMNAR_SHUFFLED_HASH_JOIN_OPTIMIZE_BUILD_SIDE)
def forceShuffledHashJoin: Boolean = getConf(COLUMNAR_FORCE_SHUFFLED_HASH_JOIN_ENABLED)
def enableColumnarSortMergeJoin: Boolean = getConf(COLUMNAR_SORTMERGEJOIN_ENABLED)
def enableColumnarUnion: Boolean = getConf(COLUMNAR_UNION_ENABLED)
def enableNativeUnion: Boolean = getConf(NATIVE_UNION_ENABLED)
def enableColumnarExpand: Boolean = getConf(COLUMNAR_EXPAND_ENABLED)
def enableColumnarBroadcastExchange: Boolean = getConf(COLUMNAR_BROADCAST_EXCHANGE_ENABLED)
def enableColumnarBroadcastJoin: Boolean = getConf(COLUMNAR_BROADCAST_JOIN_ENABLED)
def enableColumnarSample: Boolean = getConf(COLUMNAR_SAMPLE_ENABLED)
def enableColumnarArrowUDF: Boolean = getConf(COLUMNAR_ARROW_UDF_ENABLED)
def enableColumnarCoalesce: Boolean = getConf(COLUMNAR_COALESCE_ENABLED)
def enableRewriteDateTimestampComparison: Boolean =
getConf(ENABLE_REWRITE_DATE_TIMESTAMP_COMPARISON)
def enableCollapseNestedGetJsonObject: Boolean =
getConf(ENABLE_COLLAPSE_GET_JSON_OBJECT)
def enableCommonSubexpressionEliminate: Boolean =
getConf(ENABLE_COMMON_SUBEXPRESSION_ELIMINATE)
def enableCountDistinctWithoutExpand: Boolean =
getConf(ENABLE_COUNT_DISTINCT_WITHOUT_EXPAND)
def enableExtendedColumnPruning: Boolean =
getConf(ENABLE_EXTENDED_COLUMN_PRUNING)
def forceOrcCharTypeScanFallbackEnabled: Boolean =
getConf(VELOX_FORCE_ORC_CHAR_TYPE_SCAN_FALLBACK)
def scanFileSchemeValidationEnabled: Boolean =
getConf(VELOX_SCAN_FILE_SCHEME_VALIDATION_ENABLED)
// Whether to use GlutenShuffleManager (experimental).
def isUseGlutenShuffleManager: Boolean =
conf
.getConfString("spark.shuffle.manager", "sort")
.equals("org.apache.spark.shuffle.GlutenShuffleManager")
// Whether to use ColumnarShuffleManager.
def isUseColumnarShuffleManager: Boolean =
conf
.getConfString("spark.shuffle.manager", "sort")
.equals("org.apache.spark.shuffle.sort.ColumnarShuffleManager")
// Whether to use CelebornShuffleManager.
def isUseCelebornShuffleManager: Boolean =
conf
.getConfString("spark.shuffle.manager", "sort")
.contains("celeborn")
// Whether to use UniffleShuffleManager.
def isUseUniffleShuffleManager: Boolean =
conf
.getConfString("spark.shuffle.manager", "sort")
.contains("UniffleShuffleManager")
def celebornShuffleWriterType: String =
conf
.getConfString(
"spark.celeborn.client.spark.shuffle.writer",
ReservedKeys.GLUTEN_HASH_SHUFFLE_WRITER)
.toLowerCase(Locale.ROOT)
def enableColumnarShuffle: Boolean = getConf(COLUMNAR_SHUFFLE_ENABLED)
def physicalJoinOptimizationThrottle: Integer =
getConf(COLUMNAR_PHYSICAL_JOIN_OPTIMIZATION_THROTTLE)
def enablePhysicalJoinOptimize: Boolean =
getConf(COLUMNAR_PHYSICAL_JOIN_OPTIMIZATION_ENABLED)
def enableScanOnly: Boolean = getConf(COLUMNAR_SCAN_ONLY_ENABLED)
def columnarShuffleSortPartitionsThreshold: Int =
getConf(COLUMNAR_SHUFFLE_SORT_PARTITIONS_THRESHOLD)
def columnarShuffleSortColumnsThreshold: Int =
getConf(COLUMNAR_SHUFFLE_SORT_COLUMNS_THRESHOLD)
def columnarShuffleReallocThreshold: Double = getConf(COLUMNAR_SHUFFLE_REALLOC_THRESHOLD)
def columnarShuffleMergeThreshold: Double = getConf(SHUFFLE_WRITER_MERGE_THRESHOLD)
def columnarShuffleCodec: Option[String] = getConf(COLUMNAR_SHUFFLE_CODEC)
def columnarShuffleCompressionMode: String =
getConf(COLUMNAR_SHUFFLE_COMPRESSION_MODE)
def columnarShuffleCodecBackend: Option[String] = getConf(COLUMNAR_SHUFFLE_CODEC_BACKEND)
.filter(Set(GLUTEN_QAT_BACKEND_NAME, GLUTEN_IAA_BACKEND_NAME).contains(_))
def columnarShuffleEnableQat: Boolean =
columnarShuffleCodecBackend.contains(GlutenConfig.GLUTEN_QAT_BACKEND_NAME)
def columnarShuffleEnableIaa: Boolean =
columnarShuffleCodecBackend.contains(GlutenConfig.GLUTEN_IAA_BACKEND_NAME)
def columnarShuffleCompressionThreshold: Int =
getConf(COLUMNAR_SHUFFLE_COMPRESSION_THRESHOLD)
def columnarShuffleReaderBufferSize: Long =
getConf(COLUMNAR_SHUFFLE_READER_BUFFER_SIZE)
def columnarSortShuffleDeserializerBufferSize: Long =
getConf(COLUMNAR_SORT_SHUFFLE_DESERIALIZER_BUFFER_SIZE)
def maxBatchSize: Int = getConf(COLUMNAR_MAX_BATCH_SIZE)
def shuffleWriterBufferSize: Int = getConf(SHUFFLE_WRITER_BUFFER_SIZE)
.getOrElse(maxBatchSize)
def enableColumnarLimit: Boolean = getConf(COLUMNAR_LIMIT_ENABLED)
def enableColumnarGenerate: Boolean = getConf(COLUMNAR_GENERATE_ENABLED)
def enableTakeOrderedAndProject: Boolean =
getConf(COLUMNAR_TAKE_ORDERED_AND_PROJECT_ENABLED)
def enableNativeBloomFilter: Boolean = getConf(COLUMNAR_NATIVE_BLOOMFILTER_ENABLED)
def enableNativeHyperLogLogAggregateFunction: Boolean =
getConf(COLUMNAR_NATIVE_HYPERLOGLOG_AGGREGATE_ENABLED)
def columnarParquetWriteBlockSize: Long =
getConf(COLUMNAR_PARQUET_WRITE_BLOCK_SIZE)
def columnarParquetWriteBlockRows: Long =
getConf(COLUMNAR_PARQUET_WRITE_BLOCK_ROWS)
def wholeStageFallbackThreshold: Int = getConf(COLUMNAR_WHOLESTAGE_FALLBACK_THRESHOLD)
def queryFallbackThreshold: Int = getConf(COLUMNAR_QUERY_FALLBACK_THRESHOLD)
def fallbackIgnoreRowToColumnar: Boolean = getConf(COLUMNAR_FALLBACK_IGNORE_ROW_TO_COLUMNAR)
def fallbackExpressionsThreshold: Int = getConf(COLUMNAR_FALLBACK_EXPRESSIONS_THRESHOLD)
def fallbackPreferColumnar: Boolean = getConf(COLUMNAR_FALLBACK_PREFER_COLUMNAR)
def numaBindingInfo: GlutenNumaBindingInfo = {
val enableNumaBinding: Boolean = getConf(COLUMNAR_NUMA_BINDING_ENABLED)
if (!enableNumaBinding) {
GlutenNumaBindingInfo(enableNumaBinding = false)
} else {
val tmp = getConf(COLUMNAR_NUMA_BINDING_CORE_RANGE)
if (tmp.isEmpty) {
GlutenNumaBindingInfo(enableNumaBinding = false)
} else {
val numCores = conf.getConfString("spark.executor.cores", "1").toInt
val coreRangeList: Array[String] = tmp.get.split('|').map(_.trim)
GlutenNumaBindingInfo(enableNumaBinding = true, coreRangeList, numCores)
}
}
}