in shims/common/src/main/scala/io/glutenproject/GlutenConfig.scala [42:205]
def enableGluten: Boolean = conf.getConf(GLUTEN_ENABLED)
// FIXME the option currently controls both JVM and native validation against a Substrait plan.
def enableNativeValidation: Boolean = conf.getConf(NATIVE_VALIDATION_ENABLED)
def enableColumnarBatchScan: Boolean = conf.getConf(COLUMNAR_BATCHSCAN_ENABLED)
def enableColumnarFileScan: Boolean = conf.getConf(COLUMNAR_FILESCAN_ENABLED)
def enableColumnarHiveTableScan: Boolean = conf.getConf(COLUMNAR_HIVETABLESCAN_ENABLED)
def enableVanillaVectorizedReaders: Boolean = conf.getConf(VANILLA_VECTORIZED_READERS_ENABLED)
def enableColumnarHashAgg: Boolean = conf.getConf(COLUMNAR_HASHAGG_ENABLED)
def forceToUseHashAgg: Boolean = conf.getConf(COLUMNAR_FORCE_HASHAGG_ENABLED)
def enableColumnarProject: Boolean = conf.getConf(COLUMNAR_PROJECT_ENABLED)
def enableColumnarFilter: Boolean = conf.getConf(COLUMNAR_FILTER_ENABLED)
def enableColumnarSort: Boolean = conf.getConf(COLUMNAR_SORT_ENABLED)
def enableColumnarWindow: Boolean = conf.getConf(COLUMNAR_WINDOW_ENABLED)
def veloxColumnarWindowType: String = conf.getConfString(COLUMNAR_VELOX_WINDOW_TYPE.key)
def enableColumnarShuffledHashJoin: Boolean = conf.getConf(COLUMNAR_SHUFFLED_HASH_JOIN_ENABLED)
def enableNativeColumnarToRow: Boolean = conf.getConf(COLUMNAR_COLUMNAR_TO_ROW_ENABLED)
def forceShuffledHashJoin: Boolean = conf.getConf(COLUMNAR_FPRCE_SHUFFLED_HASH_JOIN_ENABLED)
def enableColumnarSortMergeJoin: Boolean = conf.getConf(COLUMNAR_SORTMERGEJOIN_ENABLED)
def enableColumnarUnion: Boolean = conf.getConf(COLUMNAR_UNION_ENABLED)
def enableColumnarExpand: Boolean = conf.getConf(COLUMNAR_EXPAND_ENABLED)
def enableColumnarBroadcastExchange: Boolean = conf.getConf(COLUMNAR_BROADCAST_EXCHANGE_ENABLED)
def enableColumnarBroadcastJoin: Boolean = conf.getConf(COLUMNAR_BROADCAST_JOIN_ENABLED)
def enableColumnarArrowUDF: Boolean = conf.getConf(COLUMNAR_ARROW_UDF_ENABLED)
def enableColumnarCoalesce: Boolean = conf.getConf(COLUMNAR_COALESCE_ENABLED)
def columnarTableCacheEnabled: Boolean = conf.getConf(COLUMNAR_TABLE_CACHE_ENABLED)
def enableRewriteDateTimestampComparison: Boolean =
conf.getConf(ENABLE_REWRITE_DATE_TIMESTAMP_COMPARISON)
def enableCommonSubexpressionEliminate: Boolean =
conf.getConf(ENABLE_COMMON_SUBEXPRESSION_ELIMINATE)
def veloxOrcScanEnabled: Boolean =
conf.getConf(VELOX_ORC_SCAN_ENABLED)
// whether to use ColumnarShuffleManager
def isUseColumnarShuffleManager: Boolean =
conf
.getConfString("spark.shuffle.manager", "sort")
.equals("org.apache.spark.shuffle.sort.ColumnarShuffleManager")
// whether to use CelebornShuffleManager
def isUseCelebornShuffleManager: Boolean =
conf
.getConfString("spark.shuffle.manager", "sort")
.contains("celeborn")
def enableColumnarShuffle: Boolean = conf.getConf(COLUMNAR_SHUFFLE_ENABLED)
def enablePreferColumnar: Boolean = conf.getConf(COLUMNAR_PREFER_ENABLED)
def enableOneRowRelationColumnar: Boolean = conf.getConf(COLUMNAR_ONE_ROW_RELATION_ENABLED)
def physicalJoinOptimizationThrottle: Integer =
conf.getConf(COLUMNAR_PHYSICAL_JOIN_OPTIMIZATION_THROTTLE)
def enablePhysicalJoinOptimize: Boolean =
conf.getConf(COLUMNAR_PHYSICAL_JOIN_OPTIMIZATION_ENABLED)
def logicalJoinOptimizationThrottle: Integer =
conf.getConf(COLUMNAR_LOGICAL_JOIN_OPTIMIZATION_THROTTLE)
def enableLogicalJoinOptimize: Boolean =
conf.getConf(COLUMNAR_LOGICAL_JOIN_OPTIMIZATION_ENABLED)
def enableScanOnly: Boolean = conf.getConf(COLUMNAR_SCAN_ONLY_ENABLED)
def tmpFile: Option[String] = conf.getConf(COLUMNAR_TEMP_DIR)
@deprecated def broadcastCacheTimeout: Int = conf.getConf(COLUMNAR_BROADCAST_CACHE_TIMEOUT)
def columnarShuffleReallocThreshold: Double = conf.getConf(COLUMNAR_SHUFFLE_REALLOC_THRESHOLD)
def columnarShuffleMergeThreshold: Double = conf.getConf(SHUFFLE_WRITER_MERGE_THRESHOLD)
def columnarShuffleCodec: Option[String] = conf.getConf(COLUMNAR_SHUFFLE_CODEC)
def columnarShuffleCompressionMode: String =
conf.getConf(COLUMNAR_SHUFFLE_COMPRESSION_MODE)
def columnarShuffleCodecBackend: Option[String] = conf
.getConf(COLUMNAR_SHUFFLE_CODEC_BACKEND)
.filter(Set(GLUTEN_QAT_BACKEND_NAME, GLUTEN_IAA_BACKEND_NAME).contains(_))
def columnarShuffleEnableQat: Boolean =
columnarShuffleCodecBackend.contains(GlutenConfig.GLUTEN_QAT_BACKEND_NAME)
def columnarShuffleEnableIaa: Boolean =
columnarShuffleCodecBackend.contains(GlutenConfig.GLUTEN_IAA_BACKEND_NAME)
def columnarShuffleCompressionThreshold: Int =
conf.getConf(COLUMNAR_SHUFFLE_COMPRESSION_THRESHOLD)
def maxBatchSize: Int = conf.getConf(COLUMNAR_MAX_BATCH_SIZE)
def shuffleWriterBufferSize: Int = conf
.getConf(SHUFFLE_WRITER_BUFFER_SIZE)
.getOrElse(maxBatchSize)
def enableColumnarLimit: Boolean = conf.getConf(COLUMNAR_LIMIT_ENABLED)
def enableColumnarGenerate: Boolean = conf.getConf(COLUMNAR_GENERATE_ENABLED)
def enableTakeOrderedAndProject: Boolean =
conf.getConf(COLUMNAR_TAKE_ORDERED_AND_PROJECT_ENABLED)
def enableNativeBloomFilter: Boolean = conf.getConf(COLUMNAR_NATIVE_BLOOMFILTER_ENABLED)
def enableNativeHyperLogLogAggregateFunction: Boolean =
conf.getConf(COLUMNAR_NATIVE_HYPERLOGLOG_AGGREGATE_ENABLED)
def columnarParquetWriteBlockSize: Long =
conf.getConf(COLUMNAR_PARQUET_WRITE_BLOCK_SIZE)
def columnarParquetWriteBlockRows: Long =
conf.getConf(COLUMNAR_PARQUET_WRITE_BLOCK_ROWS)
def wholeStageFallbackThreshold: Int = conf.getConf(COLUMNAR_WHOLESTAGE_FALLBACK_THRESHOLD)
def queryFallbackThreshold: Int = conf.getConf(COLUMNAR_QUERY_FALLBACK_THRESHOLD)
def fallbackIgnoreRowToColumnar: Boolean = conf.getConf(COLUMNAR_FALLBACK_IGNORE_ROW_TO_COLUMNAR)
def fallbackPreferColumnar: Boolean = conf.getConf(COLUMNAR_FALLBACK_PREFER_COLUMNAR)
def numaBindingInfo: GlutenNumaBindingInfo = {
val enableNumaBinding: Boolean = conf.getConf(COLUMNAR_NUMA_BINDING_ENABLED)
if (!enableNumaBinding) {
GlutenNumaBindingInfo(enableNumaBinding = false)
} else {
val tmp = conf.getConf(COLUMNAR_NUMA_BINDING_CORE_RANGE)
if (tmp.isEmpty) {
GlutenNumaBindingInfo(enableNumaBinding = false)
} else {
val numCores = conf.getConfString("spark.executor.cores", "1").toInt
val coreRangeList: Array[String] = tmp.get.split('|').map(_.trim)
GlutenNumaBindingInfo(enableNumaBinding = true, coreRangeList, numCores)
}
}
}