in integration/spark/src/main/scala/org/apache/spark/sql/execution/command/table/CarbonDescribeFormattedCommand.scala [52:309]
override def processMetadata(sparkSession: SparkSession): Seq[Row] = {
val relation = CarbonEnv.getInstance(sparkSession).carbonMetaStore
.lookupRelation(tblIdentifier)(sparkSession).asInstanceOf[CarbonRelation]
setAuditTable(relation.databaseName, relation.tableName)
var results: Seq[(String, String, String)] = child.schema.fields.map { field =>
val colComment = field.getComment().getOrElse("null")
(field.name, field.dataType.simpleString, colComment)
}
val carbonTable = relation.carbonTable
val tblProps = carbonTable.getTableInfo.getFactTable.getTableProperties.asScala
// Append spatial index columns
val indexes = tblProps.get(CarbonCommonConstants.SPATIAL_INDEX)
if (indexes.isDefined) {
results ++= Seq(
("", "", ""),
("## Spatial Index Information", "", "")
)
val indexList = indexes.get.split(",").map(_.trim)
indexList.zip(Stream from 1).foreach {
case(index, count) =>
results ++= Seq(
("Type", tblProps(s"${ CarbonCommonConstants.SPATIAL_INDEX }.$index.type"), ""),
("Class", tblProps(s"${ CarbonCommonConstants.SPATIAL_INDEX }.$index.class"), ""),
("Column Name", index, ""),
("Column Data Type",
tblProps(s"${ CarbonCommonConstants.SPATIAL_INDEX }.$index.datatype"), ""),
("Sources Columns",
tblProps(s"${ CarbonCommonConstants.SPATIAL_INDEX }.$index.sourcecolumns"), ""))
if (tblProps.contains(s"${CarbonCommonConstants.SPATIAL_INDEX}.$index.originlatitude")) {
results ++= Seq(("Origin Latitude",
tblProps(s"${CarbonCommonConstants.SPATIAL_INDEX}.$index.originlatitude"), ""))
}
if (tblProps.contains(s"${CarbonCommonConstants.SPATIAL_INDEX}.$index.gridsize")) {
results ++= Seq(("Grid Size",
tblProps(s"${CarbonCommonConstants.SPATIAL_INDEX}.$index.gridsize"), ""))
}
if (tblProps.contains(s"${CarbonCommonConstants.SPATIAL_INDEX}.$index.conversionratio")) {
results ++= Seq(("Conversion Ratio",
tblProps(s"${CarbonCommonConstants.SPATIAL_INDEX}.$index.conversionratio"), ""))
}
if (indexList.length != count) {
results ++= Seq(("", "", ""))
}
}
}
// If Sort Columns are given and Sort Scope is not given in either table properties
// or carbon properties then pass LOCAL_SORT as the sort scope,
// else pass NO_SORT
val sortScope = if (carbonTable.getNumberOfSortColumns == 0) {
"NO_SORT"
} else {
if (tblProps.contains(CarbonCommonConstants.SORT_SCOPE)) {
tblProps.get(CarbonCommonConstants.SORT_SCOPE).get
} else {
tblProps
.getOrElse(CarbonCommonConstants.SORT_SCOPE,
CarbonProperties.getInstance()
.getProperty(CarbonLoadOptionConstants.CARBON_OPTIONS_SORT_SCOPE,
CarbonProperties.getInstance().getProperty(CarbonCommonConstants.LOAD_SORT_SCOPE,
"LOCAL_SORT")))
}
}
val streaming: String = if (carbonTable.isStreamingSink) {
"sink"
} else if (carbonTable.isStreamingSource) {
"source"
} else {
"false"
}
val catalog = sparkSession.sessionState.catalog
val catalogTable = catalog.getTableMetadata(tblIdentifier)
val pageSizeInMb: String = if (tblProps.get(CarbonCommonConstants.TABLE_PAGE_SIZE_INMB)
.isDefined) {
tblProps(CarbonCommonConstants.TABLE_PAGE_SIZE_INMB)
} else {
""
}
//////////////////////////////////////////////////////////////////////////////
// Table Basic Information
//////////////////////////////////////////////////////////////////////////////
results ++= Seq(
("", "", ""),
("## Detailed Table Information", "", ""),
("Database", catalogTable.database, ""),
("Table", catalogTable.identifier.table, ""),
("Owner", catalogTable.owner, ""),
("Created", new Date(catalogTable.createTime).toString, ""))
if (!EnvHelper.isPrivacy(sparkSession, carbonTable.isExternalTable)) {
results ++= Seq(
("Location ", carbonTable.getTablePath, "")
)
}
results ++= Seq(
("External", carbonTable.isExternalTable.toString, ""),
("Transactional", carbonTable.isTransactionalTable.toString, ""),
("Streaming", streaming, ""),
("Table Block Size ", carbonTable.getBlockSizeInMB + " MB", ""),
("Table Blocklet Size ", carbonTable.getBlockletSizeInMB + " MB", ""),
("Comment", tblProps.getOrElse(CarbonCommonConstants.TABLE_COMMENT, ""), ""),
("Bad Record Path", tblProps.getOrElse("bad_record_path", ""), ""),
("Date Format", tblProps.getOrElse("dateformat", ""), ""),
("Timestamp Format", tblProps.getOrElse("timestampformat", ""), ""),
("Min Input Per Node Per Load",
Strings.formatSize(
tblProps.getOrElse(CarbonCommonConstants.CARBON_LOAD_MIN_SIZE_INMB,
CarbonCommonConstants.CARBON_LOAD_MIN_SIZE_INMB_DEFAULT).toFloat), ""),
//////////////////////////////////////////////////////////////////////////////
// Index Information
//////////////////////////////////////////////////////////////////////////////
("", "", ""),
("## Index Information", "", ""),
("Sort Scope", sortScope, ""),
("Sort Columns", relation.carbonTable.getSortColumns.asScala.mkString(", "), ""),
("Inverted Index Columns", carbonTable.getTableInfo.getFactTable.getTableProperties.asScala
.getOrElse(CarbonCommonConstants.INVERTED_INDEX, ""), ""),
("Cached Min/Max Index Columns",
carbonTable.getMinMaxCachedColumnsInCreateOrder.asScala.mkString(", "), ""),
("Min/Max Index Cache Level",
tblProps.getOrElse(CarbonCommonConstants.CACHE_LEVEL,
CarbonCommonConstants.CACHE_LEVEL_DEFAULT_VALUE), ""),
("Table page size in mb", pageSizeInMb, "")
)
if (carbonTable.getRangeColumn != null) {
results ++= Seq(("RANGE COLUMN", carbonTable.getRangeColumn.getColName, ""))
}
if (carbonTable.getGlobalSortPartitions != null) {
results ++= Seq(("GLOBAL SORT PARTITIONS", carbonTable.getGlobalSortPartitions, ""))
}
//////////////////////////////////////////////////////////////////////////////
// Encoding Information
//////////////////////////////////////////////////////////////////////////////
results ++= Seq(
("", "", ""),
("## Encoding Information", "", ""))
results ++= getLocalDictDesc(carbonTable, tblProps.toMap)
if (tblProps.contains(CarbonCommonConstants.LONG_STRING_COLUMNS)) {
results ++= Seq((CarbonCommonConstants.LONG_STRING_COLUMNS.toUpperCase,
tblProps.getOrElse(CarbonCommonConstants.LONG_STRING_COLUMNS, ""), ""))
}
//////////////////////////////////////////////////////////////////////////////
// Compaction Information
//////////////////////////////////////////////////////////////////////////////
results ++= Seq(
("", "", ""),
("## Compaction Information", "", ""),
(CarbonCommonConstants.TABLE_MAJOR_COMPACTION_SIZE.toUpperCase,
tblProps.getOrElse(CarbonCommonConstants.TABLE_MAJOR_COMPACTION_SIZE,
CarbonProperties.getInstance()
.getProperty(CarbonCommonConstants.CARBON_MAJOR_COMPACTION_SIZE,
CarbonCommonConstants.DEFAULT_CARBON_MAJOR_COMPACTION_SIZE)), ""),
(CarbonCommonConstants.TABLE_MINOR_COMPACTION_SIZE.toUpperCase,
tblProps.getOrElse(CarbonCommonConstants.TABLE_MINOR_COMPACTION_SIZE,
CarbonProperties.getInstance()
.getProperty(CarbonCommonConstants.CARBON_MINOR_COMPACTION_SIZE, "0")), ""),
(CarbonCommonConstants.TABLE_AUTO_LOAD_MERGE.toUpperCase,
tblProps.getOrElse(CarbonCommonConstants.TABLE_AUTO_LOAD_MERGE,
CarbonProperties.getInstance()
.getProperty(CarbonCommonConstants.ENABLE_AUTO_LOAD_MERGE,
CarbonCommonConstants.DEFAULT_ENABLE_AUTO_LOAD_MERGE)), ""),
(CarbonCommonConstants.TABLE_COMPACTION_LEVEL_THRESHOLD.toUpperCase,
tblProps.getOrElse(CarbonCommonConstants.TABLE_COMPACTION_LEVEL_THRESHOLD,
CarbonProperties.getInstance()
.getProperty(CarbonCommonConstants.COMPACTION_SEGMENT_LEVEL_THRESHOLD,
CarbonCommonConstants.DEFAULT_SEGMENT_LEVEL_THRESHOLD)), ""),
(CarbonCommonConstants.TABLE_COMPACTION_PRESERVE_SEGMENTS.toUpperCase,
tblProps.getOrElse(CarbonCommonConstants.TABLE_COMPACTION_PRESERVE_SEGMENTS,
CarbonProperties.getInstance()
.getProperty(CarbonCommonConstants.PRESERVE_LATEST_SEGMENTS_NUMBER,
CarbonCommonConstants.DEFAULT_PRESERVE_LATEST_SEGMENTS_NUMBER)), ""),
(CarbonCommonConstants.TABLE_ALLOWED_COMPACTION_DAYS.toUpperCase,
tblProps.getOrElse(CarbonCommonConstants.TABLE_ALLOWED_COMPACTION_DAYS,
CarbonProperties.getInstance()
.getProperty(CarbonCommonConstants.DAYS_ALLOWED_TO_COMPACT,
CarbonCommonConstants.DEFAULT_DAYS_ALLOWED_TO_COMPACT)), "")
)
//////////////////////////////////////////////////////////////////////////////
// Partition Information
//////////////////////////////////////////////////////////////////////////////
val partitionInfo = carbonTable.getPartitionInfo()
if (partitionInfo != null) {
results ++= Seq(
("", "", ""),
("## Partition Information", "", ""),
("Partition Type", partitionInfo.getPartitionType.toString, ""),
("Partition Columns",
partitionInfo.getColumnSchemaList.asScala.map {
col => s"${col.getColumnName}:${col.getDataType.getName}"}.mkString(", "), ""),
("Number of Partitions", getNumberOfPartitions(carbonTable, sparkSession), "")
)
}
if (partitionSpec.nonEmpty) {
val partitions = sparkSession.sessionState.catalog.getPartition(tblIdentifier, partitionSpec)
results ++=
Seq(("", "", ""),
("## Partition Information", "", ""),
("Partition Type", "Hive", ""),
("Partition Value:", partitions.spec.values.mkString("[", ",", "]"), ""),
("Database:", tblIdentifier.database.getOrElse(sparkSession.catalog.currentDatabase), ""),
("Table:", tblIdentifier.table, ""))
if (partitions.storage.locationUri.isDefined) {
results ++= Seq(("Location:", partitions.storage.locationUri.get.toString, ""))
}
results ++= Seq(("Partition Parameters:", partitions.parameters.mkString(", "), ""))
}
//////////////////////////////////////////////////////////////////////////////
// Bucket Information
//////////////////////////////////////////////////////////////////////////////
val bucketInfo = carbonTable.getBucketingInfo()
if (bucketInfo != null) {
results ++= Seq(
("", "", ""),
("## Bucket Information", "", ""),
("Bucket Columns",
bucketInfo.getListOfColumns.asScala.map {
col => s"${col.getColumnName}:${col.getDataType.getName}"}.mkString(", "), ""),
("Number of Buckets", bucketInfo.getNumOfRanges.toString, "")
)
}
//////////////////////////////////////////////////////////////////////////////
// Dynamic Information
//////////////////////////////////////////////////////////////////////////////
val dataIndexSize = CarbonUtil.calculateDataIndexSize(carbonTable, false)
if (!dataIndexSize.isEmpty) {
if (carbonTable.isTransactionalTable) {
results ++= Seq(
("", "", ""),
("## Dynamic Information", "", ""),
("Table Data Size", Strings.formatSize(
dataIndexSize.get(CarbonCommonConstants.CARBON_TOTAL_DATA_SIZE).floatValue()), ""),
("Table Index Size", Strings.formatSize(
dataIndexSize.get(CarbonCommonConstants.CARBON_TOTAL_INDEX_SIZE).floatValue()), ""),
("Last Update",
new Date(dataIndexSize.get(CarbonCommonConstants.LAST_UPDATE_TIME)).toString, "")
)
} else {
results ++= Seq(
("", "", ""),
("## Dynamic Information", "", ""),
("Table Total Size", Strings.formatSize(
dataIndexSize.get(CarbonCommonConstants.CARBON_TOTAL_DATA_SIZE).floatValue()), "")
)
}
}
results.map{case (c1, c2, c3) => Row(c1, c2, c3)}
}