in fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java [278:718]
public void validate(ConnectContext ctx) {
// pre-block in some cases.
if (columns.isEmpty()) {
throw new AnalysisException("table should contain at least one column");
}
// analyze catalog name
if (Strings.isNullOrEmpty(ctlName)) {
if (ctx.getCurrentCatalog() != null) {
ctlName = ctx.getCurrentCatalog().getName();
} else {
ctlName = InternalCatalog.INTERNAL_CATALOG_NAME;
}
}
paddingEngineName(ctlName, ctx);
checkEngineName();
if (properties == null) {
properties = Maps.newHashMap();
}
if (engineName.equalsIgnoreCase(ENGINE_OLAP)) {
if (distribution == null) {
distribution = new DistributionDescriptor(false, true, FeConstants.default_bucket_num, null);
}
properties = maybeRewriteByAutoBucket(distribution, properties);
}
try {
// check display name for temporary table, its inner name cannot pass validation
FeNameFormat.checkTableName(Util.getTempTableDisplayName(tableName));
} catch (Exception e) {
throw new AnalysisException(e.getMessage(), e);
}
checkEngineWithCatalog();
// analyze table name
if (Strings.isNullOrEmpty(dbName)) {
dbName = ctx.getDatabase();
}
try {
InternalDatabaseUtil.checkDatabase(dbName, ConnectContext.get());
} catch (org.apache.doris.common.AnalysisException e) {
throw new AnalysisException(e.getMessage(), e.getCause());
}
if (!Env.getCurrentEnv().getAccessManager().checkTblPriv(ConnectContext.get(), ctlName, dbName,
tableName, PrivPredicate.CREATE)) {
try {
ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR,
"CREATE");
} catch (Exception ex) {
throw new AnalysisException(ex.getMessage(), ex.getCause());
}
}
Preconditions.checkState(!Strings.isNullOrEmpty(ctlName), "catalog name is null or empty");
Preconditions.checkState(!Strings.isNullOrEmpty(dbName), "database name is null or empty");
//check datev1 and decimalv2
for (ColumnDefinition columnDef : columns) {
String columnNameUpperCase = columnDef.getName().toUpperCase();
if (columnNameUpperCase.startsWith("__DORIS_")) {
throw new AnalysisException(
"Disable to create table column with name start with __DORIS_: " + columnNameUpperCase);
}
if (columnDef.getType().isVariantType() && columnNameUpperCase.indexOf('.') != -1) {
throw new AnalysisException(
"Disable to create table of `VARIANT` type column named with a `.` character: "
+ columnNameUpperCase);
}
if (columnDef.getType().isDateType() && Config.disable_datev1) {
throw new AnalysisException(
"Disable to create table with `DATE` type columns, please use `DATEV2`.");
}
if (columnDef.getType().isDecimalV2Type() && Config.disable_decimalv2) {
throw new AnalysisException("Disable to create table with `DECIMAL` type columns,"
+ "please use `DECIMALV3`.");
}
}
// check duplicated columns
Map<String, ColumnDefinition> columnMap = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
columns.forEach(c -> {
if (columnMap.put(c.getName(), c) != null) {
try {
ErrorReport.reportAnalysisException(ErrorCode.ERR_DUP_FIELDNAME,
c.getName());
} catch (Exception e) {
throw new AnalysisException(e.getMessage(), e.getCause());
}
}
});
if (engineName.equalsIgnoreCase(ENGINE_OLAP)) {
boolean enableDuplicateWithoutKeysByDefault = false;
properties = PropertyAnalyzer.getInstance().rewriteOlapProperties(ctlName, dbName, properties);
try {
if (properties != null) {
enableDuplicateWithoutKeysByDefault =
PropertyAnalyzer.analyzeEnableDuplicateWithoutKeysByDefault(properties);
}
} catch (Exception e) {
throw new AnalysisException(e.getMessage(), e.getCause());
}
if (keys.isEmpty()) {
boolean hasAggColumn = false;
for (ColumnDefinition column : columns) {
if (column.getAggType() != null) {
hasAggColumn = true;
break;
}
}
keys = Lists.newArrayList();
if (hasAggColumn) {
for (ColumnDefinition column : columns) {
if (column.getAggType() != null) {
break;
}
keys.add(column.getName());
}
keysType = KeysType.AGG_KEYS;
} else {
if (!enableDuplicateWithoutKeysByDefault) {
int keyLength = 0;
for (ColumnDefinition column : columns) {
DataType type = column.getType();
Type catalogType = column.getType().toCatalogDataType();
keyLength += catalogType.getIndexSize();
if (keys.size() >= FeConstants.shortkey_max_column_count
|| keyLength > FeConstants.shortkey_maxsize_bytes) {
if (keys.isEmpty() && type.isStringLikeType()) {
keys.add(column.getName());
}
break;
}
if (!catalogType.couldBeShortKey()) {
break;
}
keys.add(column.getName());
if (type.isVarcharType()) {
break;
}
}
}
keysType = KeysType.DUP_KEYS;
}
// The OLAP table must have at least one short key,
// and the float and double should not be short key,
// so the float and double could not be the first column in OLAP table.
if (keys.isEmpty() && (keysType != KeysType.DUP_KEYS
|| !enableDuplicateWithoutKeysByDefault)) {
throw new AnalysisException(
"The olap table first column could not be float, double, string"
+ " or array, struct, map, please use decimal or varchar instead.");
}
} else if (enableDuplicateWithoutKeysByDefault) {
throw new AnalysisException(
"table property 'enable_duplicate_without_keys_by_default' only can"
+ " set 'true' when create olap table by default.");
}
if (properties != null
&& properties.containsKey(PropertyAnalyzer.ENABLE_UNIQUE_KEY_MERGE_ON_WRITE)) {
if (!keysType.equals(KeysType.UNIQUE_KEYS)) {
throw new AnalysisException(PropertyAnalyzer.ENABLE_UNIQUE_KEY_MERGE_ON_WRITE
+ " property only support unique key table");
}
}
if (keysType == KeysType.UNIQUE_KEYS) {
isEnableMergeOnWrite = false;
if (properties != null) {
properties = PropertyAnalyzer.enableUniqueKeyMergeOnWriteIfNotExists(properties);
// `analyzeXXX` would modify `properties`, which will be used later,
// so we just clone a properties map here.
try {
isEnableMergeOnWrite = PropertyAnalyzer.analyzeUniqueKeyMergeOnWrite(
new HashMap<>(properties));
} catch (Exception e) {
throw new AnalysisException(e.getMessage(), e.getCause());
}
}
}
try {
if (Config.random_add_cluster_keys_for_mow && isEnableMergeOnWrite && clusterKeysColumnNames.isEmpty()
&& PropertyAnalyzer.analyzeUseLightSchemaChange(new HashMap<>(properties))) {
// exclude columns whose data type can not be cluster key, see {@link ColumnDefinition#validate}
List<ColumnDefinition> clusterKeysCandidates = columns.stream().filter(c -> {
DataType type = c.getType();
return !(type.isFloatLikeType() || type.isStringType() || type.isArrayType()
|| type.isBitmapType() || type.isHllType() || type.isQuantileStateType()
|| type.isJsonType()
|| type.isVariantType()
|| type.isMapType()
|| type.isStructType());
}).collect(Collectors.toList());
if (clusterKeysCandidates.size() > 0) {
clusterKeysColumnNames = new ArrayList<>();
Random random = new Random();
int randomClusterKeysCount = random.nextInt(clusterKeysCandidates.size()) + 1;
Collections.shuffle(clusterKeysCandidates);
for (int i = 0; i < randomClusterKeysCount; i++) {
clusterKeysColumnNames.add(clusterKeysCandidates.get(i).getName());
}
LOG.info("Randomly add cluster keys for table {}.{}: {}",
dbName, tableName, clusterKeysColumnNames);
}
}
} catch (Exception e) {
throw new AnalysisException(e.getMessage(), e.getCause());
}
validateKeyColumns();
if (!clusterKeysColumnNames.isEmpty()) {
if (!isEnableMergeOnWrite) {
throw new AnalysisException(
"Cluster keys only support unique keys table which enabled "
+ PropertyAnalyzer.ENABLE_UNIQUE_KEY_MERGE_ON_WRITE);
}
}
for (int i = 0; i < keys.size(); ++i) {
columns.get(i).setIsKey(true);
}
if (keysType != KeysType.AGG_KEYS) {
AggregateType type = AggregateType.REPLACE;
if (keysType == KeysType.DUP_KEYS) {
type = AggregateType.NONE;
}
if (keysType == KeysType.UNIQUE_KEYS && isEnableMergeOnWrite) {
type = AggregateType.NONE;
}
for (int i = keys.size(); i < columns.size(); ++i) {
columns.get(i).setAggType(type);
}
}
// add hidden column
if (keysType.equals(KeysType.UNIQUE_KEYS)) {
if (isEnableMergeOnWrite) {
columns.add(ColumnDefinition.newDeleteSignColumnDefinition(AggregateType.NONE));
} else {
columns.add(
ColumnDefinition.newDeleteSignColumnDefinition(AggregateType.REPLACE));
}
}
// add a hidden column as row store
boolean storeRowColumn = false;
List<String> rowStoreColumns = null;
if (properties != null) {
try {
storeRowColumn =
PropertyAnalyzer.analyzeStoreRowColumn(Maps.newHashMap(properties));
rowStoreColumns = PropertyAnalyzer.analyzeRowStoreColumns(Maps.newHashMap(properties),
columns.stream()
.map(ColumnDefinition::getName)
.collect(Collectors.toList()));
} catch (Exception e) {
throw new AnalysisException(e.getMessage(), e.getCause());
}
}
if (storeRowColumn || (rowStoreColumns != null && !rowStoreColumns.isEmpty())) {
if (keysType.equals(KeysType.AGG_KEYS)) {
throw new AnalysisException("Aggregate table can't support row column now");
}
if (keysType.equals(KeysType.UNIQUE_KEYS)) {
if (isEnableMergeOnWrite) {
columns.add(
ColumnDefinition.newRowStoreColumnDefinition(AggregateType.NONE));
} else {
columns.add(ColumnDefinition
.newRowStoreColumnDefinition(AggregateType.REPLACE));
}
} else {
columns.add(ColumnDefinition.newRowStoreColumnDefinition(null));
}
}
if (Config.enable_hidden_version_column_by_default
&& keysType.equals(KeysType.UNIQUE_KEYS)) {
if (isEnableMergeOnWrite) {
columns.add(ColumnDefinition.newVersionColumnDefinition(AggregateType.NONE));
} else {
columns.add(ColumnDefinition.newVersionColumnDefinition(AggregateType.REPLACE));
}
}
if (properties != null) {
if (properties.containsKey(PropertyAnalyzer.ENABLE_UNIQUE_KEY_SKIP_BITMAP_COLUMN)
&& !(keysType.equals(KeysType.UNIQUE_KEYS) && isEnableMergeOnWrite)) {
throw new AnalysisException("table property enable_unique_key_skip_bitmap_column can"
+ "only be set in merge-on-write unique table.");
}
// the merge-on-write table must have enable_unique_key_skip_bitmap_column table property
// and its value should be consistent with whether the table's full schema contains
// the skip bitmap hidden column
if (keysType.equals(KeysType.UNIQUE_KEYS) && isEnableMergeOnWrite) {
properties = PropertyAnalyzer.addEnableUniqueKeySkipBitmapPropertyIfNotExists(properties);
// `analyzeXXX` would modify `properties`, which will be used later,
// so we just clone a properties map here.
try {
isEnableSkipBitmapColumn = PropertyAnalyzer.analyzeUniqueKeySkipBitmapColumn(
new HashMap<>(properties));
} catch (Exception e) {
throw new AnalysisException(e.getMessage(), e.getCause());
}
}
}
if (isEnableSkipBitmapColumn && keysType.equals(KeysType.UNIQUE_KEYS)) {
if (isEnableMergeOnWrite) {
columns.add(ColumnDefinition.newSkipBitmapColumnDef(AggregateType.NONE));
}
// TODO(bobhan1): add support for mor table
}
// validate partition
partitionTableInfo.extractPartitionColumns();
partitionTableInfo.validatePartitionInfo(
engineName, columns, columnMap, properties, ctx, isEnableMergeOnWrite, isExternal);
// validate distribution descriptor
distribution.updateCols(columns.get(0).getName());
distribution.validate(columnMap, keysType);
// validate key set.
if (!distribution.isHash()) {
if (keysType.equals(KeysType.UNIQUE_KEYS)) {
throw new AnalysisException(
"Should not be distributed by random when keys type is unique");
} else if (keysType.equals(KeysType.AGG_KEYS)) {
for (ColumnDefinition c : columns) {
if (AggregateType.REPLACE.equals(c.getAggType())
|| AggregateType.REPLACE_IF_NOT_NULL.equals(c.getAggType())) {
throw new AnalysisException(
"Should not be distributed by random when keys type is agg"
+ "and column is in replace, [" + c.getName()
+ "] is invalid");
}
}
}
}
for (RollupDefinition rollup : rollups) {
rollup.validate();
}
} else {
// mysql, broker and hive do not need key desc
if (keysType != null) {
throw new AnalysisException(
"Create " + engineName + " table should not contain keys desc");
}
if (!rollups.isEmpty()) {
throw new AnalysisException(engineName + " catalog doesn't support rollup tables.");
}
if (engineName.equalsIgnoreCase(ENGINE_ICEBERG) && distribution != null) {
throw new AnalysisException(
"Iceberg doesn't support 'DISTRIBUTE BY', "
+ "and you can use 'bucket(num, column)' in 'PARTITIONED BY'.");
}
for (ColumnDefinition columnDef : columns) {
if (!columnDef.isNullable()
&& engineName.equalsIgnoreCase(ENGINE_HIVE)) {
throw new AnalysisException(engineName + " catalog doesn't support column with 'NOT NULL'.");
}
columnDef.setIsKey(true);
}
// TODO: support iceberg partition check
if (engineName.equalsIgnoreCase(ENGINE_HIVE)) {
partitionTableInfo.validatePartitionInfo(
engineName, columns, columnMap, properties, ctx, false, true);
}
}
// validate column
try {
if (!engineName.equals(ENGINE_ELASTICSEARCH) && columns.isEmpty()) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLE_MUST_HAVE_COLUMNS);
}
} catch (Exception e) {
throw new AnalysisException(e.getMessage(), e.getCause());
}
final boolean finalEnableMergeOnWrite = isEnableMergeOnWrite;
Set<String> keysSet = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER);
keysSet.addAll(keys);
Set<String> clusterKeySet = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER);
clusterKeySet.addAll(clusterKeysColumnNames);
columns.forEach(c -> c.validate(engineName.equals(ENGINE_OLAP), keysSet, clusterKeySet, finalEnableMergeOnWrite,
keysType));
// validate index
if (!indexes.isEmpty()) {
Set<String> distinct = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
Set<Pair<IndexDef.IndexType, List<String>>> distinctCol = new HashSet<>();
TInvertedIndexFileStorageFormat invertedIndexFileStorageFormat;
try {
invertedIndexFileStorageFormat = PropertyAnalyzer.analyzeInvertedIndexFileStorageFormat(
new HashMap<>(properties));
} catch (Exception e) {
throw new AnalysisException(e.getMessage(), e.getCause());
}
for (IndexDefinition indexDef : indexes) {
indexDef.validate();
if (!engineName.equalsIgnoreCase(ENGINE_OLAP)) {
throw new AnalysisException(
"index only support in olap engine at current version.");
}
for (String indexColName : indexDef.getColumnNames()) {
boolean found = false;
for (ColumnDefinition column : columns) {
if (column.getName().equalsIgnoreCase(indexColName)) {
indexDef.checkColumn(column, keysType, isEnableMergeOnWrite,
invertedIndexFileStorageFormat);
found = true;
break;
}
}
if (!found) {
throw new AnalysisException(
"Column does not exist in table. invalid column: " + indexColName);
}
}
distinct.add(indexDef.getIndexName());
distinctCol.add(Pair.of(indexDef.getIndexType(), indexDef.getColumnNames().stream()
.map(String::toUpperCase).collect(Collectors.toList())));
}
if (distinct.size() != indexes.size()) {
throw new AnalysisException("index name must be unique.");
}
if (distinctCol.size() != indexes.size()) {
throw new AnalysisException(
"same index columns have multiple same type index is not allowed.");
}
}
generatedColumnCheck(ctx);
}