in fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java [307:650]
public void analyze(Analyzer analyzer) throws UserException {
if (Strings.isNullOrEmpty(engineName) || engineName.equalsIgnoreCase(DEFAULT_ENGINE_NAME)) {
this.properties = maybeRewriteByAutoBucket(distributionDesc, properties);
}
if (isTemp && !engineName.equalsIgnoreCase(DEFAULT_ENGINE_NAME)) {
throw new AnalysisException("Temporary table should be OLAP table");
}
super.analyze(analyzer);
tableName.analyze(analyzer);
FeNameFormat.checkTableName(tableName.getTbl());
InternalDatabaseUtil.checkDatabase(tableName.getDb(), ConnectContext.get());
if (!Env.getCurrentEnv().getAccessManager()
.checkTblPriv(ConnectContext.get(), tableName.getCtl(), tableName.getDb(), tableName.getTbl(),
PrivPredicate.CREATE)) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "CREATE");
}
analyzeEngineName();
boolean enableDuplicateWithoutKeysByDefault = false;
if (properties != null) {
enableDuplicateWithoutKeysByDefault = PropertyAnalyzer.analyzeEnableDuplicateWithoutKeysByDefault(
properties);
}
// pre-block creation with column type ALL
for (ColumnDef columnDef : columnDefs) {
if (Objects.equals(columnDef.getType(), Type.ALL)) {
throw new AnalysisException("Disable to create table with `ALL` type columns.");
}
String columnNameUpperCase = columnDef.getName().toUpperCase();
if (columnNameUpperCase.startsWith("__DORIS_")) {
throw new AnalysisException(
"Disable to create table column with name start with __DORIS_: " + columnNameUpperCase);
}
if (Objects.equals(columnDef.getType(), Type.VARIANT) && columnNameUpperCase.indexOf('.') != -1) {
throw new AnalysisException(
"Disable to create table of `VARIANT` type column named with a `.` character: "
+ columnNameUpperCase);
}
if (Objects.equals(columnDef.getType(), Type.DATE) && Config.disable_datev1) {
throw new AnalysisException("Disable to create table with `DATE` type columns, please use `DATEV2`.");
}
if (Objects.equals(columnDef.getType(), Type.DECIMALV2) && Config.disable_decimalv2) {
throw new AnalysisException(
"Disable to create table with `DECIMAL` type columns," + "please use `DECIMALV3`.");
}
}
boolean enableUniqueKeyMergeOnWrite = false;
boolean enableSkipBitmapColumn = false;
// analyze key desc
if (engineName.equalsIgnoreCase(DEFAULT_ENGINE_NAME)) {
// olap table
if (keysDesc == null) {
List<String> keysColumnNames = Lists.newArrayList();
int keyLength = 0;
boolean hasAggregate = false;
for (ColumnDef columnDef : columnDefs) {
if (columnDef.getAggregateType() != null) {
hasAggregate = true;
break;
}
}
if (hasAggregate) {
for (ColumnDef columnDef : columnDefs) {
if (columnDef.getAggregateType() == null && !columnDef.getType()
.isScalarType(PrimitiveType.STRING) && !columnDef.getType()
.isScalarType(PrimitiveType.JSONB)) {
keysColumnNames.add(columnDef.getName());
}
}
keysDesc = new KeysDesc(KeysType.AGG_KEYS, keysColumnNames);
} else {
if (!enableDuplicateWithoutKeysByDefault) {
for (ColumnDef columnDef : columnDefs) {
keyLength += columnDef.getType().getIndexSize();
if (keysColumnNames.size() >= FeConstants.shortkey_max_column_count
|| keyLength > FeConstants.shortkey_maxsize_bytes) {
if (keysColumnNames.isEmpty() && columnDef.getType().getPrimitiveType()
.isCharFamily()) {
keysColumnNames.add(columnDef.getName());
}
break;
}
if (!columnDef.getType().couldBeShortKey()) {
break;
}
if (columnDef.getType().getPrimitiveType() == PrimitiveType.VARCHAR) {
keysColumnNames.add(columnDef.getName());
break;
}
keysColumnNames.add(columnDef.getName());
}
// The OLAP table must have at least one short key,
// and the float and double should not be short key,
// so the float and double could not be the first column in OLAP table.
if (keysColumnNames.isEmpty()) {
throw new AnalysisException("The olap table first column could not be float, double, string"
+ " or array, struct, map, please use decimal or varchar instead.");
}
}
keysDesc = new KeysDesc(KeysType.DUP_KEYS, keysColumnNames);
}
} else {
if (enableDuplicateWithoutKeysByDefault) {
throw new AnalysisException("table property 'enable_duplicate_without_keys_by_default' only can"
+ " set 'true' when create olap table by default.");
}
}
if (properties != null && properties.containsKey(PropertyAnalyzer.ENABLE_UNIQUE_KEY_MERGE_ON_WRITE)
&& keysDesc.getKeysType() != KeysType.UNIQUE_KEYS) {
throw new AnalysisException(
PropertyAnalyzer.ENABLE_UNIQUE_KEY_MERGE_ON_WRITE + " property only support unique key table");
}
if (keysDesc.getKeysType() == KeysType.UNIQUE_KEYS) {
enableUniqueKeyMergeOnWrite = false;
if (properties != null) {
properties = PropertyAnalyzer.enableUniqueKeyMergeOnWriteIfNotExists(properties);
// `analyzeXXX` would modify `properties`, which will be used later,
// so we just clone a properties map here.
enableUniqueKeyMergeOnWrite = PropertyAnalyzer.analyzeUniqueKeyMergeOnWrite(
new HashMap<>(properties));
}
}
if (properties != null) {
if (properties.containsKey(PropertyAnalyzer.ENABLE_UNIQUE_KEY_SKIP_BITMAP_COLUMN)
&& !(keysDesc.getKeysType() == KeysType.UNIQUE_KEYS && enableUniqueKeyMergeOnWrite)) {
throw new AnalysisException("table property enable_unique_key_skip_bitmap_column can"
+ "only be set in merge-on-write unique table.");
}
// the merge-on-write table must have enable_unique_key_skip_bitmap_column table property
// and its value should be consistent with whether the table's full schema contains
// the skip bitmap hidden column
if (keysDesc.getKeysType() == KeysType.UNIQUE_KEYS && enableUniqueKeyMergeOnWrite) {
properties = PropertyAnalyzer.addEnableUniqueKeySkipBitmapPropertyIfNotExists(properties);
// `analyzeXXX` would modify `properties`, which will be used later,
// so we just clone a properties map here.
enableSkipBitmapColumn = PropertyAnalyzer.analyzeUniqueKeySkipBitmapColumn(
new HashMap<>(properties));
}
}
keysDesc.analyze(columnDefs);
if (!CollectionUtils.isEmpty(keysDesc.getClusterKeysColumnNames())) {
if (!enableUniqueKeyMergeOnWrite) {
throw new AnalysisException("Cluster keys only support unique keys table which enabled "
+ PropertyAnalyzer.ENABLE_UNIQUE_KEY_MERGE_ON_WRITE);
}
}
for (int i = 0; i < keysDesc.keysColumnSize(); ++i) {
columnDefs.get(i).setIsKey(true);
}
if (keysDesc.getKeysType() != KeysType.AGG_KEYS) {
AggregateType type = AggregateType.REPLACE;
if (keysDesc.getKeysType() == KeysType.DUP_KEYS) {
type = AggregateType.NONE;
}
if (keysDesc.getKeysType() == KeysType.UNIQUE_KEYS && enableUniqueKeyMergeOnWrite) {
type = AggregateType.NONE;
}
for (int i = keysDesc.keysColumnSize(); i < columnDefs.size(); ++i) {
columnDefs.get(i).setAggregateType(type);
}
}
} else {
// mysql, broker and hive do not need key desc
if (keysDesc != null) {
throw new AnalysisException("Create " + engineName + " table should not contain keys desc");
}
for (ColumnDef columnDef : columnDefs) {
columnDef.setIsKey(true);
}
}
// analyze column def
if (!(engineName.equalsIgnoreCase("elasticsearch")) && (columnDefs == null || columnDefs.isEmpty())) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLE_MUST_HAVE_COLUMNS);
}
// add a hidden column as delete flag for unique table
if (keysDesc != null && keysDesc.getKeysType() == KeysType.UNIQUE_KEYS) {
if (enableUniqueKeyMergeOnWrite) {
columnDefs.add(ColumnDef.newDeleteSignColumnDef(AggregateType.NONE));
} else {
columnDefs.add(ColumnDef.newDeleteSignColumnDef(AggregateType.REPLACE));
}
}
// add a hidden column as row store
if (properties != null && PropertyAnalyzer.analyzeStoreRowColumn(new HashMap<>(properties))) {
if (keysDesc != null && keysDesc.getKeysType() == KeysType.AGG_KEYS) {
throw new AnalysisException("Aggregate table can't support row column now");
}
if (keysDesc != null && keysDesc.getKeysType() == KeysType.UNIQUE_KEYS) {
if (enableUniqueKeyMergeOnWrite) {
columnDefs.add(ColumnDef.newRowStoreColumnDef(AggregateType.NONE));
} else {
columnDefs.add(ColumnDef.newRowStoreColumnDef(AggregateType.REPLACE));
}
} else {
columnDefs.add(ColumnDef.newRowStoreColumnDef(null));
}
}
if (Config.enable_hidden_version_column_by_default && keysDesc != null
&& keysDesc.getKeysType() == KeysType.UNIQUE_KEYS) {
if (enableUniqueKeyMergeOnWrite) {
columnDefs.add(ColumnDef.newVersionColumnDef(AggregateType.NONE));
} else {
columnDefs.add(ColumnDef.newVersionColumnDef(AggregateType.REPLACE));
}
}
if (enableSkipBitmapColumn && keysDesc != null
&& keysDesc.getKeysType() == KeysType.UNIQUE_KEYS) {
if (enableUniqueKeyMergeOnWrite) {
columnDefs.add(ColumnDef.newSkipBitmapColumnDef(AggregateType.NONE));
}
// TODO(bobhan1): add support for mor table
}
Set<String> columnSet = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER);
for (ColumnDef columnDef : columnDefs) {
columnDef.analyze(engineName.equalsIgnoreCase(DEFAULT_ENGINE_NAME));
if (columnDef.getType().isComplexType() && engineName.equalsIgnoreCase(DEFAULT_ENGINE_NAME)) {
if (columnDef.getAggregateType() != null && columnDef.getAggregateType() != AggregateType.NONE
&& columnDef.getAggregateType() != AggregateType.REPLACE
&& columnDef.getAggregateType() != AggregateType.REPLACE_IF_NOT_NULL) {
throw new AnalysisException(
columnDef.getType().getPrimitiveType() + " column can't support aggregation "
+ columnDef.getAggregateType());
}
if (columnDef.isKey() || columnDef.getClusterKeyId() != -1) {
throw new AnalysisException(columnDef.getType().getPrimitiveType()
+ " can only be used in the non-key column of the duplicate table at present.");
}
}
if (columnDef.getType().isTime() || columnDef.getType().isTimeV2()) {
throw new AnalysisException("Time type is not supported for olap table");
}
if (!columnSet.add(columnDef.getName())) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_DUP_FIELDNAME, columnDef.getName());
}
}
if (engineName.equalsIgnoreCase(DEFAULT_ENGINE_NAME)) {
// before analyzing partition, handle the replication allocation info
properties = PropertyAnalyzer.getInstance().rewriteOlapProperties(tableName.getCtl(),
tableName.getDb(), properties);
// analyze partition
if (partitionDesc != null) {
if (partitionDesc instanceof ListPartitionDesc || partitionDesc instanceof RangePartitionDesc) {
partitionDesc.analyze(columnDefs, properties);
} else {
throw new AnalysisException(
"Currently only support range" + " and list partition with engine type olap");
}
}
// analyze distribution
if (distributionDesc == null) {
distributionDesc = new RandomDistributionDesc(FeConstants.default_bucket_num, true);
}
distributionDesc.analyze(columnSet, columnDefs, keysDesc);
if (distributionDesc.type == DistributionInfo.DistributionInfoType.RANDOM) {
if (keysDesc.getKeysType() == KeysType.UNIQUE_KEYS) {
throw new AnalysisException("Create unique keys table should not contain random distribution desc");
} else if (keysDesc.getKeysType() == KeysType.AGG_KEYS) {
for (ColumnDef columnDef : columnDefs) {
if (columnDef.getAggregateType() == AggregateType.REPLACE
|| columnDef.getAggregateType() == AggregateType.REPLACE_IF_NOT_NULL) {
throw new AnalysisException(
"Create aggregate keys table with value columns of which" + " aggregate type is "
+ columnDef.getAggregateType() + " should not contain random"
+ " distribution desc");
}
}
}
}
} else if (engineName.equalsIgnoreCase("elasticsearch")) {
EsUtil.analyzePartitionAndDistributionDesc(partitionDesc, distributionDesc);
} else {
if (partitionDesc != null || distributionDesc != null) {
throw new AnalysisException(
"Create " + engineName + " table should not contain partition or distribution desc");
}
}
for (ColumnDef columnDef : columnDefs) {
Column col = columnDef.toColumn();
if (keysDesc != null && keysDesc.getKeysType() == KeysType.UNIQUE_KEYS) {
if (!col.isKey()) {
col.setAggregationTypeImplicit(true);
}
}
columns.add(col);
}
if (CollectionUtils.isNotEmpty(indexDefs)) {
Set<String> distinct = new TreeSet<>(String.CASE_INSENSITIVE_ORDER);
Set<Pair<IndexType, List<String>>> distinctCol = new HashSet<>();
TInvertedIndexFileStorageFormat invertedIndexFileStorageFormat = PropertyAnalyzer
.analyzeInvertedIndexFileStorageFormat(new HashMap<>(properties));
for (IndexDef indexDef : indexDefs) {
indexDef.analyze();
if (!engineName.equalsIgnoreCase(DEFAULT_ENGINE_NAME)) {
throw new AnalysisException("index only support in olap engine at current version.");
}
for (String indexColName : indexDef.getColumns()) {
boolean found = false;
for (Column column : columns) {
if (column.getName().equalsIgnoreCase(indexColName)) {
indexDef.checkColumn(column,
getKeysDesc().getKeysType(),
enableUniqueKeyMergeOnWrite,
invertedIndexFileStorageFormat);
found = true;
break;
}
}
if (!found) {
throw new AnalysisException("Column does not exist in table. invalid column: " + indexColName);
}
}
indexes.add(new Index(Env.getCurrentEnv().getNextId(), indexDef.getIndexName(), indexDef.getColumns(),
indexDef.getIndexType(), indexDef.getProperties(), indexDef.getComment()));
distinct.add(indexDef.getIndexName());
distinctCol.add(Pair.of(indexDef.getIndexType(),
indexDef.getColumns().stream().map(String::toUpperCase).collect(Collectors.toList())));
}
if (distinct.size() != indexes.size()) {
throw new AnalysisException("index name must be unique.");
}
if (distinctCol.size() != indexes.size()) {
throw new AnalysisException("same index columns have multiple same type index is not allowed.");
}
}
generatedColumnCheck(analyzer);
}