in fe/fe-core/src/main/java/org/apache/doris/analysis/NativeInsertStmt.java [569:839]
private void analyzeSubquery(Analyzer analyzer, boolean skipCheck) throws UserException {
// Analyze columns mentioned in the statement.
Set<String> mentionedColumns = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER);
if (targetColumnNames == null) {
hasEmptyTargetColumns = true;
// the mentioned columns are columns which are visible to user, so here we use
// getBaseSchema(), not getFullSchema()
for (Column col : targetTable.getBaseSchema(false)) {
mentionedColumns.add(col.getName());
targetColumns.add(col);
}
} else {
for (String colName : targetColumnNames) {
Column col = targetTable.getColumn(colName);
if (col == null) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_BAD_FIELD_ERROR, colName, targetTable.getName());
}
if (!mentionedColumns.add(colName)) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_FIELD_SPECIFIED_TWICE, colName);
}
targetColumns.add(col);
}
// hll column must in mentionedColumns
for (Column col : targetTable.getBaseSchema()) {
if (col.getType().isObjectStored() && !col.hasDefaultValue()
&& !mentionedColumns.contains(col.getName())) {
throw new AnalysisException(
"object-stored column " + col.getName() + " must in insert into columns");
}
}
}
/*
* When doing schema change, there may be some shadow columns. we should add
* them to the end of targetColumns. And use 'origColIdxsForExtendCols' to save
* the index of column in 'targetColumns' which the shadow column related to.
* eg: origin targetColumns: (A,B,C), shadow column: __doris_shadow_B after
* processing, targetColumns: (A, B, C, __doris_shadow_B), and
* origColIdxsForExtendCols has 1 element: "1", which is the index of column B
* in targetColumns.
*
* Rule A: If the column which the shadow column related to is not mentioned,
* then do not add the shadow column to targetColumns. They will be filled by
* null or default value when loading.
*
* When table have materialized view, there may be some materialized view columns.
* we should add them to the end of targetColumns.
* eg: origin targetColumns: (A,B,C), shadow column: mv_bitmap_union_C
* after processing, targetColumns: (A, B, C, mv_bitmap_union_C), and
* origColIdx2MVColumn has 1 element: "2, mv_bitmap_union_C"
* will be used in as a mapping from queryStmt.getResultExprs() to targetColumns define expr
*/
List<Pair<Integer, Column>> origColIdxsForExtendCols = Lists.newArrayList();
if (!ConnectContext.get().isTxnModel()) {
for (Column column : targetTable.getFullSchema()) {
if (column.isNameWithPrefix(SchemaChangeHandler.SHADOW_NAME_PREFIX)) {
String origName = Column.removeNamePrefix(column.getName());
for (int i = 0; i < targetColumns.size(); i++) {
if (targetColumns.get(i).nameEquals(origName, false)) {
// Rule A
origColIdxsForExtendCols.add(Pair.of(i, null));
targetColumns.add(column);
break;
}
}
}
if (column.isNameWithPrefix(CreateMaterializedViewStmt.MATERIALIZED_VIEW_NAME_PREFIX)
|| column.isNameWithPrefix(
CreateMaterializedViewStmt.MATERIALIZED_VIEW_AGGREGATE_NAME_PREFIX)) {
List<SlotRef> refColumns = column.getRefColumns();
if (refColumns == null) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_BAD_FIELD_ERROR,
column.getName(), targetTable.getName());
}
for (SlotRef refColumn : refColumns) {
String origName = refColumn.getColumnName();
for (int originColumnIdx = 0; originColumnIdx < targetColumns.size(); originColumnIdx++) {
if (targetColumns.get(originColumnIdx).nameEquals(origName, false)) {
origColIdxsForExtendCols.add(Pair.of(originColumnIdx, column));
targetColumns.add(column);
break;
}
}
}
}
}
}
// parse query statement
queryStmt.setFromInsert(true);
queryStmt.analyze(analyzer);
// deal with this case: insert into tbl values();
// should try to insert default values for all columns in tbl if set
if (isValuesOrConstantSelect) {
final ValueList valueList = ((SelectStmt) queryStmt).getValueList();
if (valueList != null && valueList.getFirstRow().isEmpty() && CollectionUtils.isEmpty(targetColumnNames)) {
final int rowSize = mentionedColumns.size();
final List<String> colLabels = queryStmt.getColLabels();
final List<Expr> resultExprs = queryStmt.getResultExprs();
Preconditions.checkState(resultExprs.isEmpty(), "result exprs should be empty.");
for (int i = 0; i < rowSize; i++) {
resultExprs.add(new StringLiteral(SelectStmt.DEFAULT_VALUE));
final DefaultValueExpr defaultValueExpr = new DefaultValueExpr();
valueList.getFirstRow().add(defaultValueExpr);
colLabels.add(defaultValueExpr.toColumnLabel());
}
}
}
if (analyzer.getContext().getSessionVariable().isEnableUniqueKeyPartialUpdate()) {
trySetPartialUpdate();
}
// check if size of select item equal with columns mentioned in statement
if (mentionedColumns.size() != queryStmt.getResultExprs().size()) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_VALUE_COUNT);
}
// Check if all columns mentioned is enough
// For JdbcTable, it is allowed to insert without specifying all columns and without checking
if (!(targetTable instanceof JdbcTable)) {
checkColumnCoverage(mentionedColumns, targetTable.getBaseSchema());
}
List<String> realTargetColumnNames = targetColumns.stream().map(Column::getName).collect(Collectors.toList());
// handle VALUES() or SELECT constant list
if (isValuesOrConstantSelect) {
SelectStmt selectStmt = (SelectStmt) queryStmt;
if (selectStmt.getValueList() != null) {
// INSERT INTO VALUES(...)
List<ArrayList<Expr>> rows = selectStmt.getValueList().getRows();
for (int rowIdx = 0; rowIdx < rows.size(); ++rowIdx) {
// Only check for JdbcTable
if (targetTable instanceof JdbcTable) {
// Check for NULL values in not-nullable columns
for (int colIdx = 0; colIdx < targetColumns.size(); ++colIdx) {
Column column = targetColumns.get(colIdx);
// Ensure rows.get(rowIdx) has enough columns to match targetColumns
if (colIdx < rows.get(rowIdx).size()) {
Expr expr = rows.get(rowIdx).get(colIdx);
if (!column.isAllowNull() && expr instanceof NullLiteral) {
throw new AnalysisException("Column `" + column.getName()
+ "` is not nullable, but the inserted value is nullable.");
}
}
}
}
analyzeRow(analyzer, targetColumns, rows, rowIdx, origColIdxsForExtendCols, realTargetColumnNames,
skipCheck);
}
// clear these 2 structures, rebuild them using VALUES exprs
selectStmt.getResultExprs().clear();
selectStmt.getBaseTblResultExprs().clear();
for (int i = 0; i < selectStmt.getValueList().getFirstRow().size(); ++i) {
selectStmt.getResultExprs().add(selectStmt.getValueList().getFirstRow().get(i));
selectStmt.getBaseTblResultExprs().add(selectStmt.getValueList().getFirstRow().get(i));
}
} else {
// INSERT INTO SELECT 1,2,3 ...
List<ArrayList<Expr>> rows = Lists.newArrayList();
// ATTN: must copy the `selectStmt.getResultExprs()`, otherwise the following
// `selectStmt.getResultExprs().clear();` will clear the `rows` too, causing
// error.
rows.add(Lists.newArrayList(selectStmt.getResultExprs()));
analyzeRow(analyzer, targetColumns, rows, 0, origColIdxsForExtendCols, realTargetColumnNames,
skipCheck);
// rows may be changed in analyzeRow(), so rebuild the result exprs
selectStmt.getResultExprs().clear();
// For JdbcTable, need to check whether there is a NULL value inserted into the NOT NULL column
if (targetTable instanceof JdbcTable) {
for (int colIdx = 0; colIdx < targetColumns.size(); ++colIdx) {
Column column = targetColumns.get(colIdx);
Expr expr = rows.get(0).get(colIdx);
if (!column.isAllowNull() && expr instanceof NullLiteral) {
throw new AnalysisException("Column `" + column.getName()
+ "` is not nullable, but the inserted value is nullable.");
}
}
}
for (Expr expr : rows.get(0)) {
selectStmt.getResultExprs().add(expr);
}
}
} else {
// INSERT INTO SELECT ... FROM tbl
if (!origColIdxsForExtendCols.isEmpty()) {
// extend the result expr by duplicating the related exprs
Map<String, Expr> slotToIndex = buildSlotToIndex(queryStmt.getResultExprs(), realTargetColumnNames,
analyzer);
for (Pair<Integer, Column> entry : origColIdxsForExtendCols) {
if (entry.second == null) {
queryStmt.getResultExprs().add(queryStmt.getResultExprs().get(entry.first));
} else {
// substitute define expr slot with select statement result expr
ExprSubstitutionMap smap = new ExprSubstitutionMap();
List<SlotRef> columns = entry.second.getRefColumns();
for (SlotRef slot : columns) {
smap.getLhs().add(slot);
smap.getRhs()
.add(Load.getExprFromDesc(analyzer, slotToIndex.get(slot.getColumnName()), slot));
}
Expr e = entry.second.getDefineExpr().clone(smap);
e.analyze(analyzer);
queryStmt.getResultExprs().add(e);
}
}
}
// check compatibility
for (int i = 0; i < targetColumns.size(); ++i) {
Column column = targetColumns.get(i);
Expr expr = queryStmt.getResultExprs().get(i);
queryStmt.getResultExprs().set(i, expr.checkTypeCompatibility(column.getType()));
}
}
// expand colLabels in QueryStmt
if (!origColIdxsForExtendCols.isEmpty()) {
if (queryStmt.getResultExprs().size() != queryStmt.getBaseTblResultExprs().size()) {
Map<String, Expr> slotToIndex = buildSlotToIndex(queryStmt.getBaseTblResultExprs(),
realTargetColumnNames, analyzer);
for (Pair<Integer, Column> entry : origColIdxsForExtendCols) {
if (entry.second == null) {
queryStmt.getBaseTblResultExprs().add(queryStmt.getBaseTblResultExprs().get(entry.first));
} else {
// substitute define expr slot with select statement result expr
ExprSubstitutionMap smap = new ExprSubstitutionMap();
List<SlotRef> columns = entry.second.getRefColumns();
for (SlotRef slot : columns) {
smap.getLhs().add(slot);
smap.getRhs()
.add(Load.getExprFromDesc(analyzer, slotToIndex.get(slot.getColumnName()), slot));
}
Expr e = entry.second.getDefineExpr().clone(smap);
e.analyze(analyzer);
queryStmt.getBaseTblResultExprs().add(e);
}
}
}
if (queryStmt.getResultExprs().size() != queryStmt.getColLabels().size()) {
for (Pair<Integer, Column> entry : origColIdxsForExtendCols) {
queryStmt.getColLabels().add(queryStmt.getColLabels().get(entry.first));
}
}
}
if (LOG.isDebugEnabled()) {
for (Expr expr : queryStmt.getResultExprs()) {
if (LOG.isDebugEnabled()) {
LOG.debug("final result expr: {}, {}", expr, System.identityHashCode(expr));
}
}
for (Expr expr : queryStmt.getBaseTblResultExprs()) {
if (LOG.isDebugEnabled()) {
LOG.debug("final base table result expr: {}, {}", expr, System.identityHashCode(expr));
}
}
for (String colLabel : queryStmt.getColLabels()) {
if (LOG.isDebugEnabled()) {
LOG.debug("final col label: {}", colLabel);
}
}
}
}