in src/datasource-service/src/main/java/org/apache/kylin/rest/service/TableService.java [1424:1515]
private ReloadTableContext calcReloadContext(String project, String tableIdentity, boolean failFast)
throws Exception {
val context = new ReloadTableContext();
UserGroupInformation ugi = KerberosLoginManager.getInstance().getProjectUGI(project);
val tableMeta = ugi.doAs(new PrivilegedExceptionAction<Pair<TableDesc, TableExtDesc>>() {
@Override
public Pair<TableDesc, TableExtDesc> run() throws Exception {
return extractTableMeta(new String[] { tableIdentity }, project).get(0);
}
});
TableDesc newTableDesc = new TableDesc(tableMeta.getFirst());
context.setTableDesc(newTableDesc);
context.setTableExtDesc(tableMeta.getSecond());
handleExcludedColumns(project, context, newTableDesc, tableIdentity);
TableDesc originTableDesc = getManager(NTableMetadataManager.class, project).getTableDesc(tableIdentity);
val collector = Collectors.toMap(ColumnDesc::getName, col -> Pair.newPair(col.getDatatype(), col.getComment()));
val diff = Maps.difference(Stream.of(originTableDesc.getColumns()).collect(collector),
Stream.of(newTableDesc.getColumns()).collect(collector));
val dataTypeCollector = Collectors.toMap(ColumnDesc::getName, ColumnDesc::getDatatype);
val originCols = Stream.of(originTableDesc.getColumns()).collect(dataTypeCollector);
val newCols = Stream.of(newTableDesc.getColumns()).collect(dataTypeCollector);
val dataTypeDiff = Maps.difference(newCols, originCols);
assert diff.entriesDiffering().keySet().containsAll(dataTypeDiff.entriesDiffering().keySet());
context.setAddColumns(dataTypeDiff.entriesOnlyOnLeft().keySet());
context.setRemoveColumns(dataTypeDiff.entriesOnlyOnRight().keySet());
context.setChangedColumns(diff.entriesDiffering().keySet());
context.setChangeTypeColumns(dataTypeDiff.entriesDiffering().keySet());
context.setTableCommentChanged(
!Objects.equals(originTableDesc.getTableComment(), newTableDesc.getTableComment()));
if (!context.isNeedProcess()) {
return context;
}
if (failFast) {
checkNewColumn(project, newTableDesc.getIdentity(), Sets.newHashSet(context.getAddColumns()));
checkEffectedJobs(newTableDesc, context.isOnlyAddCols());
} else {
Set<String> duplicatedColumnsSet = Sets.newHashSet();
Multimap<String, String> duplicatedColumns = getDuplicatedColumns(project, newTableDesc.getIdentity(),
Sets.newHashSet(context.getAddColumns()));
for (Map.Entry<String, String> entry : duplicatedColumns.entries()) {
duplicatedColumnsSet.add(entry.getKey() + "." + entry.getValue());
}
context.setDuplicatedColumns(duplicatedColumnsSet);
context.setEffectedJobs(getEffectedJobIds(newTableDesc));
}
if (context.isOnlyAddCols()) {
return context;
}
val dependencyGraph = SchemaUtil.dependencyGraph(project, tableIdentity);
Map<String, Set<Pair<NDataModel.Measure, NDataModel.Measure>>> suitableColumnTypeChangedMeasuresMap //
= getSuitableColumnTypeChangedMeasures(dependencyGraph, project, originTableDesc,
dataTypeDiff.entriesDiffering());
BiFunction<Set<String>, Boolean, Map<String, AffectedModelContext>> toAffectedModels = (cols, isDelete) -> {
Set<SchemaNode> affectedNodes = Sets.newHashSet();
val columnMap = Arrays.stream(originTableDesc.getColumns())
.collect(Collectors.toMap(ColumnDesc::getName, Function.identity()));
cols.forEach(colName -> {
if (columnMap.get(colName) != null) {
affectedNodes.addAll(
Graphs.reachableNodes(dependencyGraph, SchemaNode.ofTableColumn(columnMap.get(colName))));
}
});
val nodesMap = affectedNodes.stream().filter(SchemaNode::isModelNode)
.collect(Collectors.groupingBy(SchemaNode::getSubject, Collectors.toSet()));
Map<String, AffectedModelContext> modelContexts = Maps.newHashMap();
nodesMap.forEach((key, nodes) -> {
val indexPlan = NIndexPlanManager.getInstance(KylinConfig.readSystemKylinConfig(), project)
.getIndexPlanByModelAlias(key);
Set<Pair<NDataModel.Measure, NDataModel.Measure>> updateMeasures = Sets.newHashSet();
if (!isDelete) {
updateMeasures = suitableColumnTypeChangedMeasuresMap.getOrDefault(key, updateMeasures);
}
val modelContext = new AffectedModelContext(project, indexPlan, nodes, updateMeasures, isDelete);
modelContexts.put(indexPlan.getUuid(), modelContext);
});
return modelContexts;
};
context.setRemoveAffectedModels(toAffectedModels.apply(context.getRemoveColumns(), true));
context.setChangeTypeAffectedModels(toAffectedModels.apply(context.getChangeTypeColumns(), false));
return context;
}