in fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java [175:1321]
public static void loadJournal(Env env, Long logId, JournalEntity journal) {
short opCode = journal.getOpCode();
if (opCode != OperationType.OP_SAVE_NEXTID && opCode != OperationType.OP_TIMESTAMP) {
if (LOG.isDebugEnabled()) {
LOG.debug("replay journal op code: {}, log id: {}", opCode, logId);
}
}
try {
switch (opCode) {
case OperationType.OP_SAVE_NEXTID: {
String idString = ((Text) journal.getData()).toString();
long id = Long.parseLong(idString);
env.setNextId(id + 1);
break;
}
case OperationType.OP_SAVE_TRANSACTION_ID: {
String idString = ((Text) journal.getData()).toString();
long id = Long.parseLong(idString);
Env.getCurrentGlobalTransactionMgr().getTransactionIDGenerator().initTransactionId(id + 1);
break;
}
case OperationType.OP_CREATE_DB: {
Database db = (Database) journal.getData();
CreateDbInfo info = new CreateDbInfo(db.getCatalog().getName(), db.getName(), db);
env.replayCreateDb(info);
break;
}
case OperationType.OP_NEW_CREATE_DB: {
CreateDbInfo info = (CreateDbInfo) journal.getData();
env.replayCreateDb(info);
break;
}
case OperationType.OP_DROP_DB: {
DropDbInfo dropDbInfo = (DropDbInfo) journal.getData();
env.replayDropDb(dropDbInfo);
break;
}
case OperationType.OP_ALTER_DB: {
DatabaseInfo dbInfo = (DatabaseInfo) journal.getData();
String dbName = dbInfo.getDbName();
LOG.info("Begin to unprotect alter db info {}", dbName);
env.replayAlterDatabaseQuota(dbName, dbInfo.getQuota(), dbInfo.getQuotaType());
break;
}
case OperationType.OP_ERASE_DB: {
Text dbId = (Text) journal.getData();
env.replayEraseDatabase(Long.parseLong(dbId.toString()));
break;
}
case OperationType.OP_RECOVER_DB: {
RecoverInfo info = (RecoverInfo) journal.getData();
env.replayRecoverDatabase(info);
break;
}
case OperationType.OP_RENAME_DB: {
DatabaseInfo dbInfo = (DatabaseInfo) journal.getData();
String dbName = dbInfo.getDbName();
LOG.info("Begin to unprotect rename db {}", dbName);
env.replayRenameDatabase(dbName, dbInfo.getNewDbName());
break;
}
case OperationType.OP_CREATE_TABLE: {
CreateTableInfo info = (CreateTableInfo) journal.getData();
LOG.info("Begin to unprotect create table. {}", info);
env.replayCreateTable(info);
if (Strings.isNullOrEmpty(info.getCtlName()) || info.getCtlName().equals(
InternalCatalog.INTERNAL_CATALOG_NAME)) {
CreateTableRecord record = new CreateTableRecord(logId, info);
env.getBinlogManager().addCreateTableRecord(record);
}
break;
}
case OperationType.OP_ALTER_EXTERNAL_TABLE_SCHEMA: {
RefreshExternalTableInfo info = (RefreshExternalTableInfo) journal.getData();
LOG.info("Begin to unprotect alter external table schema. db = {} table = {}", info.getDbName(),
info.getTableName());
env.replayAlterExternalTableSchema(info.getDbName(), info.getTableName(), info.getNewSchema());
break;
}
case OperationType.OP_DROP_TABLE: {
DropInfo info = (DropInfo) journal.getData();
LOG.info("Begin to unprotect drop table: {}", info);
if (Strings.isNullOrEmpty(info.getCtl()) || info.getCtl().equals(
InternalCatalog.INTERNAL_CATALOG_NAME)) {
Database db = Env.getCurrentInternalCatalog().getDbOrMetaException(info.getDbId());
env.replayDropTable(db, info.getTableId(), info.isForceDrop(), info.getRecycleTime());
DropTableRecord record = new DropTableRecord(logId, info);
env.getBinlogManager().addDropTableRecord(record);
} else {
ExternalCatalog ctl = (ExternalCatalog) Env.getCurrentEnv().getCatalogMgr()
.getCatalog(info.getCtl());
if (ctl != null) {
ctl.replayDropTable(info.getDb(), info.getTableName());
}
}
break;
}
case OperationType.OP_ADD_PARTITION: {
PartitionPersistInfo info = (PartitionPersistInfo) journal.getData();
LOG.info(
"Begin to unprotect add partition. db = " + info.getDbId() + " table = " + info.getTableId()
+ " partitionName = " + info.getPartition().getName());
AddPartitionRecord addPartitionRecord = new AddPartitionRecord(logId, info);
env.replayAddPartition(info);
env.getBinlogManager().addAddPartitionRecord(addPartitionRecord);
break;
}
case OperationType.OP_DROP_PARTITION: {
DropPartitionInfo info = (DropPartitionInfo) journal.getData();
LOG.info("Begin to unprotect drop partition. db = " + info.getDbId() + " table = "
+ info.getTableId() + " partitionName = " + info.getPartitionName());
env.replayDropPartition(info);
env.getBinlogManager().addDropPartitionRecord(info, logId);
break;
}
case OperationType.OP_MODIFY_PARTITION: {
ModifyPartitionInfo info = (ModifyPartitionInfo) journal.getData();
LOG.info("Begin to unprotect modify partition. db = " + info.getDbId() + " table = "
+ info.getTableId() + " partitionId = " + info.getPartitionId());
BatchModifyPartitionsInfo infos = new BatchModifyPartitionsInfo(info);
env.getAlterInstance().replayModifyPartition(info);
env.getBinlogManager().addModifyPartitions(infos, logId);
break;
}
case OperationType.OP_BATCH_MODIFY_PARTITION: {
BatchModifyPartitionsInfo info = (BatchModifyPartitionsInfo) journal.getData();
for (ModifyPartitionInfo modifyPartitionInfo : info.getModifyPartitionInfos()) {
env.getAlterInstance().replayModifyPartition(modifyPartitionInfo);
}
env.getBinlogManager().addModifyPartitions(info, logId);
break;
}
case OperationType.OP_ERASE_TABLE: {
Text tableId = (Text) journal.getData();
env.replayEraseTable(Long.parseLong(tableId.toString()));
break;
}
case OperationType.OP_ERASE_PARTITION: {
Text partitionId = (Text) journal.getData();
env.replayErasePartition(Long.parseLong(partitionId.toString()));
break;
}
case OperationType.OP_RECOVER_TABLE: {
RecoverInfo info = (RecoverInfo) journal.getData();
env.replayRecoverTable(info);
env.getBinlogManager().addRecoverTableRecord(info, logId);
break;
}
case OperationType.OP_RECOVER_PARTITION: {
RecoverInfo info = (RecoverInfo) journal.getData();
env.replayRecoverPartition(info);
env.getBinlogManager().addRecoverTableRecord(info, logId);
break;
}
case OperationType.OP_RENAME_TABLE: {
TableInfo info = (TableInfo) journal.getData();
env.replayRenameTable(info);
env.getBinlogManager().addTableRename(info, logId);
break;
}
case OperationType.OP_MODIFY_VIEW_DEF: {
AlterViewInfo info = (AlterViewInfo) journal.getData();
env.getAlterInstance().replayModifyViewDef(info);
env.getBinlogManager().addModifyViewDef(info, logId);
break;
}
case OperationType.OP_RENAME_PARTITION: {
TableInfo info = (TableInfo) journal.getData();
env.replayRenamePartition(info);
env.getBinlogManager().addPartitionRename(info, logId);
break;
}
case OperationType.OP_RENAME_COLUMN: {
TableRenameColumnInfo info = (TableRenameColumnInfo) journal.getData();
env.replayRenameColumn(info);
env.getBinlogManager().addColumnRename(info, logId);
break;
}
case OperationType.OP_BACKUP_JOB: {
BackupJob job = (BackupJob) journal.getData();
env.getBackupHandler().replayAddJob(job);
break;
}
case OperationType.OP_RESTORE_JOB: {
RestoreJob job = (RestoreJob) journal.getData();
job.setEnv(env);
env.getBackupHandler().replayAddJob(job);
break;
}
case OperationType.OP_DROP_ROLLUP: {
DropInfo info = (DropInfo) journal.getData();
env.getMaterializedViewHandler().replayDropRollup(info, env);
env.getBinlogManager().addDropRollup(info, logId);
break;
}
case OperationType.OP_BATCH_DROP_ROLLUP: {
BatchDropInfo batchDropInfo = (BatchDropInfo) journal.getData();
if (batchDropInfo.hasIndexNameMap()) {
for (Map.Entry<Long, String> entry : batchDropInfo.getIndexNameMap().entrySet()) {
long indexId = entry.getKey();
String indexName = entry.getValue();
DropInfo info = new DropInfo(batchDropInfo.getDbId(), batchDropInfo.getTableId(),
batchDropInfo.getTableName(), indexId, indexName, false, false, 0);
env.getMaterializedViewHandler().replayDropRollup(info, env);
env.getBinlogManager().addDropRollup(info, logId);
}
} else {
for (Long indexId : batchDropInfo.getIndexIdSet()) {
DropInfo info = new DropInfo(batchDropInfo.getDbId(), batchDropInfo.getTableId(),
batchDropInfo.getTableName(), indexId, "", false, false, 0);
env.getMaterializedViewHandler().replayDropRollup(info, env);
}
}
break;
}
case OperationType.OP_FINISH_CONSISTENCY_CHECK: {
ConsistencyCheckInfo info = (ConsistencyCheckInfo) journal.getData();
env.getConsistencyChecker().replayFinishConsistencyCheck(info, env);
break;
}
case OperationType.OP_CLEAR_ROLLUP_INFO: {
ReplicaPersistInfo info = (ReplicaPersistInfo) journal.getData();
env.getLoadInstance().replayClearRollupInfo(info, env);
break;
}
case OperationType.OP_RENAME_ROLLUP: {
TableInfo info = (TableInfo) journal.getData();
env.replayRenameRollup(info);
env.getBinlogManager().addRollupRename(info, logId);
break;
}
case OperationType.OP_LOAD_START:
case OperationType.OP_LOAD_ETL:
case OperationType.OP_LOAD_LOADING:
case OperationType.OP_LOAD_QUORUM:
case OperationType.OP_LOAD_DONE:
case OperationType.OP_LOAD_CANCEL: {
LOG.warn("load job is deprecated");
break;
}
case OperationType.OP_EXPORT_CREATE: {
ExportJob job = (ExportJob) journal.getData();
ExportMgr exportMgr = env.getExportMgr();
job.cancelReplayedExportJob();
exportMgr.replayCreateExportJob(job);
break;
}
case OperationType.OP_EXPORT_UPDATE_STATE: {
ExportJobStateTransfer op = (ExportJobStateTransfer) journal.getData();
ExportMgr exportMgr = env.getExportMgr();
exportMgr.replayUpdateJobState(op);
break;
}
case OperationType.OP_FINISH_DELETE: {
DeleteInfo info = (DeleteInfo) journal.getData();
DeleteHandler deleteHandler = env.getDeleteHandler();
deleteHandler.replayDelete(info, env);
break;
}
case OperationType.OP_ADD_REPLICA: {
ReplicaPersistInfo info = (ReplicaPersistInfo) journal.getData();
env.replayAddReplica(info);
break;
}
case OperationType.OP_UPDATE_REPLICA: {
ReplicaPersistInfo info = (ReplicaPersistInfo) journal.getData();
env.replayUpdateReplica(info);
break;
}
case OperationType.OP_MODIFY_CLOUD_WARM_UP_JOB: {
CloudWarmUpJob cloudWarmUpJob = (CloudWarmUpJob) journal.getData();
((CloudEnv) env).getCacheHotspotMgr().replayCloudWarmUpJob(cloudWarmUpJob);
break;
}
case OperationType.OP_DELETE_REPLICA: {
ReplicaPersistInfo info = (ReplicaPersistInfo) journal.getData();
env.replayDeleteReplica(info);
break;
}
case OperationType.OP_ADD_BACKEND: {
Backend be = (Backend) journal.getData();
Env.getCurrentSystemInfo().replayAddBackend(be);
break;
}
case OperationType.OP_DROP_BACKEND: {
Backend be = (Backend) journal.getData();
Env.getCurrentSystemInfo().replayDropBackend(be);
break;
}
case OperationType.OP_MODIFY_BACKEND: {
Backend be = (Backend) journal.getData();
Env.getCurrentSystemInfo().replayModifyBackend(be);
break;
}
case OperationType.OP_BACKEND_STATE_CHANGE: {
Backend be = (Backend) journal.getData();
Env.getCurrentSystemInfo().updateBackendState(be);
break;
}
case OperationType.OP_ADD_FIRST_FRONTEND:
case OperationType.OP_ADD_FRONTEND: {
Frontend fe = (Frontend) journal.getData();
env.replayAddFrontend(fe);
break;
}
case OperationType.OP_MODIFY_FRONTEND: {
Frontend fe = (Frontend) journal.getData();
env.replayModifyFrontend(fe);
break;
}
case OperationType.OP_REMOVE_FRONTEND: {
Frontend fe = (Frontend) journal.getData();
env.replayDropFrontend(fe);
if (fe.getNodeName().equals(Env.getCurrentEnv().getNodeName())) {
LOG.warn("current fe {} is removed. will exit", fe);
System.exit(-1);
}
break;
}
case OperationType.OP_CREATE_USER: {
PrivInfo privInfo = (PrivInfo) journal.getData();
env.getAuth().replayCreateUser(privInfo);
break;
}
case OperationType.OP_NEW_DROP_USER: {
UserIdentity userIdent = (UserIdentity) journal.getData();
env.getAuth().replayDropUser(userIdent);
break;
}
case OperationType.OP_GRANT_PRIV: {
PrivInfo privInfo = (PrivInfo) journal.getData();
env.getAuth().replayGrant(privInfo);
break;
}
case OperationType.OP_REVOKE_PRIV: {
PrivInfo privInfo = (PrivInfo) journal.getData();
env.getAuth().replayRevoke(privInfo);
break;
}
case OperationType.OP_SET_PASSWORD: {
PrivInfo privInfo = (PrivInfo) journal.getData();
env.getAuth().replaySetPassword(privInfo);
break;
}
case OperationType.OP_SET_LDAP_PASSWORD: {
LdapInfo ldapInfo = (LdapInfo) journal.getData();
env.getAuth().replaySetLdapPassword(ldapInfo);
break;
}
case OperationType.OP_CREATE_ROLE: {
PrivInfo privInfo = (PrivInfo) journal.getData();
env.getAuth().replayCreateRole(privInfo);
break;
}
case OperationType.OP_ALTER_ROLE: {
PrivInfo privInfo = (PrivInfo) journal.getData();
env.getAuth().replayAlterRole(privInfo);
break;
}
case OperationType.OP_DROP_ROLE: {
PrivInfo privInfo = (PrivInfo) journal.getData();
env.getAuth().replayDropRole(privInfo);
break;
}
case OperationType.OP_UPDATE_USER_PROPERTY: {
UserPropertyInfo propertyInfo = (UserPropertyInfo) journal.getData();
env.getAuth().replayUpdateUserProperty(propertyInfo);
break;
}
case OperationType.OP_TIMESTAMP: {
Timestamp stamp = (Timestamp) journal.getData();
env.setSynchronizedTime(stamp.getTimestamp());
break;
}
case OperationType.OP_MASTER_INFO_CHANGE: {
MasterInfo info = (MasterInfo) journal.getData();
env.setMaster(info);
break;
}
case OperationType.OP_META_VERSION: {
String versionString = ((Text) journal.getData()).toString();
int version = Integer.parseInt(versionString);
if (version > FeConstants.meta_version) {
LOG.error("meta data version is out of date, image: {}. meta: {}."
+ "please update FeConstants.meta_version and restart.", version,
FeConstants.meta_version);
System.exit(-1);
}
MetaContext.get().setMetaVersion(version);
break;
}
case OperationType.OP_CREATE_CLUSTER: {
// Do nothing
break;
}
case OperationType.OP_ADD_BROKER: {
final BrokerMgr.ModifyBrokerInfo param = (BrokerMgr.ModifyBrokerInfo) journal.getData();
env.getBrokerMgr().replayAddBrokers(param.brokerName, param.brokerAddresses);
break;
}
case OperationType.OP_DROP_BROKER: {
final BrokerMgr.ModifyBrokerInfo param = (BrokerMgr.ModifyBrokerInfo) journal.getData();
env.getBrokerMgr().replayDropBrokers(param.brokerName, param.brokerAddresses);
break;
}
case OperationType.OP_DROP_ALL_BROKER: {
final String param = journal.getData().toString();
env.getBrokerMgr().replayDropAllBroker(param);
break;
}
case OperationType.OP_SET_LOAD_ERROR_HUB: {
// final LoadErrorHub.Param param = (LoadErrorHub.Param) journal.getData();
// ignore load error hub
break;
}
case OperationType.OP_UPSERT_TRANSACTION_STATE: {
final TransactionState state = (TransactionState) journal.getData();
Env.getCurrentGlobalTransactionMgr().replayUpsertTransactionState(state);
if (LOG.isDebugEnabled()) {
LOG.debug("logid: {}, opcode: {}, tid: {}", logId, opCode, state.getTransactionId());
}
// state.loadedTableIndexIds is updated after replay
if (state.getTransactionStatus() == TransactionStatus.VISIBLE) {
UpsertRecord upsertRecord = new UpsertRecord(logId, state);
Env.getCurrentEnv().getBinlogManager().addUpsertRecord(upsertRecord);
}
break;
}
case OperationType.OP_DELETE_TRANSACTION_STATE: {
final TransactionState state = (TransactionState) journal.getData();
Env.getCurrentGlobalTransactionMgr().replayDeleteTransactionState(state);
if (LOG.isDebugEnabled()) {
LOG.debug("opcode: {}, tid: {}", opCode, state.getTransactionId());
}
break;
}
case OperationType.OP_BATCH_REMOVE_TXNS: {
final BatchRemoveTransactionsOperation operation =
(BatchRemoveTransactionsOperation) journal.getData();
Env.getCurrentGlobalTransactionMgr().replayBatchRemoveTransactions(operation);
break;
}
case OperationType.OP_BATCH_REMOVE_TXNS_V2: {
final BatchRemoveTransactionsOperationV2 operation =
(BatchRemoveTransactionsOperationV2) journal.getData();
Env.getCurrentGlobalTransactionMgr().replayBatchRemoveTransactionV2(operation);
break;
}
case OperationType.OP_SET_TABLE_STATUS: {
final SetTableStatusOperationLog log = (SetTableStatusOperationLog) journal.getData();
env.replaySetTableStatus(log);
break;
}
case OperationType.OP_CREATE_REPOSITORY: {
Repository repository = (Repository) journal.getData();
env.getBackupHandler().getRepoMgr().addAndInitRepoIfNotExist(repository, true);
break;
}
case OperationType.OP_DROP_REPOSITORY: {
String repoName = ((Text) journal.getData()).toString();
env.getBackupHandler().getRepoMgr().removeRepo(repoName, true);
break;
}
case OperationType.OP_TRUNCATE_TABLE: {
TruncateTableInfo info = (TruncateTableInfo) journal.getData();
env.replayTruncateTable(info);
env.getBinlogManager().addTruncateTable(info, logId);
break;
}
case OperationType.OP_COLOCATE_ADD_TABLE: {
final ColocatePersistInfo info = (ColocatePersistInfo) journal.getData();
env.getColocateTableIndex().replayAddTableToGroup(info);
break;
}
case OperationType.OP_COLOCATE_REMOVE_TABLE: {
final ColocatePersistInfo info = (ColocatePersistInfo) journal.getData();
env.getColocateTableIndex().replayRemoveTable(info);
break;
}
case OperationType.OP_COLOCATE_BACKENDS_PER_BUCKETSEQ: {
final ColocatePersistInfo info = (ColocatePersistInfo) journal.getData();
env.getColocateTableIndex().replayAddBackendsPerBucketSeq(info);
break;
}
case OperationType.OP_COLOCATE_MARK_UNSTABLE: {
final ColocatePersistInfo info = (ColocatePersistInfo) journal.getData();
env.getColocateTableIndex().replayMarkGroupUnstable(info);
break;
}
case OperationType.OP_COLOCATE_MARK_STABLE: {
final ColocatePersistInfo info = (ColocatePersistInfo) journal.getData();
env.getColocateTableIndex().replayMarkGroupStable(info);
break;
}
case OperationType.OP_COLOCATE_MOD_REPLICA_ALLOC: {
final ColocatePersistInfo info = (ColocatePersistInfo) journal.getData();
env.getColocateTableIndex().replayModifyReplicaAlloc(info);
break;
}
case OperationType.OP_MODIFY_TABLE_COLOCATE: {
final TablePropertyInfo info = (TablePropertyInfo) journal.getData();
env.replayModifyTableColocate(info);
break;
}
case OperationType.OP_HEARTBEAT: {
final HbPackage hbPackage = (HbPackage) journal.getData();
Env.getCurrentHeartbeatMgr().replayHearbeat(hbPackage);
break;
}
case OperationType.OP_ADD_FUNCTION: {
final Function function = (Function) journal.getData();
Env.getCurrentEnv().replayCreateFunction(function);
break;
}
case OperationType.OP_DROP_FUNCTION: {
FunctionSearchDesc function = (FunctionSearchDesc) journal.getData();
Env.getCurrentEnv().replayDropFunction(function);
break;
}
case OperationType.OP_ADD_GLOBAL_FUNCTION: {
final Function function = (Function) journal.getData();
Env.getCurrentEnv().replayCreateGlobalFunction(function);
break;
}
case OperationType.OP_DROP_GLOBAL_FUNCTION: {
FunctionSearchDesc function = (FunctionSearchDesc) journal.getData();
Env.getCurrentEnv().replayDropGlobalFunction(function);
break;
}
case OperationType.OP_CREATE_ENCRYPTKEY: {
final EncryptKey encryptKey = (EncryptKey) journal.getData();
EncryptKeyHelper.replayCreateEncryptKey(encryptKey);
break;
}
case OperationType.OP_DROP_ENCRYPTKEY: {
EncryptKeySearchDesc encryptKeySearchDesc = (EncryptKeySearchDesc) journal.getData();
EncryptKeyHelper.replayDropEncryptKey(encryptKeySearchDesc);
break;
}
case OperationType.OP_BACKEND_TABLETS_INFO: {
BackendTabletsInfo backendTabletsInfo = (BackendTabletsInfo) journal.getData();
Env.getCurrentEnv().replayBackendTabletsInfo(backendTabletsInfo);
break;
}
case OperationType.OP_BACKEND_REPLICAS_INFO: {
BackendReplicasInfo backendReplicasInfo = (BackendReplicasInfo) journal.getData();
Env.getCurrentEnv().replayBackendReplicasInfo(backendReplicasInfo);
break;
}
case OperationType.OP_CREATE_ROUTINE_LOAD_JOB: {
RoutineLoadJob routineLoadJob = (RoutineLoadJob) journal.getData();
Env.getCurrentEnv().getRoutineLoadManager().replayCreateRoutineLoadJob(routineLoadJob);
break;
}
case OperationType.OP_CREATE_SCHEDULER_JOB: {
AbstractJob job = (AbstractJob) journal.getData();
Env.getCurrentEnv().getJobManager().replayCreateJob(job);
break;
}
case OperationType.OP_UPDATE_SCHEDULER_JOB: {
AbstractJob job = (AbstractJob) journal.getData();
Env.getCurrentEnv().getJobManager().replayUpdateJob(job);
break;
}
case OperationType.OP_DELETE_SCHEDULER_JOB: {
AbstractJob job = (AbstractJob) journal.getData();
Env.getCurrentEnv().getJobManager().replayDeleteJob(job);
break;
}
/*case OperationType.OP_CREATE_SCHEDULER_TASK: {
JobTask task = (JobTask) journal.getData();
Env.getCurrentEnv().getJobTaskManager().replayCreateTask(task);
break;
}
case OperationType.OP_DELETE_SCHEDULER_TASK: {
JobTask task = (JobTask) journal.getData();
Env.getCurrentEnv().getJobTaskManager().replayDeleteTask(task);
break;
}*/
case OperationType.OP_CHANGE_ROUTINE_LOAD_JOB: {
RoutineLoadOperation operation = (RoutineLoadOperation) journal.getData();
Env.getCurrentEnv().getRoutineLoadManager().replayChangeRoutineLoadJob(operation);
break;
}
case OperationType.OP_REMOVE_ROUTINE_LOAD_JOB: {
RoutineLoadOperation operation = (RoutineLoadOperation) journal.getData();
env.getRoutineLoadManager().replayRemoveOldRoutineLoad(operation);
break;
}
case OperationType.OP_CREATE_LOAD_JOB: {
org.apache.doris.load.loadv2.LoadJob loadJob =
(org.apache.doris.load.loadv2.LoadJob) journal.getData();
env.getLoadManager().replayCreateLoadJob(loadJob);
break;
}
case OperationType.OP_END_LOAD_JOB: {
LoadJobFinalOperation operation = (LoadJobFinalOperation) journal.getData();
env.getLoadManager().replayEndLoadJob(operation);
break;
}
case OperationType.OP_UPDATE_LOAD_JOB: {
LoadJobStateUpdateInfo info = (LoadJobStateUpdateInfo) journal.getData();
env.getLoadManager().replayUpdateLoadJobStateInfo(info);
break;
}
case OperationType.OP_CREATE_SYNC_JOB: {
SyncJob syncJob = (SyncJob) journal.getData();
env.getSyncJobManager().replayAddSyncJob(syncJob);
break;
}
case OperationType.OP_UPDATE_SYNC_JOB_STATE: {
SyncJob.SyncJobUpdateStateInfo info = (SyncJob.SyncJobUpdateStateInfo) journal.getData();
env.getSyncJobManager().replayUpdateSyncJobState(info);
break;
}
case OperationType.OP_FETCH_STREAM_LOAD_RECORD: {
FetchStreamLoadRecord fetchStreamLoadRecord = (FetchStreamLoadRecord) journal.getData();
env.getStreamLoadRecordMgr().replayFetchStreamLoadRecord(fetchStreamLoadRecord);
break;
}
case OperationType.OP_CREATE_RESOURCE: {
final Resource resource = (Resource) journal.getData();
env.getResourceMgr().replayCreateResource(resource);
break;
}
case OperationType.OP_DROP_RESOURCE: {
final DropResourceOperationLog operationLog = (DropResourceOperationLog) journal.getData();
env.getResourceMgr().replayDropResource(operationLog);
break;
}
case OperationType.OP_ALTER_RESOURCE: {
final Resource resource = (Resource) journal.getData();
env.getResourceMgr().replayAlterResource(resource);
break;
}
case OperationType.OP_CREATE_SMALL_FILE: {
SmallFile smallFile = (SmallFile) journal.getData();
env.getSmallFileMgr().replayCreateFile(smallFile);
break;
}
case OperationType.OP_DROP_SMALL_FILE: {
SmallFile smallFile = (SmallFile) journal.getData();
env.getSmallFileMgr().replayRemoveFile(smallFile);
break;
}
case OperationType.OP_ALTER_JOB_V2: {
AlterJobV2 alterJob = (AlterJobV2) journal.getData();
switch (alterJob.getType()) {
case ROLLUP:
env.getMaterializedViewHandler().replayAlterJobV2(alterJob);
break;
case SCHEMA_CHANGE:
env.getSchemaChangeHandler().replayAlterJobV2(alterJob);
break;
default:
break;
}
env.getBinlogManager().addAlterJobV2(alterJob, logId);
break;
}
case OperationType.OP_UPDATE_COOLDOWN_CONF:
CooldownConfList cooldownConfList = (CooldownConfList) journal.getData();
CooldownConfHandler.replayUpdateCooldownConf(cooldownConfList);
break;
case OperationType.OP_COOLDOWN_DELETE:
// noop
break;
case OperationType.OP_BATCH_ADD_ROLLUP: {
BatchAlterJobPersistInfo batchAlterJobV2 = (BatchAlterJobPersistInfo) journal.getData();
for (AlterJobV2 alterJobV2 : batchAlterJobV2.getAlterJobV2List()) {
env.getMaterializedViewHandler().replayAlterJobV2(alterJobV2);
}
break;
}
case OperationType.OP_MODIFY_DISTRIBUTION_TYPE: {
TableInfo tableInfo = (TableInfo) journal.getData();
env.replayConvertDistributionType(tableInfo);
env.getBinlogManager().addModifyDistributionType(tableInfo, logId);
break;
}
case OperationType.OP_DYNAMIC_PARTITION:
case OperationType.OP_MODIFY_TABLE_PROPERTIES:
case OperationType.OP_UPDATE_BINLOG_CONFIG:
case OperationType.OP_MODIFY_REPLICATION_NUM: {
ModifyTablePropertyOperationLog log = (ModifyTablePropertyOperationLog) journal.getData();
env.replayModifyTableProperty(opCode, log);
env.getBinlogManager().addModifyTableProperty(log, logId);
break;
}
case OperationType.OP_MODIFY_DISTRIBUTION_BUCKET_NUM: {
ModifyTableDefaultDistributionBucketNumOperationLog log =
(ModifyTableDefaultDistributionBucketNumOperationLog) journal.getData();
env.replayModifyTableDefaultDistributionBucketNum(log);
env.getBinlogManager().addModifyDistributionNum(log, logId);
break;
}
case OperationType.OP_REPLACE_TEMP_PARTITION: {
ReplacePartitionOperationLog replaceTempPartitionLog =
(ReplacePartitionOperationLog) journal.getData();
env.replayReplaceTempPartition(replaceTempPartitionLog);
env.getBinlogManager().addReplacePartitions(replaceTempPartitionLog, logId);
break;
}
case OperationType.OP_INSTALL_PLUGIN: {
PluginInfo pluginInfo = (PluginInfo) journal.getData();
env.replayInstallPlugin(pluginInfo);
break;
}
case OperationType.OP_UNINSTALL_PLUGIN: {
PluginInfo pluginInfo = (PluginInfo) journal.getData();
env.replayUninstallPlugin(pluginInfo);
break;
}
case OperationType.OP_SET_REPLICA_STATUS: {
SetReplicaStatusOperationLog log = (SetReplicaStatusOperationLog) journal.getData();
env.replaySetReplicaStatus(log);
break;
}
case OperationType.OP_SET_REPLICA_VERSION: {
SetReplicaVersionOperationLog log = (SetReplicaVersionOperationLog) journal.getData();
env.replaySetReplicaVersion(log);
break;
}
case OperationType.OP_REMOVE_ALTER_JOB_V2: {
RemoveAlterJobV2OperationLog log = (RemoveAlterJobV2OperationLog) journal.getData();
switch (log.getType()) {
case ROLLUP:
env.getMaterializedViewHandler().replayRemoveAlterJobV2(log);
break;
case SCHEMA_CHANGE:
env.getSchemaChangeHandler().replayRemoveAlterJobV2(log);
break;
default:
break;
}
break;
}
case OperationType.OP_MODIFY_COMMENT: {
ModifyCommentOperationLog operation = (ModifyCommentOperationLog) journal.getData();
env.getAlterInstance().replayModifyComment(operation);
env.getBinlogManager().addModifyComment(operation, logId);
break;
}
case OperationType.OP_SET_PARTITION_VERSION: {
SetPartitionVersionOperationLog log = (SetPartitionVersionOperationLog) journal.getData();
env.replaySetPartitionVersion(log);
break;
}
case OperationType.OP_ALTER_ROUTINE_LOAD_JOB: {
AlterRoutineLoadJobOperationLog log = (AlterRoutineLoadJobOperationLog) journal.getData();
env.getRoutineLoadManager().replayAlterRoutineLoadJob(log);
break;
}
case OperationType.OP_GLOBAL_VARIABLE_V2: {
GlobalVarPersistInfo info = (GlobalVarPersistInfo) journal.getData();
env.replayGlobalVariableV2(info);
break;
}
case OperationType.OP_REPLACE_TABLE: {
ReplaceTableOperationLog log = (ReplaceTableOperationLog) journal.getData();
env.getAlterInstance().replayReplaceTable(log);
env.getBinlogManager().addReplaceTable(log, logId);
break;
}
case OperationType.OP_CREATE_SQL_BLOCK_RULE: {
SqlBlockRule rule = (SqlBlockRule) journal.getData();
env.getSqlBlockRuleMgr().replayCreate(rule);
break;
}
case OperationType.OP_ALTER_SQL_BLOCK_RULE: {
SqlBlockRule rule = (SqlBlockRule) journal.getData();
env.getSqlBlockRuleMgr().replayAlter(rule);
break;
}
case OperationType.OP_DROP_SQL_BLOCK_RULE: {
DropSqlBlockRuleOperationLog log = (DropSqlBlockRuleOperationLog) journal.getData();
env.getSqlBlockRuleMgr().replayDrop(log.getRuleNames());
break;
}
case OperationType.OP_MODIFY_TABLE_ENGINE: {
ModifyTableEngineOperationLog log = (ModifyTableEngineOperationLog) journal.getData();
env.getAlterInstance().replayProcessModifyEngine(log);
break;
}
case OperationType.OP_CREATE_POLICY: {
Policy log = (Policy) journal.getData();
env.getPolicyMgr().replayCreate(log);
break;
}
case OperationType.OP_DROP_POLICY: {
DropPolicyLog log = (DropPolicyLog) journal.getData();
env.getPolicyMgr().replayDrop(log);
break;
}
case OperationType.OP_ALTER_STORAGE_POLICY: {
StoragePolicy log = (StoragePolicy) journal.getData();
env.getPolicyMgr().replayStoragePolicyAlter(log);
break;
}
case OperationType.OP_CREATE_CATALOG: {
CatalogLog log = (CatalogLog) journal.getData();
env.getCatalogMgr().replayCreateCatalog(log);
break;
}
case OperationType.OP_DROP_CATALOG: {
CatalogLog log = (CatalogLog) journal.getData();
env.getCatalogMgr().replayDropCatalog(log);
break;
}
case OperationType.OP_ALTER_CATALOG_NAME: {
CatalogLog log = (CatalogLog) journal.getData();
env.getCatalogMgr().replayAlterCatalogName(log);
break;
}
case OperationType.OP_ALTER_CATALOG_COMMENT: {
CatalogLog log = (CatalogLog) journal.getData();
env.getCatalogMgr().replayAlterCatalogComment(log);
break;
}
case OperationType.OP_ALTER_CATALOG_PROPS: {
CatalogLog log = (CatalogLog) journal.getData();
env.getCatalogMgr().replayAlterCatalogProps(log, null, true);
break;
}
case OperationType.OP_REFRESH_CATALOG: {
CatalogLog log = (CatalogLog) journal.getData();
env.getRefreshManager().replayRefreshCatalog(log);
break;
}
case OperationType.OP_MODIFY_TABLE_LIGHT_SCHEMA_CHANGE: {
final TableAddOrDropColumnsInfo info = (TableAddOrDropColumnsInfo) journal.getData();
env.getSchemaChangeHandler().replayModifyTableLightSchemaChange(info);
env.getBinlogManager().addModifyTableAddOrDropColumns(info, logId);
break;
}
case OperationType.OP_ALTER_LIGHT_SCHEMA_CHANGE: {
final AlterLightSchemaChangeInfo info = (AlterLightSchemaChangeInfo) journal.getData();
env.getSchemaChangeHandler().replayAlterLightSchChange(info);
break;
}
case OperationType.OP_CLEAN_QUERY_STATS: {
final CleanQueryStatsInfo info = (CleanQueryStatsInfo) journal.getData();
env.getQueryStats().clear(info);
break;
}
case OperationType.OP_MODIFY_TABLE_ADD_OR_DROP_INVERTED_INDICES: {
final TableAddOrDropInvertedIndicesInfo info =
(TableAddOrDropInvertedIndicesInfo) journal.getData();
env.getSchemaChangeHandler().replayModifyTableAddOrDropInvertedIndices(info);
env.getBinlogManager().addModifyTableAddOrDropInvertedIndices(info, logId);
break;
}
case OperationType.OP_INVERTED_INDEX_JOB: {
IndexChangeJob indexChangeJob = (IndexChangeJob) journal.getData();
env.getSchemaChangeHandler().replayIndexChangeJob(indexChangeJob);
env.getBinlogManager().addIndexChangeJob(indexChangeJob, logId);
break;
}
case OperationType.OP_CLEAN_LABEL: {
final CleanLabelOperationLog log = (CleanLabelOperationLog) journal.getData();
env.getLoadManager().replayCleanLabel(log);
break;
}
case OperationType.OP_CREATE_MTMV_JOB:
case OperationType.OP_CHANGE_MTMV_JOB:
case OperationType.OP_DROP_MTMV_JOB:
case OperationType.OP_CREATE_MTMV_TASK:
case OperationType.OP_CHANGE_MTMV_TASK:
case OperationType.OP_DROP_MTMV_TASK:
case OperationType.OP_ALTER_MTMV_STMT: {
break;
}
case OperationType.OP_ADD_CONSTRAINT: {
final AlterConstraintLog log = (AlterConstraintLog) journal.getData();
try {
log.getTableIf().replayAddConstraint(log.getConstraint());
} catch (Exception e) {
LOG.error("Failed to replay add constraint", e);
}
break;
}
case OperationType.OP_DROP_CONSTRAINT: {
final AlterConstraintLog log = (AlterConstraintLog) journal.getData();
try {
log.getTableIf().replayDropConstraint(log.getConstraint().getName());
} catch (Exception e) {
LOG.error("Failed to replay drop constraint", e);
}
break;
}
case OperationType.OP_ALTER_USER: {
final AlterUserOperationLog log = (AlterUserOperationLog) journal.getData();
env.getAuth().replayAlterUser(log);
break;
}
case OperationType.OP_INIT_CATALOG:
case OperationType.OP_INIT_CATALOG_COMP: {
final InitCatalogLog log = (InitCatalogLog) journal.getData();
env.getCatalogMgr().replayInitCatalog(log);
break;
}
case OperationType.OP_REFRESH_EXTERNAL_DB: {
final ExternalObjectLog log = (ExternalObjectLog) journal.getData();
env.getRefreshManager().replayRefreshDb(log);
break;
}
case OperationType.OP_INIT_EXTERNAL_DB: {
final InitDatabaseLog log = (InitDatabaseLog) journal.getData();
env.getCatalogMgr().replayInitExternalDb(log);
break;
}
case OperationType.OP_REFRESH_EXTERNAL_TABLE: {
final ExternalObjectLog log = (ExternalObjectLog) journal.getData();
env.getRefreshManager().replayRefreshTable(log);
break;
}
case OperationType.OP_DROP_EXTERNAL_TABLE: {
break;
}
case OperationType.OP_CREATE_EXTERNAL_TABLE: {
break;
}
case OperationType.OP_DROP_EXTERNAL_DB: {
break;
}
case OperationType.OP_CREATE_EXTERNAL_DB: {
break;
}
case OperationType.OP_ADD_EXTERNAL_PARTITIONS: {
break;
}
case OperationType.OP_DROP_EXTERNAL_PARTITIONS: {
break;
}
case OperationType.OP_REFRESH_EXTERNAL_PARTITIONS: {
break;
}
case OperationType.OP_CREATE_WORKLOAD_GROUP: {
final WorkloadGroup workloadGroup = (WorkloadGroup) journal.getData();
env.getWorkloadGroupMgr().replayCreateWorkloadGroup(workloadGroup);
break;
}
case OperationType.OP_DROP_WORKLOAD_GROUP: {
final DropWorkloadGroupOperationLog operationLog =
(DropWorkloadGroupOperationLog) journal.getData();
env.getWorkloadGroupMgr().replayDropWorkloadGroup(operationLog);
break;
}
case OperationType.OP_ALTER_WORKLOAD_GROUP: {
final WorkloadGroup resource = (WorkloadGroup) journal.getData();
env.getWorkloadGroupMgr().replayAlterWorkloadGroup(resource);
break;
}
case OperationType.OP_CREATE_WORKLOAD_SCHED_POLICY: {
final WorkloadSchedPolicy policy = (WorkloadSchedPolicy) journal.getData();
env.getWorkloadSchedPolicyMgr().replayCreateWorkloadSchedPolicy(policy);
break;
}
case OperationType.OP_ALTER_WORKLOAD_SCHED_POLICY: {
final WorkloadSchedPolicy policy = (WorkloadSchedPolicy) journal.getData();
env.getWorkloadSchedPolicyMgr().replayAlterWorkloadSchedPolicy(policy);
break;
}
case OperationType.OP_DROP_WORKLOAD_SCHED_POLICY: {
final DropWorkloadSchedPolicyOperatorLog dropLog
= (DropWorkloadSchedPolicyOperatorLog) journal.getData();
env.getWorkloadSchedPolicyMgr().replayDropWorkloadSchedPolicy(dropLog.getId());
break;
}
case OperationType.OP_INIT_EXTERNAL_TABLE: {
// Do nothing.
break;
}
case OperationType.OP_CREATE_ANALYSIS_JOB: {
if (journal.getData() instanceof AnalysisJobInfo) {
// For rollback compatible.
break;
}
AnalysisInfo info = (AnalysisInfo) journal.getData();
if (AnalysisManager.needAbandon(info)) {
break;
}
env.getAnalysisManager().replayCreateAnalysisJob(info);
break;
}
case OperationType.OP_CREATE_ANALYSIS_TASK: {
if (journal.getData() instanceof AnalysisTaskInfo) {
// For rollback compatible.
break;
}
AnalysisInfo info = (AnalysisInfo) journal.getData();
if (AnalysisManager.needAbandon(info)) {
break;
}
env.getAnalysisManager().replayCreateAnalysisTask(info);
break;
}
case OperationType.OP_DELETE_ANALYSIS_JOB: {
env.getAnalysisManager().replayDeleteAnalysisJob((AnalyzeDeletionLog) journal.getData());
break;
}
case OperationType.OP_DELETE_ANALYSIS_TASK: {
env.getAnalysisManager().replayDeleteAnalysisTask((AnalyzeDeletionLog) journal.getData());
break;
}
case OperationType.OP_ADD_PLSQL_STORED_PROCEDURE: {
env.getPlsqlManager().replayAddPlsqlStoredProcedure((PlsqlStoredProcedure) journal.getData());
break;
}
case OperationType.OP_DROP_PLSQL_STORED_PROCEDURE: {
env.getPlsqlManager().replayDropPlsqlStoredProcedure((PlsqlProcedureKey) journal.getData());
break;
}
case OperationType.OP_ADD_PLSQL_PACKAGE: {
env.getPlsqlManager().replayAddPlsqlPackage((PlsqlPackage) journal.getData());
break;
}
case OperationType.OP_DROP_PLSQL_PACKAGE: {
env.getPlsqlManager().replayDropPlsqlPackage((PlsqlProcedureKey) journal.getData());
break;
}
case OperationType.OP_ALTER_DATABASE_PROPERTY: {
AlterDatabasePropertyInfo alterDatabasePropertyInfo = (AlterDatabasePropertyInfo) journal.getData();
LOG.info("replay alter database property: {}", alterDatabasePropertyInfo);
env.replayAlterDatabaseProperty(alterDatabasePropertyInfo.getDbName(),
alterDatabasePropertyInfo.getProperties());
env.getBinlogManager().addAlterDatabaseProperty(alterDatabasePropertyInfo, logId);
break;
}
case OperationType.OP_GC_BINLOG: {
BinlogGcInfo binlogGcInfo = (BinlogGcInfo) journal.getData();
LOG.info("replay gc binlog: {}", binlogGcInfo);
env.replayGcBinlog(binlogGcInfo);
break;
}
case OperationType.OP_BARRIER: {
BarrierLog log = (BarrierLog) journal.getData();
env.getBinlogManager().addBarrierLog(log, logId);
break;
}
case OperationType.OP_UPDATE_AUTO_INCREMENT_ID: {
env.replayAutoIncrementIdUpdateLog((AutoIncrementIdUpdateLog) journal.getData());
break;
}
case OperationType.OP_UPDATE_TABLE_STATS: {
env.getAnalysisManager().replayUpdateTableStatsStatus((TableStatsMeta) journal.getData());
break;
}
case OperationType.OP_PERSIST_AUTO_JOB: {
// Do nothing
break;
}
case OperationType.OP_DELETE_TABLE_STATS: {
long tableId = ((TableStatsDeletionLog) journal.getData()).id;
LOG.info("replay delete table stat tableId: {}", tableId);
Env.getCurrentEnv().getAnalysisManager().removeTableStats(tableId);
break;
}
case OperationType.OP_ALTER_MTMV: {
final AlterMTMV alterMtmv = (AlterMTMV) journal.getData();
env.getAlterInstance().processAlterMTMV(alterMtmv, true);
break;
}
case OperationType.OP_INSERT_OVERWRITE: {
final InsertOverwriteLog insertOverwriteLog = (InsertOverwriteLog) journal.getData();
env.getInsertOverwriteManager().replayInsertOverwriteLog(insertOverwriteLog);
break;
}
case OperationType.OP_ALTER_REPOSITORY: {
Repository repository = (Repository) journal.getData();
env.getBackupHandler().getRepoMgr().alterRepo(repository, true);
break;
}
case OperationType.OP_ADD_META_ID_MAPPINGS: {
env.getExternalMetaIdMgr().replayMetaIdMappingsLog((MetaIdMappingsLog) journal.getData());
break;
}
case OperationType.OP_LOG_UPDATE_ROWS: {
env.getAnalysisManager().replayUpdateRowsRecord((UpdateRowsEvent) journal.getData());
break;
}
case OperationType.OP_LOG_NEW_PARTITION_LOADED: {
env.getAnalysisManager().replayNewPartitionLoadedEvent((NewPartitionLoadedEvent) journal.getData());
break;
}
case OperationType.OP_LOG_ALTER_COLUMN_STATS: {
// TODO: implement this while statistics finished related work.
break;
}
case OperationType.OP_UPDATE_CLOUD_REPLICA: {
UpdateCloudReplicaInfo info = (UpdateCloudReplicaInfo) journal.getData();
((CloudEnv) env).replayUpdateCloudReplica(info);
break;
}
case OperationType.OP_CREATE_DICTIONARY: {
CreateDictionaryPersistInfo info = (CreateDictionaryPersistInfo) journal.getData();
env.getDictionaryManager().replayCreateDictionary(info);
break;
}
case OperationType.OP_DROP_DICTIONARY: {
DropDictionaryPersistInfo info = (DropDictionaryPersistInfo) journal.getData();
env.getDictionaryManager().replayDropDictionary(info);
break;
}
case OperationType.OP_DICTIONARY_INC_VERSION: {
DictionaryIncreaseVersionInfo info = (DictionaryIncreaseVersionInfo) journal.getData();
env.getDictionaryManager().replayIncreaseVersion(info);
break;
}
case OperationType.OP_DICTIONARY_DEC_VERSION: {
DictionaryDecreaseVersionInfo info = (DictionaryDecreaseVersionInfo) journal.getData();
env.getDictionaryManager().replayDecreaseVersion(info);
break;
}
default: {
IOException e = new IOException();
LOG.error("UNKNOWN Operation Type {}, log id: {}", opCode, logId, e);
throw e;
}
}
} catch (MetaNotFoundException e) {
/*
* In the following cases, doris may record metadata modification information
* for a table that no longer exists.
* 1. Thread 1: get TableA object
* 2. Thread 2: lock db and drop table and record edit log of the dropped TableA
* 3. Thread 1: lock table, modify table and record edit log of the modified
* TableA
* **The modified edit log is after the dropped edit log**
* Because the table has been dropped, the olapTable in here is null when the
* modified edit log is replayed.
* So in this case, we will ignore the edit log of the modified table after the
* table is dropped.
* This could make the meta inconsistent, for example, an edit log on a dropped
* table is ignored, but
* this table is restored later, so there may be an inconsistent situation
* between master and followers. We
* log a warning here to debug when happens. This could happen to other meta
* like DB.
*/
LOG.warn("[INCONSISTENT META] replay log {} failed, journal {}: {}", logId, journal, e.getMessage(), e);
} catch (Exception e) {
LOG.error("replay Operation Type {}, log id: {}", opCode, logId, e);
System.exit(-1);
}
}