in hbase-hbck2/src/main/java/org/apache/hbase/hbck1/HBaseFsck.java [2704:2841]
private void checkRegionConsistency(final String key, final HbckInfo hbi)
throws IOException, KeeperException, InterruptedException {
if (hbi.isSkipChecks()) {
return;
}
boolean inMeta = hbi.metaEntry != null;
// In case not checking HDFS, assume the region is on HDFS
boolean inHdfs = !shouldCheckHdfs() || hbi.getHdfsRegionDir() != null;
boolean hasMetaAssignment = inMeta && hbi.metaEntry.regionServer != null;
boolean isDeployed = !hbi.deployedOn.isEmpty();
boolean isMultiplyDeployed = hbi.deployedOn.size() > 1;
boolean deploymentMatchesMeta = hasMetaAssignment && isDeployed && !isMultiplyDeployed &&
hbi.metaEntry.regionServer.equals(hbi.deployedOn.get(0));
boolean splitParent = inMeta && hbi.metaEntry.isSplit() && hbi.metaEntry.isOffline();
boolean shouldBeDeployed = inMeta && !isTableDisabled(hbi.metaEntry.getTable());
boolean recentlyModified = inHdfs && hbi.getModTime() + timelag > System.currentTimeMillis();
// ========== First the healthy cases =============
if (hbi.containsOnlyHdfsEdits()) {
return;
}
if (inMeta && inHdfs && isDeployed && deploymentMatchesMeta && shouldBeDeployed) {
return;
} else if (inMeta && inHdfs && !shouldBeDeployed && !isDeployed) {
LOG.info(hbi + " is in META and in a disabled table that is not deployed");
return;
} else if (recentlyModified) {
LOG.warn(hbi + " was recently modified -- skipping");
return;
// ========== Cases where the region is not in hbase:meta =============
} else if (!inMeta && !inHdfs && !isDeployed) {
// We shouldn't have record of this region at all then!
assert false : "Entry for region with no data";
} else if (!inMeta && !inHdfs && isDeployed) {
errors.reportError(ErrorReporter.ERROR_CODE.NOT_IN_META_HDFS, hbi + ", key=" + key +
", not on HDFS or hbase:meta but deployed on " + Joiner.on(", ").join(hbi.deployedOn));
if (shouldFixAssignments()) {
undeployRegions(hbi);
}
} else if (!inMeta && inHdfs && !isDeployed) {
if (hbi.isMerged()) {
// Already been merged, remaining hdfs file will be cleaned by CatalogJanitor later
hbi.setSkipChecks(true);
LOG.info(hbi + " was recently merged; file(s) will be cleaned by CatalogJanitor later");
return;
}
errors.reportError(ErrorReporter.ERROR_CODE.NOT_IN_META_OR_DEPLOYED,
hbi +" on HDFS but not in hbase:meta nor deployed on any region server");
// restore region consistency of an adopted orphan
if (shouldFixMeta()) {
if (!hbi.isHdfsRegioninfoPresent()) {
LOG.error(hbi.getHdfsHRI() + " could have been repaired in table integrity repair " +
"phase if -fixHdfsOrphans was used.");
return;
}
if (deletedLeftoverSplitRegion(hbi)) {
return;
}
fixHolesAndTryAssignmentRepair(hbi, "Trying to reassign region...");
}
} else if (!inMeta && inHdfs && isDeployed) {
errors.reportError(ErrorReporter.ERROR_CODE.NOT_IN_META, hbi + " not in META, but " +
"deployed on " + Joiner.on(", ").join(hbi.deployedOn));
debugLsr(hbi.getHdfsRegionDir());
if (hbi.getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) {
// For replicas, this means that we should undeploy the region (we would have gone over the
// primaries and fixed meta holes in first phase under checkAndFixConsistency; we shouldn't
// get the condition !inMeta at this stage unless unwanted replica)
if (shouldFixAssignments()) {
undeployRegionsForHbi(hbi);
}
}
if (shouldFixMeta() && hbi.getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID) {
if (!hbi.isHdfsRegioninfoPresent()) {
LOG.error("This should have been repaired in table integrity repair phase");
return;
}
fixHolesAndTryAssignmentRepair(hbi, "Trying to fix unassigned region...");
}
// ========== Cases where the region is in hbase:meta =============
} else if (inMeta && inHdfs && !isDeployed && splitParent) {
if (skipIfErrorOrTransient(hbi)) {
return;
}
} else if (inMeta && !inHdfs && !isDeployed) {
errors.reportError(ErrorReporter.ERROR_CODE.NOT_IN_HDFS_OR_DEPLOYED,
hbi + " found in META, but not in HDFS or deployed on any region server.");
if (shouldFixMeta()) {
deleteMetaRegion(hbi);
}
} else if (inMeta && !inHdfs && isDeployed) {
errors.reportError(ErrorReporter.ERROR_CODE.NOT_IN_HDFS, hbi + " found in META, but " +
"not in HDFS, and deployed on " + Joiner.on(", ").join(hbi.deployedOn));
// We treat HDFS as ground truth. Any information in meta is transient
// and equivalent data can be regenerated. So, lets unassign and remove
// these problems from META.
if (shouldFixAssignments()) {
errors.print("Trying to fix unassigned region...");
undeployRegions(hbi);
}
if (shouldFixMeta()) {
// wait for it to complete
deleteMetaRegion(hbi);
}
} else if (inMeta && inHdfs && !isDeployed && shouldBeDeployed) {
errors.reportError(ErrorReporter.ERROR_CODE.NOT_DEPLOYED, hbi + " not deployed.");
tryAssignmentRepair(hbi, "Trying to fix unassigned region...");
} else if (inMeta && inHdfs && isDeployed && !shouldBeDeployed) {
errors.reportError(ErrorReporter.ERROR_CODE.SHOULD_NOT_BE_DEPLOYED, hbi + " should not be " +
"deployed according to META, but deployed on " + Joiner.on(", ").join(hbi.deployedOn));
if (shouldFixAssignments()) {
setShouldRunAndFixMultiAssignment(connection, hbi, "Trying to close " + hbi);
}
} else if (inMeta && inHdfs && isMultiplyDeployed) {
errors.reportError(ErrorReporter.ERROR_CODE.MULTI_DEPLOYED, hbi + " listed in hbase:meta " +
"on " + hbi.metaEntry.regionServer + " but multiply assigned to " +
Joiner.on(", ").join(hbi.deployedOn));
// If we are trying to fix the errors
if (shouldFixAssignments()) {
setShouldRunAndFixMultiAssignment(connection, hbi, "Trying to fix assignment error...");
}
} else if (inMeta && inHdfs && isDeployed && !deploymentMatchesMeta) {
errors.reportError(ErrorReporter.ERROR_CODE.SERVER_DOES_NOT_MATCH_META, hbi + " listed in " +
"hbase:meta on " + hbi.metaEntry.regionServer + " but found on " + hbi.deployedOn.get(0));
// If we are trying to fix the errors
if (shouldFixAssignments()) {
HBaseFsckRepair.waitUntilAssigned(admin, hbi.getHdfsHRI());
errors.print("Trying to fix assignment error...");
}
} else {
errors.reportError(ErrorReporter.ERROR_CODE.UNKNOWN, hbi + " is in an unforeseen state:" +
" inMeta=" + inMeta + " inHdfs=" + inHdfs + " isDeployed=" + isDeployed +
" isMultiplyDeployed=" + isMultiplyDeployed +
" deploymentMatchesMeta=" + deploymentMatchesMeta +
" shouldBeDeployed=" + shouldBeDeployed);
}
}