in standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java [545:794]
public Partition alterPartition(RawStore msdb, Warehouse wh, String catName, String dbname,
String name, List<String> part_vals, final Partition new_part,
EnvironmentContext environmentContext, IHMSHandler handler, String validWriteIds)
throws InvalidOperationException, InvalidObjectException, AlreadyExistsException, MetaException, NoSuchObjectException {
boolean success = false;
Partition oldPart;
List<TransactionalMetaStoreEventListener> transactionalListeners = null;
if (handler != null) {
transactionalListeners = handler.getTransactionalListeners();
}
// Set DDL time to now if not specified
if (new_part.getParameters() == null ||
new_part.getParameters().get(hive_metastoreConstants.DDL_TIME) == null ||
Integer.parseInt(new_part.getParameters().get(hive_metastoreConstants.DDL_TIME)) == 0) {
new_part.putToParameters(hive_metastoreConstants.DDL_TIME, Long.toString(System
.currentTimeMillis() / 1000));
}
//alter partition
if (part_vals == null || part_vals.size() == 0) {
try {
msdb.openTransaction();
Table tbl = msdb.getTable(catName, dbname, name, null);
if (tbl == null) {
throw new InvalidObjectException(
"Unable to alter partition because table or database does not exist.");
}
oldPart = msdb.getPartition(catName, dbname, name, new_part.getValues());
if (MetaStoreServerUtils.requireCalStats(oldPart, new_part, tbl, environmentContext)) {
// if stats are same, no need to update
if (MetaStoreServerUtils.isFastStatsSame(oldPart, new_part)) {
MetaStoreServerUtils.updateBasicState(environmentContext, new_part.getParameters());
} else {
MetaStoreServerUtils.updatePartitionStatsFast(
new_part, tbl, wh, false, true, environmentContext, false);
}
}
// PartitionView does not have SD. We do not need update its column stats
if (oldPart.getSd() != null) {
updateOrGetPartitionColumnStats(msdb, catName, dbname, name, new_part.getValues(),
oldPart.getSd().getCols(), tbl, new_part, null, null);
}
Deadline.checkTimeout();
msdb.alterPartition(
catName, dbname, name, new_part.getValues(), new_part, validWriteIds);
if (transactionalListeners != null && !transactionalListeners.isEmpty()) {
MetaStoreListenerNotifier.notifyEvent(transactionalListeners,
EventMessage.EventType.ALTER_PARTITION,
new AlterPartitionEvent(oldPart, new_part, tbl, false,
true, new_part.getWriteId(), handler),
environmentContext);
}
success = msdb.commitTransaction();
} catch (InvalidObjectException e) {
LOG.warn("Alter failed", e);
throw new InvalidOperationException("alter is not possible: " + e.getMessage());
} catch (NoSuchObjectException e) {
//old partition does not exist
throw new InvalidOperationException("alter is not possible: " + e.getMessage());
} finally {
if(!success) {
msdb.rollbackTransaction();
}
}
return oldPart;
}
//rename partition
String oldPartLoc;
String newPartLoc;
Path srcPath = null;
Path destPath = null;
FileSystem srcFs;
FileSystem destFs = null;
boolean dataWasMoved = false;
Database db;
try {
msdb.openTransaction();
Table tbl = msdb.getTable(catName, dbname, name, null);
if (tbl == null) {
throw new InvalidObjectException(
"Unable to alter partition because table or database does not exist.");
}
MTable mTable = msdb.ensureGetMTable(catName, dbname, name);
try {
oldPart = msdb.getPartition(catName, dbname, name, part_vals);
} catch (NoSuchObjectException e) {
// this means there is no existing partition
throw new InvalidObjectException(
"Unable to rename partition because old partition does not exist");
}
Partition check_part;
try {
check_part = msdb.getPartition(catName, dbname, name, new_part.getValues());
} catch(NoSuchObjectException e) {
// this means there is no existing partition
check_part = null;
}
if (check_part != null) {
throw new AlreadyExistsException("Partition already exists:" + dbname + "." + name + "." +
new_part.getValues());
}
// when renaming a partition, we should update
// 1) partition SD Location
// 2) partition column stats if there are any because of part_name field in HMS table PART_COL_STATS
// 3) rename the partition directory if it is not an external table
boolean shouldMoveData = !(MetaStoreUtils.isExternalTable(tbl) &&
!MetaStoreUtils.isPropertyTrue(tbl.getParameters(), "TRANSLATED_TO_EXTERNAL"));
if (shouldMoveData) {
// TODO: refactor this into a separate method after master merge, this one is too big.
try {
db = msdb.getDatabase(catName, dbname);
// if tbl location is available use it
// else derive the tbl location from database location
destPath = wh.getPartitionPath(db, tbl, new_part.getValues());
destPath = constructRenamedPath(destPath, new Path(new_part.getSd().getLocation()));
} catch (NoSuchObjectException e) {
LOG.debug("Didn't find object in metastore ", e);
throw new InvalidOperationException(
"Unable to change partition or table. Database " + dbname + " does not exist"
+ " Check metastore logs for detailed stack." + e.getMessage());
}
if (destPath != null) {
newPartLoc = destPath.toString();
oldPartLoc = oldPart.getSd().getLocation();
LOG.info("srcPath:" + oldPartLoc);
LOG.info("descPath:" + newPartLoc);
srcPath = new Path(oldPartLoc);
srcFs = wh.getFs(srcPath);
destFs = wh.getFs(destPath);
// check that src and dest are on the same file system
if (!FileUtils.equalsFileSystem(srcFs, destFs)) {
throw new InvalidOperationException("New table location " + destPath
+ " is on a different file system than the old location "
+ srcPath + ". This operation is not supported.");
}
try {
if (srcFs.exists(srcPath)) {
if (newPartLoc.compareTo(oldPartLoc) != 0 && destFs.exists(destPath)) {
throw new InvalidOperationException("New location for this table "
+ tbl.getDbName() + "." + tbl.getTableName()
+ " already exists : " + destPath);
}
//if destPath's parent path doesn't exist, we should mkdir it
Path destParentPath = destPath.getParent();
if (!wh.mkdirs(destParentPath)) {
throw new MetaException("Unable to create path " + destParentPath);
}
boolean clonePart = Optional.ofNullable(environmentContext)
.map(EnvironmentContext::getProperties)
.map(prop -> prop.get(RENAME_PARTITION_MAKE_COPY))
.map(Boolean::parseBoolean)
.orElse(false);
long writeId = new_part.getWriteId();
if (writeId > 0 && clonePart) {
LOG.debug("Making a copy of the partition directory: {} under a new location: {}", srcPath, destPath);
if (!wh.copyDir(srcPath, destPath, ReplChangeManager.shouldEnableCm(db, tbl))) {
LOG.error("Copy failed for source: " + srcPath + " to destination: " + destPath);
throw new IOException("File copy failed.");
}
addTruncateBaseFile(srcPath, writeId, conf, DataFormat.DROPPED);
} else {
//rename the data directory
wh.renameDir(srcPath, destPath, ReplChangeManager.shouldEnableCm(db, tbl));
}
LOG.info("Partition directory rename from " + srcPath + " to " + destPath + " done.");
dataWasMoved = true;
}
} catch (IOException e) {
LOG.error("Cannot rename partition directory from " + srcPath + " to " + destPath, e);
throw new InvalidOperationException("Unable to access src or dest location for partition "
+ tbl.getDbName() + "." + tbl.getTableName() + " " + new_part.getValues());
} catch (MetaException me) {
LOG.error("Cannot rename partition directory from " + srcPath + " to " + destPath, me);
throw me;
}
new_part.getSd().setLocation(newPartLoc);
}
} else {
new_part.getSd().setLocation(oldPart.getSd().getLocation());
}
if (MetaStoreServerUtils.requireCalStats(oldPart, new_part, tbl, environmentContext)) {
MetaStoreServerUtils.updatePartitionStatsFast(
new_part, tbl, wh, false, true, environmentContext, false);
}
String newPartName = Warehouse.makePartName(tbl.getPartitionKeys(), new_part.getValues());
List<ColumnStatistics> multiColumnStats = updateOrGetPartitionColumnStats(msdb, catName, dbname, name, oldPart.getValues(),
oldPart.getSd().getCols(), tbl, new_part, null, null);
msdb.alterPartition(catName, dbname, name, part_vals, new_part, validWriteIds);
if (!multiColumnStats.isEmpty()) {
for (ColumnStatistics cs : multiColumnStats) {
cs.getStatsDesc().setPartName(newPartName);
try {
msdb.updatePartitionColumnStatistics(tbl, mTable, cs, new_part.getValues(),
validWriteIds, new_part.getWriteId());
} catch (InvalidInputException iie) {
throw new InvalidOperationException("Unable to update partition stats in table rename." + iie);
} catch (NoSuchObjectException nsoe) {
// It is ok, ignore
}
}
}
if (transactionalListeners != null && !transactionalListeners.isEmpty()) {
MetaStoreListenerNotifier.notifyEvent(transactionalListeners,
EventMessage.EventType.ALTER_PARTITION,
new AlterPartitionEvent(oldPart, new_part, tbl, false,
true, new_part.getWriteId(), handler),
environmentContext);
}
success = msdb.commitTransaction();
} finally {
if (!success) {
LOG.error("Failed to rename a partition. Rollback transaction");
msdb.rollbackTransaction();
if (dataWasMoved) {
LOG.error("Revert the data move in renaming a partition.");
try {
if (destFs.exists(destPath)) {
wh.renameDir(destPath, srcPath, false);
}
} catch (MetaException me) {
LOG.error("Failed to restore partition data from " + destPath + " to " + srcPath
+ " in alter partition failure. Manual restore is needed.");
} catch (IOException ioe) {
LOG.error("Failed to restore partition data from " + destPath + " to " + srcPath
+ " in alter partition failure. Manual restore is needed.");
}
}
}
}
return oldPart;
}