in plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtMigrateCommandWrapper.java [106:399]
public Answer execute(final MigrateCommand command, final LibvirtComputingResource libvirtComputingResource) {
final String vmName = command.getVmName();
final Map<String, Boolean> vlanToPersistenceMap = command.getVlanToPersistenceMap();
final String destinationUri = createMigrationURI(command.getDestinationIp(), libvirtComputingResource);
final List<MigrateDiskInfo> migrateDiskInfoList = command.getMigrateDiskInfoList();
if (logger.isDebugEnabled()) {
logger.debug(String.format("Trying to migrate VM [%s] to destination host: [%s].", vmName, destinationUri));
}
String result = null;
Command.State commandState = null;
List<InterfaceDef> ifaces = null;
List<DiskDef> disks;
Domain dm = null;
Connect dconn = null;
Domain destDomain = null;
Connect conn = null;
String xmlDesc = null;
List<Ternary<String, Boolean, String>> vmsnapshots = null;
MigrationCancelHook cancelHook = null;
try {
final LibvirtUtilitiesHelper libvirtUtilitiesHelper = libvirtComputingResource.getLibvirtUtilitiesHelper();
conn = libvirtUtilitiesHelper.getConnectionByVmName(vmName);
ifaces = libvirtComputingResource.getInterfaces(conn, vmName);
disks = libvirtComputingResource.getDisks(conn, vmName);
if (logger.isDebugEnabled()) {
logger.debug(String.format("Found domain with name [%s]. Starting VM migration to host [%s].", vmName, destinationUri));
}
VirtualMachineTO to = command.getVirtualMachine();
dm = conn.domainLookupByName(vmName);
/*
We replace the private IP address with the address of the destination host.
This is because the VNC listens on the private IP address of the hypervisor,
but that address is of course different on the target host.
MigrateCommand.getDestinationIp() returns the private IP address of the target
hypervisor. So it's safe to use.
The Domain.migrate method from libvirt supports passing a different XML
description for the instance to be used on the target host.
This is supported by libvirt-java from version 0.50.0
CVE-2015-3252: Get XML with sensitive information suitable for migration by using
VIR_DOMAIN_XML_MIGRATABLE flag (value = 8)
https://libvirt.org/html/libvirt-libvirt-domain.html#virDomainXMLFlags
Use VIR_DOMAIN_XML_SECURE (value = 1) prior to v1.0.0.
*/
final int xmlFlag = conn.getLibVirVersion() >= 1000000 ? 8 : 1; // 1000000 equals v1.0.0
final String target = command.getDestinationIp();
xmlDesc = dm.getXMLDesc(xmlFlag);
if (logger.isDebugEnabled()) {
logger.debug(String.format("VM [%s] with XML configuration [%s] will be migrated to host [%s].", vmName, xmlDesc, target));
}
// Limit the VNC password in case the length is greater than 8 characters
// Since libvirt version 8 VNC passwords are limited to 8 characters
String vncPassword = org.apache.commons.lang3.StringUtils.truncate(to.getVncPassword(), 8);
xmlDesc = replaceIpForVNCInDescFileAndNormalizePassword(xmlDesc, target, vncPassword, vmName);
// Replace Config Drive ISO path
String oldIsoVolumePath = getOldVolumePath(disks, vmName);
String newIsoVolumePath = getNewVolumePathIfDatastoreHasChanged(libvirtComputingResource, conn, to);
if (newIsoVolumePath != null && !newIsoVolumePath.equals(oldIsoVolumePath)) {
logger.debug(String.format("Editing mount path of iso from %s to %s", oldIsoVolumePath, newIsoVolumePath));
xmlDesc = replaceDiskSourceFile(xmlDesc, newIsoVolumePath, vmName);
if (logger.isDebugEnabled()) {
logger.debug(String.format("Replaced disk mount point [%s] with [%s] in VM [%s] XML configuration. New XML configuration is [%s].", oldIsoVolumePath, newIsoVolumePath, vmName, xmlDesc));
}
}
// Replace CDROM ISO path
String oldCdromIsoPath = getOldVolumePathForCdrom(disks, vmName);
String newCdromIsoPath = getNewVolumePathForCdrom(libvirtComputingResource, conn, to);
if (newCdromIsoPath != null && !newCdromIsoPath.equals(oldCdromIsoPath)) {
xmlDesc = replaceCdromIsoPath(xmlDesc, vmName, oldCdromIsoPath, newCdromIsoPath);
}
// delete the metadata of vm snapshots before migration
vmsnapshots = libvirtComputingResource.cleanVMSnapshotMetadata(dm);
// Verify Format of backing file
for (DiskDef disk : disks) {
if (disk.getDeviceType() == DiskDef.DeviceType.DISK
&& disk.getDiskFormatType() == DiskDef.DiskFmtType.QCOW2) {
libvirtComputingResource.setBackingFileFormat(disk.getDiskPath());
}
}
Map<String, MigrateCommand.MigrateDiskInfo> mapMigrateStorage = command.getMigrateStorage();
// migrateStorage is declared as final because the replaceStorage method may mutate mapMigrateStorage, but
// migrateStorage's value should always only be associated with the initial state of mapMigrateStorage.
final boolean migrateStorage = MapUtils.isNotEmpty(mapMigrateStorage);
final boolean migrateStorageManaged = command.isMigrateStorageManaged();
Set<String> migrateDiskLabels = null;
if (migrateStorage) {
if (logger.isDebugEnabled()) {
logger.debug(String.format("Changing VM [%s] volumes during migration to host: [%s].", vmName, target));
}
xmlDesc = replaceStorage(xmlDesc, mapMigrateStorage, migrateStorageManaged);
if (logger.isDebugEnabled()) {
logger.debug(String.format("Changed VM [%s] XML configuration of used storage. New XML configuration is [%s].", vmName, xmlDesc));
}
migrateDiskLabels = getMigrateStorageDeviceLabels(disks, mapMigrateStorage);
}
Map<String, DpdkTO> dpdkPortsMapping = command.getDpdkInterfaceMapping();
if (MapUtils.isNotEmpty(dpdkPortsMapping)) {
if (logger.isTraceEnabled()) {
logger.trace(String.format("Changing VM [%s] DPDK interfaces during migration to host: [%s].", vmName, target));
}
xmlDesc = replaceDpdkInterfaces(xmlDesc, dpdkPortsMapping);
if (logger.isDebugEnabled()) {
logger.debug(String.format("Changed VM [%s] XML configuration of DPDK interfaces. New XML configuration is [%s].", vmName, xmlDesc));
}
}
xmlDesc = updateVmSharesIfNeeded(command, xmlDesc, libvirtComputingResource);
dconn = libvirtUtilitiesHelper.retrieveQemuConnection(destinationUri);
if (to.getType() == VirtualMachine.Type.User) {
libvirtComputingResource.detachAndAttachConfigDriveISO(conn, vmName);
}
//run migration in thread so we can monitor it
logger.info(String.format("Starting live migration of instance [%s] to destination host [%s] having the final XML configuration: [%s].", vmName, dconn.getURI(), xmlDesc));
final ExecutorService executor = Executors.newFixedThreadPool(1);
boolean migrateNonSharedInc = command.isMigrateNonSharedInc() && !migrateStorageManaged;
// add cancel hook before we start. If migration fails to start and hook is called, it's non-fatal
cancelHook = new MigrationCancelHook(dm);
libvirtComputingResource.addDisconnectHook(cancelHook);
libvirtComputingResource.createOrUpdateLogFileForCommand(command, Command.State.PROCESSING);
final Callable<Domain> worker = new MigrateKVMAsync(libvirtComputingResource, dm, dconn, xmlDesc,
migrateStorage, migrateNonSharedInc,
command.isAutoConvergence(), vmName, command.getDestinationIp(), migrateDiskLabels);
final Future<Domain> migrateThread = executor.submit(worker);
executor.shutdown();
long sleeptime = 0;
while (!executor.isTerminated()) {
Thread.sleep(100);
sleeptime += 100;
if (sleeptime == 1000) { // wait 1s before attempting to set downtime on migration, since I don't know of a VIR_DOMAIN_MIGRATING state
final int migrateDowntime = libvirtComputingResource.getMigrateDowntime();
if (migrateDowntime > 0 ) {
try {
final int setDowntime = dm.migrateSetMaxDowntime(migrateDowntime);
if (setDowntime == 0 ) {
logger.debug("Set max downtime for migration of " + vmName + " to " + String.valueOf(migrateDowntime) + "ms");
}
} catch (final LibvirtException e) {
logger.debug("Failed to set max downtime for migration, perhaps migration completed? Error: " + e.getMessage());
}
}
}
if (sleeptime % 1000 == 0) {
logger.info("Waiting for migration of " + vmName + " to complete, waited " + sleeptime + "ms");
}
// abort the vm migration if the job is executed more than vm.migrate.wait
final int migrateWait = libvirtComputingResource.getMigrateWait();
if (migrateWait > 0 && sleeptime > migrateWait * 1000) {
DomainState state = null;
try {
state = dm.getInfo().state;
} catch (final LibvirtException e) {
logger.info("Couldn't get VM domain state after " + sleeptime + "ms: " + e.getMessage());
}
if (state != null && state == DomainState.VIR_DOMAIN_RUNNING) {
try {
DomainJobInfo job = dm.getJobInfo();
logger.info(String.format("Aborting migration of VM [%s] with domain job [%s] due to time out after %d seconds.", vmName, job, migrateWait));
dm.abortJob();
result = String.format("Migration of VM [%s] was cancelled by CloudStack due to time out after %d seconds.", vmName, migrateWait);
commandState = Command.State.FAILED;
libvirtComputingResource.createOrUpdateLogFileForCommand(command, commandState);
logger.debug(result);
break;
} catch (final LibvirtException e) {
logger.error(String.format("Failed to abort the VM migration job of VM [%s] due to: [%s].", vmName, e.getMessage()), e);
}
}
}
// pause vm if we meet the vm.migrate.pauseafter threshold and not already paused
final int migratePauseAfter = libvirtComputingResource.getMigratePauseAfter();
if (migratePauseAfter > 0 && sleeptime > migratePauseAfter) {
DomainState state = null;
try {
state = dm.getInfo().state;
} catch (final LibvirtException e) {
logger.info("Couldn't get VM domain state after " + sleeptime + "ms: " + e.getMessage());
}
if (state != null && state == DomainState.VIR_DOMAIN_RUNNING) {
try {
logger.info("Pausing VM " + vmName + " due to property vm.migrate.pauseafter setting to " + migratePauseAfter + "ms to complete migration");
dm.suspend();
} catch (final LibvirtException e) {
// pause could be racy if it attempts to pause right when vm is finished, simply warn
logger.info("Failed to pause vm " + vmName + " : " + e.getMessage());
}
}
}
}
logger.info(String.format("Migration thread of VM [%s] finished.", vmName));
destDomain = migrateThread.get(AgentPropertiesFileHandler.getPropertyValue(AgentProperties.VM_MIGRATE_DOMAIN_RETRIEVE_TIMEOUT), TimeUnit.SECONDS);
if (destDomain != null) {
if (logger.isDebugEnabled()) {
logger.debug(String.format("Cleaning the disks of VM [%s] in the source pool after VM migration finished.", vmName));
}
deleteOrDisconnectDisksOnSourcePool(libvirtComputingResource, migrateDiskInfoList, disks);
libvirtComputingResource.cleanOldSecretsByDiskDef(conn, disks);
}
} catch (final LibvirtException e) {
logger.error(String.format("Can't migrate domain [%s] due to: [%s].", vmName, e.getMessage()), e);
result = e.getMessage();
if (result.startsWith("unable to connect to server") && result.endsWith("refused")) {
logger.debug("Migration failed as connection to destination [{}] was refused. Please check libvirt configuration compatibility and firewall rules on the source and destination hosts.", destinationUri);
result = String.format("Failed to migrate domain [%s].", vmName);
}
} catch (final InterruptedException
| ExecutionException
| TimeoutException
| IOException
| ParserConfigurationException
| SAXException
| TransformerException
| URISyntaxException e) {
logger.error(String.format("Can't migrate domain [%s] due to: [%s].", vmName, e.getMessage()), e);
if (result == null) {
result = "Exception during migrate: " + e.getMessage();
}
} finally {
if (cancelHook != null) {
libvirtComputingResource.removeDisconnectHook(cancelHook);
}
try {
if (dm != null && result != null) {
// restore vm snapshots in case of failed migration
if (vmsnapshots != null) {
libvirtComputingResource.restoreVMSnapshotMetadata(dm, vmName, vmsnapshots);
}
}
if (dm != null) {
if (dm.isPersistent() == 1) {
dm.undefine();
}
dm.free();
}
if (dconn != null) {
dconn.close();
}
if (destDomain != null) {
destDomain.free();
}
} catch (final LibvirtException e) {
logger.trace("Ignoring libvirt error.", e);
}
}
if (result == null) {
libvirtComputingResource.destroyNetworkRulesForVM(conn, vmName);
for (final InterfaceDef iface : ifaces) {
String vlanId = libvirtComputingResource.getVlanIdFromBridgeName(iface.getBrName());
// We don't know which "traffic type" is associated with
// each interface at this point, so inform all vif drivers
final List<VifDriver> allVifDrivers = libvirtComputingResource.getAllVifDrivers();
for (final VifDriver vifDriver : allVifDrivers) {
vifDriver.unplug(iface, libvirtComputingResource.shouldDeleteBridge(vlanToPersistenceMap, vlanId));
}
}
commandState = Command.State.COMPLETED;
libvirtComputingResource.createOrUpdateLogFileForCommand(command, commandState);
} else if (commandState == null) {
commandState = Command.State.FAILED;
libvirtComputingResource.createOrUpdateLogFileForCommand(command, commandState);
}
return new MigrateAnswer(command, result == null, result, null);
}