in engine/orchestration/src/main/java/com/cloud/vm/VirtualMachineManagerImpl.java [1128:1470]
public void orchestrateStart(final String vmUuid, final Map<VirtualMachineProfile.Param, Object> params, final DeploymentPlan planToDeploy, final DeploymentPlanner planner)
throws InsufficientCapacityException, ConcurrentOperationException, ResourceUnavailableException {
logger.debug(() -> LogUtils.logGsonWithoutException("Trying to start VM [%s] using plan [%s] and planner [%s].", vmUuid, planToDeploy, planner));
final CallContext cctxt = CallContext.current();
final Account account = cctxt.getCallingAccount();
final User caller = cctxt.getCallingUser();
VMInstanceVO vm = _vmDao.findByUuid(vmUuid);
final VirtualMachineGuru vmGuru = getVmGuru(vm);
final Account owner = _entityMgr.findById(Account.class, vm.getAccountId());
final ServiceOfferingVO offering = _offeringDao.findById(vm.getId(), vm.getServiceOfferingId());
final VirtualMachineTemplate template = _entityMgr.findByIdIncludingRemoved(VirtualMachineTemplate.class, vm.getTemplateId());
final Ternary<VMInstanceVO, ReservationContext, ItWorkVO> start = changeToStartState(vmGuru, vm, caller, account, owner, offering, template);
if (start == null) {
return;
}
vm = start.first();
final ReservationContext ctx = start.second();
ItWorkVO work = start.third();
VMInstanceVO startedVm = null;
DataCenterDeployment plan = new DataCenterDeployment(vm.getDataCenterId(), vm.getPodIdToDeployIn(), null, null, null, null, ctx);
if (planToDeploy != null && planToDeploy.getDataCenterId() != 0) {
VMInstanceVO finalVm = vm;
logger.debug(() -> DeploymentPlanningManagerImpl.logDeploymentWithoutException(finalVm, planToDeploy, planToDeploy.getAvoids(), planner));
plan =
new DataCenterDeployment(planToDeploy.getDataCenterId(), planToDeploy.getPodId(), planToDeploy.getClusterId(), planToDeploy.getHostId(),
planToDeploy.getPoolId(), planToDeploy.getPhysicalNetworkId(), ctx);
}
final HypervisorGuru hvGuru = _hvGuruMgr.getGuru(vm.getHypervisorType());
boolean canRetry = true;
ExcludeList avoids = null;
try {
final Journal journal = start.second().getJournal();
if (planToDeploy != null) {
avoids = planToDeploy.getAvoids();
ExcludeList finalAvoids = avoids;
logger.debug(() -> LogUtils.logGsonWithoutException("Avoiding components [%s] in deployment of VM [%s].", finalAvoids, vmUuid));
}
if (avoids == null) {
avoids = new ExcludeList();
}
logger.debug("Deploy avoids pods: {}, clusters: {}, hosts: {}", avoids.getPodsToAvoid(), avoids.getClustersToAvoid(), avoids.getHostsToAvoid());
boolean planChangedByVolume = false;
boolean reuseVolume = true;
final DataCenterDeployment originalPlan = plan;
checkIfTemplateNeededForCreatingVmVolumes(vm);
int retry = StartRetry.value();
while (retry-- != 0) {
logger.debug("VM start attempt #{}", (StartRetry.value() - retry));
if (reuseVolume) {
final List<VolumeVO> vols = _volsDao.findReadyRootVolumesByInstance(vm.getId());
for (final VolumeVO vol : vols) {
final Long volTemplateId = vol.getTemplateId();
if (volTemplateId != null && volTemplateId != template.getId()) {
logger.debug("{} of {} is READY, but template ids don't match, let the planner reassign a new pool", vol, vm);
continue;
}
final StoragePool pool = (StoragePool)dataStoreMgr.getPrimaryDataStore(vol.getPoolId());
if (!pool.isInMaintenance()) {
logger.debug("Root volume is ready, need to place VM in volume's cluster");
final long rootVolDcId = pool.getDataCenterId();
final Long rootVolPodId = pool.getPodId();
final Long rootVolClusterId = pool.getClusterId();
if (planToDeploy != null && planToDeploy.getDataCenterId() != 0) {
final Long clusterIdSpecified = planToDeploy.getClusterId();
if (clusterIdSpecified != null && rootVolClusterId != null) {
if (!rootVolClusterId.equals(clusterIdSpecified)) {
logger.debug("Cannot satisfy the deployment plan passed in since " +
"the ready Root volume is in different cluster. volume's cluster: {}, cluster specified: {}",
() -> _clusterDao.findById(rootVolClusterId), () -> _clusterDao.findById(clusterIdSpecified));
throw new ResourceUnavailableException(
"Root volume is ready in different cluster, Deployment plan provided cannot be satisfied, unable to create a deployment for " +
vm, Cluster.class, clusterIdSpecified);
}
}
plan =
new DataCenterDeployment(planToDeploy.getDataCenterId(), planToDeploy.getPodId(), planToDeploy.getClusterId(),
planToDeploy.getHostId(), vol.getPoolId(), null, ctx);
} else {
plan = new DataCenterDeployment(rootVolDcId, rootVolPodId, rootVolClusterId, null, vol.getPoolId(), null, ctx);
logger.debug("{} is READY, changing deployment plan to use this pool's dcId: {}, podId: {}, and clusterId: {}", vol, rootVolDcId,
rootVolPodId, rootVolClusterId);
planChangedByVolume = true;
}
}
}
}
final VirtualMachineProfileImpl vmProfile = new VirtualMachineProfileImpl(vm, template, offering, owner, params);
logBootModeParameters(params);
DeployDestination dest = null;
try {
dest = _dpMgr.planDeployment(vmProfile, plan, avoids, planner);
} catch (final AffinityConflictException e2) {
logger.warn("Unable to create deployment, affinity rules associated to the VM conflict", e2);
throw new CloudRuntimeException("Unable to create deployment, affinity rules associated to the VM conflict");
}
if (dest == null) {
if (planChangedByVolume) {
plan = originalPlan;
planChangedByVolume = false;
reuseVolume = false;
continue;
}
throw new InsufficientServerCapacityException("Unable to create a deployment for " + vmProfile, DataCenter.class, plan.getDataCenterId(),
areAffinityGroupsAssociated(vmProfile));
}
avoids.addHost(dest.getHost().getId());
if (!template.isDeployAsIs()) {
journal.record("Deployment found - Attempt #" + (StartRetry.value() - retry), vmProfile, dest);
}
long destHostId = dest.getHost().getId();
vm.setPodIdToDeployIn(dest.getPod().getId());
final Long clusterId = dest.getCluster().getId();
updateOverCommitRatioForVmProfile(vmProfile, clusterId);
StartAnswer startAnswer = null;
try {
if (!changeState(vm, Event.OperationRetry, destHostId, work, Step.Prepare)) {
throw new ConcurrentOperationException("Unable to update the state of the Virtual Machine "+vm.getUuid()+" oldstate: "+vm.getState()+ "Event :"+Event.OperationRetry);
}
} catch (final NoTransitionException e1) {
throw new ConcurrentOperationException(e1.getMessage());
}
try {
resetVmNicsDeviceId(vm.getId());
_networkMgr.prepare(vmProfile, dest, ctx);
if (vm.getHypervisorType() != HypervisorType.BareMetal) {
checkAndAttemptMigrateVmAcrossCluster(vm, clusterId, dest.getStorageForDisks());
volumeMgr.prepare(vmProfile, dest);
}
if (!reuseVolume) {
reuseVolume = true;
}
vmGuru.finalizeVirtualMachineProfile(vmProfile, dest, ctx);
final VirtualMachineTO vmTO = hvGuru.implement(vmProfile);
updateVmMetadataManufacturerAndProduct(vmTO, vm);
checkAndSetEnterSetupMode(vmTO, params);
handlePath(vmTO.getDisks(), vm.getHypervisorType());
setVmNetworkDetails(vm, vmTO);
Commands cmds = new Commands(Command.OnError.Stop);
final Map<String, String> sshAccessDetails = _networkMgr.getSystemVMAccessDetails(vm);
final Map<String, String> ipAddressDetails = new HashMap<>(sshAccessDetails);
ipAddressDetails.remove(NetworkElementCommand.ROUTER_NAME);
StartCommand command = new StartCommand(vmTO, dest.getHost(), getExecuteInSequence(vm.getHypervisorType()));
cmds.addCommand(command);
vmGuru.finalizeDeployment(cmds, vmProfile, dest, ctx);
addExtraConfig(vmTO);
work = _workDao.findById(work.getId());
if (work == null || work.getStep() != Step.Prepare) {
throw new ConcurrentOperationException("Work steps have been changed: " + work);
}
_workDao.updateStep(work, Step.Starting);
_agentMgr.send(destHostId, cmds);
_workDao.updateStep(work, Step.Started);
startAnswer = cmds.getAnswer(StartAnswer.class);
if (startAnswer != null && startAnswer.getResult()) {
handlePath(vmTO.getDisks(), startAnswer.getIqnToData());
final String host_guid = startAnswer.getHost_guid();
if (host_guid != null) {
final HostVO finalHost = _resourceMgr.findHostByGuid(host_guid);
if (finalHost == null) {
throw new CloudRuntimeException("Host Guid " + host_guid + " doesn't exist in DB, something went wrong while processing start answer: "+startAnswer);
}
destHostId = finalHost.getId();
}
if (vmGuru.finalizeStart(vmProfile, destHostId, cmds, ctx)) {
syncDiskChainChange(startAnswer);
if (!changeState(vm, Event.OperationSucceeded, destHostId, work, Step.Done)) {
logger.error("Unable to transition to a new state. VM uuid: {}, VM oldstate: {}, Event: {}", vm, vm.getState(), Event.OperationSucceeded);
throw new ConcurrentOperationException(String.format("Failed to deploy VM %s", vm));
}
final GPUDeviceTO gpuDevice = startAnswer.getVirtualMachine().getGpuDevice();
if (gpuDevice != null) {
_resourceMgr.updateGPUDetails(destHostId, gpuDevice.getGroupDetails());
}
if (userVmDetailsDao.findDetail(vm.getId(), VmDetailConstants.DEPLOY_VM) != null) {
userVmDetailsDao.removeDetail(vm.getId(), VmDetailConstants.DEPLOY_VM);
}
startedVm = vm;
logger.debug("Start completed for VM {}", vm);
final Host vmHost = _hostDao.findById(destHostId);
if (vmHost != null && (VirtualMachine.Type.ConsoleProxy.equals(vm.getType()) ||
VirtualMachine.Type.SecondaryStorageVm.equals(vm.getType())) && caManager.canProvisionCertificates()) {
for (int retries = 3; retries > 0; retries--) {
try {
final Certificate certificate = caManager.issueCertificate(null, Arrays.asList(vm.getHostName(), vm.getInstanceName()),
new ArrayList<>(ipAddressDetails.values()), CAManager.CertValidityPeriod.value(), null);
final boolean result = caManager.deployCertificate(vmHost, certificate, false, sshAccessDetails);
if (!result) {
logger.error("Failed to setup certificate for system vm: {}", vm.getInstanceName());
}
return;
} catch (final Exception e) {
logger.error("Retrying after catching exception while trying to secure agent for systemvm {}", vm, e);
}
}
throw new CloudRuntimeException(String.format("Failed to setup and secure agent for systemvm %s", vm));
}
return;
} else {
logger.info("The guru did not like the answers so stopping {}", vm);
StopCommand stopCmd = new StopCommand(vm, getExecuteInSequence(vm.getHypervisorType()), false);
stopCmd.setControlIp(getControlNicIpForVM(vm));
Map<String, Boolean> vlanToPersistenceMap = getVlanToPersistenceMapForVM(vm.getId());
if (MapUtils.isNotEmpty(vlanToPersistenceMap)) {
stopCmd.setVlanToPersistenceMap(vlanToPersistenceMap);
}
final StopCommand cmd = stopCmd;
final Answer answer = _agentMgr.easySend(destHostId, cmd);
if (answer != null && answer instanceof StopAnswer) {
final StopAnswer stopAns = (StopAnswer)answer;
if (vm.getType() == VirtualMachine.Type.User) {
final String platform = stopAns.getPlatform();
if (platform != null) {
final Map<String,String> vmmetadata = new HashMap<>();
vmmetadata.put(vm.getInstanceName(), platform);
syncVMMetaData(vmmetadata);
}
}
}
if (answer == null || !answer.getResult()) {
logger.warn("Unable to stop {} dut to {}", vm, (answer != null ? answer.getDetails() : "no answers"));
_haMgr.scheduleStop(vm, destHostId, WorkType.ForceStop);
throw new ExecutionException("Unable to stop this VM, " + vm.getUuid() + " so we are unable to retry the start operation");
}
throw new ExecutionException("Unable to start VM:" + vm.getUuid() + " due to error in finalizeStart, not retrying");
}
}
logger.info("Unable to start VM on {} due to {}", dest.getHost(), (startAnswer == null ? " no start answer" : startAnswer.getDetails()));
if (startAnswer != null && startAnswer.getContextParam("stopRetry") != null) {
break;
}
} catch (OperationTimedoutException e) {
logger.debug("Unable to send the start command to host {} failed to start VM: {}", dest.getHost(), vm);
if (e.isActive()) {
_haMgr.scheduleStop(vm, destHostId, WorkType.CheckStop);
}
canRetry = false;
throw new AgentUnavailableException("Unable to start " + vm.getHostName(), destHostId, e);
} catch (final ResourceUnavailableException e) {
logger.warn("Unable to contact resource.", e);
if (!avoids.add(e)) {
if (e.getScope() == Volume.class || e.getScope() == Nic.class) {
throw e;
} else {
logger.warn("unexpected ResourceUnavailableException : {}", e.getScope().getName(), e);
throw e;
}
}
} catch (final InsufficientCapacityException e) {
logger.warn("Insufficient capacity ", e);
if (!avoids.add(e)) {
if (e.getScope() == Volume.class || e.getScope() == Nic.class) {
throw e;
} else {
logger.warn("unexpected InsufficientCapacityException : {}", e.getScope().getName(), e);
}
}
} catch (ExecutionException | NoTransitionException | ResourceAllocationException e) {
logger.error("Failed to start instance {}", vm, e);
throw new AgentUnavailableException("Unable to start instance due to " + e.getMessage(), destHostId, e);
} catch (final StorageAccessException e) {
logger.warn("Unable to access storage on host", e);
} finally {
if (startedVm == null && canRetry) {
final Step prevStep = work.getStep();
_workDao.updateStep(work, Step.Release);
if ((prevStep == Step.Started || prevStep == Step.Starting) && startAnswer != null && startAnswer.getResult()) {
cleanup(vmGuru, vmProfile, work, Event.OperationFailed, false);
} else {
cleanup(vmGuru, vmProfile, work, Event.OperationFailed, true);
}
}
}
}
} finally {
if (startedVm == null) {
if (VirtualMachine.Type.User.equals(vm.type) && ResourceCountRunningVMsonly.value()) {
_resourceLimitMgr.decrementVmResourceCount(owner.getAccountId(), vm.isDisplay(), offering, template);
}
if (canRetry) {
try {
conditionallySetPodToDeployIn(vm);
changeState(vm, Event.OperationFailed, null, work, Step.Done);
} catch (final NoTransitionException e) {
throw new ConcurrentOperationException(e.getMessage());
}
}
}
if (planToDeploy != null) {
planToDeploy.setAvoids(avoids);
}
}
if (startedVm == null) {
throw new CloudRuntimeException("Unable to start instance '" + vm.getHostName() + "' (" + vm.getUuid() + "), see management server log for details");
}
}