in src/library/Connect/KubernetesRemoteEnvironmentManager.cs [549:683]
private async Task<(V1Pod pod, V1Container container)> _ClonePodAsync(
RemoteContainerConnectionDetails remoteContainerConnectionDetails,
ILocalProcessConfig localProcessConfig,
CancellationToken cancellationToken)
{
// If we are cloning because of routing and the pod has istio we should fail and let the user know that we do not support this scenario
var isRoutingSession = !string.IsNullOrEmpty(remoteContainerConnectionDetails.RoutingHeaderValue);
if (isRoutingSession)
{
if ((remoteContainerConnectionDetails.Pod?.Spec?.InitContainers != null &&
remoteContainerConnectionDetails.Pod.Spec.InitContainers.Where(c => c.Name.Contains("istio")).Any()) ||
(remoteContainerConnectionDetails.Pod?.Spec?.Containers != null &&
remoteContainerConnectionDetails.Pod.Spec.Containers.Where(c => c.Name.Contains("istio")).Any()))
{
throw new UserVisibleException(this._operationContext, Resources.IsolationNotSupportedWithIstio);
}
}
V1Pod clonedPod = _GetClonedPodSpec(remoteContainerConnectionDetails.Pod, remoteContainerConnectionDetails.ContainerName, _imageProvider.DevHostImage, isRoutingSession, localProcessConfig);
V1Pod userPodToRestore = null;
// If routing option is selected, add a routing label and annotation to the new pod.
if (isRoutingSession)
{
var routingHeaderValue = remoteContainerConnectionDetails.RoutingHeaderValue;
var sourceServiceName = remoteContainerConnectionDetails.ServiceName;
if (string.IsNullOrEmpty(sourceServiceName))
{
throw new UserVisibleException(_operationContext, Resources.ServiceNameNotDefinedForRoutingFormat, remoteContainerConnectionDetails.PodName, remoteContainerConnectionDetails.NamespaceName);
}
// Set routing label value to the name of the source service.
if (clonedPod.Metadata.Labels == null)
{
clonedPod.Metadata.Labels = new Dictionary<string, string>();
}
clonedPod.Metadata.Labels[Routing.RouteFromLabelName] = sourceServiceName;
// Set routing annotation value to the routing header value provided by user.
if (clonedPod.Metadata.Annotations == null)
{
clonedPod.Metadata.Annotations = new Dictionary<string, string>();
}
clonedPod.Metadata.Annotations[Routing.RouteOnHeaderAnnotationName] = $"{Routing.KubernetesRouteAsHeaderName}={routingHeaderValue}";
clonedPod.Metadata.Annotations[Routing.DebuggedContainerNameAnnotationName] = remoteContainerConnectionDetails.ContainerName;
if (remoteContainerConnectionDetails.RoutingManagerFeatureFlags != null && remoteContainerConnectionDetails.RoutingManagerFeatureFlags.Any())
{
clonedPod.Metadata.Annotations[Routing.FeatureFlagsAnnotationName] = string.Join(',', remoteContainerConnectionDetails.RoutingManagerFeatureFlags);
}
_log.Verbose($"Successfully set routing label and annotation on devhost agent pod.");
this._ReportProgress(Resources.RoutingSuccessfullyEnabledFormat, clonedPod.Metadata.Name, clonedPod.Metadata.NamespaceProperty);
}
else
{
// We want to save the user pod spec to restore it later. The spec saved in the remoteContainerConnectionDetails was edited in _GetClonedPodSpec, so we resolve a fresh copy.
userPodToRestore = await this._kubernetesClient.GetV1PodAsync(remoteContainerConnectionDetails.Pod.Metadata.NamespaceProperty, remoteContainerConnectionDetails.Pod.Metadata.Name, cancellationToken);
}
// Delete existing pod (if any) and wait for it to be removed
var isPodDeleted = await WebUtilities.RetryUntilTimeWithWaitAsync(async (t) =>
{
try
{
await _kubernetesClient.DeleteV1PodAsync(clonedPod.Namespace(), clonedPod.Name(), cancellationToken: cancellationToken);
return (await this._kubernetesClient.GetV1PodAsync(clonedPod.Namespace(), clonedPod.Name(), cancellationToken: cancellationToken)) == null;
}
catch
{
return false;
}
},
maxWaitTime: TimeSpan.FromMinutes(3),
waitInterval: TimeSpan.FromMilliseconds(500),
cancellationToken: cancellationToken);
if (!isPodDeleted)
{
throw new InvalidOperationException(string.Format(Resources.FailedToDeletePodFormat, clonedPod.Name()));
}
// Create pod and wait for it to run
var deployedPod = await _kubernetesClient.CreateV1PodAsync(clonedPod.Metadata.NamespaceProperty, clonedPod, cancellationToken: cancellationToken);
if (deployedPod == null)
{
throw new InvalidUsageException(_operationContext, Resources.FailedToDeployPodMessage);
}
var podDeployment = new PodDeployment(deployedPod);
if (userPodToRestore != null)
{
// K8s doesn't like us to pass in an old resource version when we create a new pod. We remove it from the pod spec so we
// don't get an error when we restore the pod later on.
userPodToRestore.Metadata.ResourceVersion = null;
podDeployment.UserPodToRestore = userPodToRestore;
}
try
{
// Deploy remote restore job
await _remoteRestoreJobDeployer.CreateRemoteRestoreJobAsync(deployedPod.Name(), deployedPod.Namespace(), podDeployment, cancellationToken);
// Save the pod to deployed pods context
_patchState.AddPodDeployment(podDeployment);
}
catch (Exception ex)
{
// Try to clean up deployed pod
this._ReportProgress(EventLevel.Warning, Resources.RestoringPodDeploymentMessage);
await _workloadRestorationService.Value.RemovePodDeploymentAsync(
podDeployment,
cancellationToken,
progressCallback: p => this._ReportProgress(p.Message, p.Level),
noThrow: true);
var errorMessage = $"Failed to deploy remote restore job for pod deployment {deployedPod.Namespace()}/{deployedPod.Name()}. {ex.Message}";
// In this case we cannot really do anything so logging it as warning
if (this.IsForbiddenHttpOperationException(ex))
{
_log.Warning(errorMessage);
}
else
{
_log.Error(errorMessage);
}
throw new UserVisibleException(_operationContext, ex, Resources.FailedToDeployRemoteRestoreJobFormat, deployedPod.Namespace(), deployedPod.Name(), ex.Message);
}
deployedPod = await this._WaitForPodToRunAsync(deployedPod, TimeSpan.FromMinutes(5), cancellationToken);
this._ReportProgress(Resources.PodCreatedInNamespaceFormat, deployedPod.Metadata.Name, deployedPod.Metadata.NamespaceProperty);
var container = deployedPod.Spec.Containers.Where(c => StringComparer.OrdinalIgnoreCase.Equals(c.Name, remoteContainerConnectionDetails.ContainerName)).First();
return (deployedPod, container);
}