in src/ccf/ccf-provider/CcfNetworkProvider.cs [383:510]
public async Task<CcfNetwork> RecoverPublicNetwork(
string targetNetworkName,
string networkToRecoverName,
int nodeCount,
string? nodeLogLevel,
SecurityPolicyConfiguration policyOption,
string previousServiceCertificate,
JsonObject? providerConfig)
{
List<string> recoveryNodeNames =
await this.nodeProvider.GetCandidateRecoveryNodes(networkToRecoverName, providerConfig);
List<Task> recoverTasks = new();
ConcurrentBag<
(NetworkNode node, long lastSignedSeqNo, NodeEndpoint ep, string serviceCertPem)>
recoverNodeEndpoints = new();
string lbName = "lb-nw-" + targetNetworkName;
var lbFqdn = this.lbProvider.GenerateLoadBalancerFqdn(
lbName,
targetNetworkName,
providerConfig);
foreach (var recoveryNode in recoveryNodeNames)
{
recoverTasks.Add(Task.Run(async () =>
{
var recoverNodeName = recoveryNode;
var nodeData = new NodeData
{
Name = recoverNodeName,
Id = ToId(recoverNodeName)
};
var recoverNodeEndpoint = await this.nodeProvider.CreateRecoverNode(
recoverNodeName,
targetNetworkName,
networkToRecoverName,
previousServiceCertificate,
nodeLogLevel,
policyOption,
nodeData,
lbFqdn.NodeSanFormat(),
providerConfig);
var serviceCertPem = await this.GetServiceCert(
recoverNodeName,
recoverNodeEndpoint.ClientRpcAddress,
onRetry: () => this.CheckNodeHealthy(targetNetworkName, recoverNodeName, providerConfig));
(NetworkNode node, long lastSignedSeqNo) = await this.WaitForRecoverNodeReady(
recoverNodeEndpoint,
serviceCertPem,
DesiredJoinNodeState.PartOfPublicNetwork);
this.logger.LogInformation(
$"Node {recoverNodeName} is up at {recoverNodeEndpoint.ClientRpcAddress} " +
$"with lastSignedSeqNo: {lastSignedSeqNo}.");
(NetworkNode, long, NodeEndpoint, string) candidate =
(node, lastSignedSeqNo, recoverNodeEndpoint, serviceCertPem);
recoverNodeEndpoints.Add(candidate);
}));
}
await Task.WhenAll(recoverTasks);
var nodesToCheck = recoverNodeEndpoints.OrderByDescending(x => x.lastSignedSeqNo);
var nodeToUse = nodesToCheck.First();
var nodesToShutdown = nodesToCheck.Skip(1);
if (nodesToShutdown.Any())
{
this.logger.LogInformation(
$"Node {nodeToUse.ep.NodeName} up at {nodeToUse.ep.ClientRpcAddress} with " +
$"lastSignedSeqNo: {nodeToUse.lastSignedSeqNo} will be used for recovery. " +
$"Shutting down other {nodesToShutdown.Count()} nodes.");
List<Task> deleteTasks = new();
foreach (var (node, lastSignedSeqNo, ep, serviceCertPem) in nodesToShutdown)
{
deleteTasks.Add(
this.nodeProvider.DeleteNode(
targetNetworkName,
node.NodeData.Name,
node.NodeData,
providerConfig));
}
await Task.WhenAll(deleteTasks);
}
this.logger.LogInformation(
$"Node {nodeToUse.ep.NodeName} up at {nodeToUse.ep.ClientRpcAddress} with " +
$"lastSignedSeqNo: {nodeToUse.lastSignedSeqNo} will be used for recovery. " +
$"Now starting " +
$"remaining {nodeCount - 1} node(s) in join mode.");
var nodeEndpoints = await this.nodeProvider.GetNodes(targetNetworkName, providerConfig);
int ordinal = int.Parse(nodeEndpoints.OrderBy(
n => n.NodeName.PadForNaturalNumberOrdering())
.Last().NodeName.Split("-").Last()) + 1;
List<NodeEndpoint> joinNodes = await this.CreateJoinNodes(
targetNetworkName,
nodeCount - 1,
nodeLogLevel,
policyOption,
providerConfig,
nodeToUse.ep,
lbFqdn.NodeSanFormat(),
nodeToUse.serviceCertPem,
DesiredJoinNodeState.PartOfPublicNetwork,
ordinal);
List<string> servers =
[nodeToUse.ep.ClientRpcAddress, .. joinNodes.Select(n => n.ClientRpcAddress)];
var lbEndpoint = await this.lbProvider.CreateLoadBalancer(
lbName,
targetNetworkName,
servers,
providerConfig);
await this.WaitForLoadBalancerReady(lbEndpoint, nodeToUse.serviceCertPem);
this.logger.LogInformation($"CCF endpoint is up at: {lbEndpoint.Endpoint}.");
return new CcfNetwork
{
Name = targetNetworkName,
InfraType = this.nodeProvider.InfraType.ToString(),
NodeCount = nodeCount,
Endpoint = lbEndpoint.Endpoint,
Nodes = servers
};
}