public async Task RecoverPublicNetwork()

in src/ccf/ccf-provider/CcfNetworkProvider.cs [383:510]


    public async Task<CcfNetwork> RecoverPublicNetwork(
        string targetNetworkName,
        string networkToRecoverName,
        int nodeCount,
        string? nodeLogLevel,
        SecurityPolicyConfiguration policyOption,
        string previousServiceCertificate,
        JsonObject? providerConfig)
    {
        List<string> recoveryNodeNames =
            await this.nodeProvider.GetCandidateRecoveryNodes(networkToRecoverName, providerConfig);
        List<Task> recoverTasks = new();
        ConcurrentBag<
            (NetworkNode node, long lastSignedSeqNo, NodeEndpoint ep, string serviceCertPem)>
            recoverNodeEndpoints = new();
        string lbName = "lb-nw-" + targetNetworkName;
        var lbFqdn = this.lbProvider.GenerateLoadBalancerFqdn(
            lbName,
            targetNetworkName,
            providerConfig);
        foreach (var recoveryNode in recoveryNodeNames)
        {
            recoverTasks.Add(Task.Run(async () =>
            {
                var recoverNodeName = recoveryNode;
                var nodeData = new NodeData
                {
                    Name = recoverNodeName,
                    Id = ToId(recoverNodeName)
                };

                var recoverNodeEndpoint = await this.nodeProvider.CreateRecoverNode(
                    recoverNodeName,
                    targetNetworkName,
                    networkToRecoverName,
                    previousServiceCertificate,
                    nodeLogLevel,
                    policyOption,
                    nodeData,
                    lbFqdn.NodeSanFormat(),
                    providerConfig);

                var serviceCertPem = await this.GetServiceCert(
                    recoverNodeName,
                    recoverNodeEndpoint.ClientRpcAddress,
                    onRetry: () => this.CheckNodeHealthy(targetNetworkName, recoverNodeName, providerConfig));

                (NetworkNode node, long lastSignedSeqNo) = await this.WaitForRecoverNodeReady(
                    recoverNodeEndpoint,
                    serviceCertPem,
                    DesiredJoinNodeState.PartOfPublicNetwork);

                this.logger.LogInformation(
                    $"Node {recoverNodeName} is up at {recoverNodeEndpoint.ClientRpcAddress} " +
                    $"with lastSignedSeqNo: {lastSignedSeqNo}.");
                (NetworkNode, long, NodeEndpoint, string) candidate =
                (node, lastSignedSeqNo, recoverNodeEndpoint, serviceCertPem);
                recoverNodeEndpoints.Add(candidate);
            }));
        }

        await Task.WhenAll(recoverTasks);
        var nodesToCheck = recoverNodeEndpoints.OrderByDescending(x => x.lastSignedSeqNo);
        var nodeToUse = nodesToCheck.First();
        var nodesToShutdown = nodesToCheck.Skip(1);

        if (nodesToShutdown.Any())
        {
            this.logger.LogInformation(
                $"Node {nodeToUse.ep.NodeName} up at {nodeToUse.ep.ClientRpcAddress} with " +
                $"lastSignedSeqNo: {nodeToUse.lastSignedSeqNo} will be used for recovery. " +
                $"Shutting down other {nodesToShutdown.Count()} nodes.");
            List<Task> deleteTasks = new();
            foreach (var (node, lastSignedSeqNo, ep, serviceCertPem) in nodesToShutdown)
            {
                deleteTasks.Add(
                    this.nodeProvider.DeleteNode(
                        targetNetworkName,
                        node.NodeData.Name,
                        node.NodeData,
                        providerConfig));
            }

            await Task.WhenAll(deleteTasks);
        }

        this.logger.LogInformation(
            $"Node {nodeToUse.ep.NodeName} up at {nodeToUse.ep.ClientRpcAddress} with " +
            $"lastSignedSeqNo: {nodeToUse.lastSignedSeqNo} will be used for recovery. " +
            $"Now starting " +
            $"remaining {nodeCount - 1} node(s) in join mode.");

        var nodeEndpoints = await this.nodeProvider.GetNodes(targetNetworkName, providerConfig);
        int ordinal = int.Parse(nodeEndpoints.OrderBy(
            n => n.NodeName.PadForNaturalNumberOrdering())
            .Last().NodeName.Split("-").Last()) + 1;
        List<NodeEndpoint> joinNodes = await this.CreateJoinNodes(
            targetNetworkName,
            nodeCount - 1,
            nodeLogLevel,
            policyOption,
            providerConfig,
            nodeToUse.ep,
            lbFqdn.NodeSanFormat(),
            nodeToUse.serviceCertPem,
            DesiredJoinNodeState.PartOfPublicNetwork,
            ordinal);

        List<string> servers =
            [nodeToUse.ep.ClientRpcAddress, .. joinNodes.Select(n => n.ClientRpcAddress)];
        var lbEndpoint = await this.lbProvider.CreateLoadBalancer(
            lbName,
            targetNetworkName,
            servers,
            providerConfig);

        await this.WaitForLoadBalancerReady(lbEndpoint, nodeToUse.serviceCertPem);

        this.logger.LogInformation($"CCF endpoint is up at: {lbEndpoint.Endpoint}.");
        return new CcfNetwork
        {
            Name = targetNetworkName,
            InfraType = this.nodeProvider.InfraType.ToString(),
            NodeCount = nodeCount,
            Endpoint = lbEndpoint.Endpoint,
            Nodes = servers
        };
    }