in src/ccf/caci-ccf-provider/CAciNodeProvider.cs [176:329]
public async Task<NodeEndpoint> CreateJoinNode(
string nodeName,
string networkName,
string serviceCertPem,
string targetNodeName,
string targetRpcAddress,
string? nodeLogLevel,
SecurityPolicyConfiguration policyOption,
NodeData nodeData,
List<string> san,
JsonObject? providerConfig)
{
this.ValidateCreateInput(providerConfig);
string containerGroupName = nodeName;
ContainerGroupData? cgData =
await AciUtils.TryGetContainerGroupData(containerGroupName, providerConfig!);
if (cgData != null)
{
this.logger.LogWarning($"Found existing {containerGroupName} instance." +
$" Not re-creating join node.");
return AciUtils.ToNodeEndpoint(cgData);
}
var nodeStorageProvider = NodeStorageProviderFactory.Create(
networkName,
providerConfig,
this.InfraType,
this.logger);
List<IDirectory>? roLedgerDirs = null;
IDirectory? roSnapshotsDir = null;
if (providerConfig.FastJoin(nodeStorageProvider.GetNodeStorageType()))
{
this.logger.LogInformation($"Will look for snapshots to use for joining {nodeName}.");
(roLedgerDirs, roSnapshotsDir) =
await nodeStorageProvider.GetReadonlyLedgerSnapshotsDir();
}
string nodeConfigDataDir = WorkspaceDirectories.GetConfigurationDirectory(
nodeName,
networkName,
this.InfraType);
Directory.CreateDirectory(nodeConfigDataDir);
var cchostConfig = await CCHostConfig.InitConfig(
"templates/snp/join-config.json",
outDir: nodeConfigDataDir);
string dnsNameLabel = this.GenerateDnsName(nodeName, networkName, providerConfig!);
string location = providerConfig!["location"]!.ToString();
var fqdn = $"{dnsNameLabel}.{location}.azurecontainer.io";
string instanceId = Guid.NewGuid().ToString();
this.AddInfraProviderData(nodeData, instanceId);
cchostConfig.SetPublishedAddress(fqdn);
cchostConfig.SetNodeLogLevel(nodeLogLevel);
await cchostConfig.SetNodeData(nodeData);
var altNames = fqdn.NodeSanFormat();
altNames.AddRange(san);
cchostConfig.SetSubjectAltNames(altNames);
await cchostConfig.SetJoinConfiguration(targetRpcAddress, serviceCertPem);
// Set the ledger and snapshots directory mount paths that are mapped to the docker host.
if (await nodeStorageProvider.NodeStorageDirectoryExists(nodeName))
{
this.logger.LogWarning(
$"{nodeName} node storage folder already exists from a previous run and " +
$"will be mounted as-is for creating the join node container.");
await nodeStorageProvider.DeleteUncommittedLedgerFiles(nodeName);
}
await nodeStorageProvider.CreateNodeStorageDirectory(nodeName);
(var rwLedgerDir, var rwSnapshotsDir, var logsDir) =
await nodeStorageProvider.GetReadWriteLedgerSnapshotsDir(nodeName);
// Copy the latest snapshot from RO snapshots into the joining node's storage or else
// we may not have access to the snapshot when creating new nodes in join mode which use
// this node as the target for joining and then can get StartupSeqnoIsOld error if
// new nodes start w/o a snapshot while this node acting as primary started from a
// snapshot. We need to point the joining nodes to at least the snapshot
// from which the recovery node started.
await nodeStorageProvider.CopyLatestSnapshot(roSnapshotsDir, rwSnapshotsDir);
// Not specifying roSNapshotsDirectory as roSnapshotsDir?.MountPath since we copied the
// latest snapshot into the snapshots rw dir above.
cchostConfig.SetLedgerSnapshotsDirectory(
rwLedgerDir.MountPath,
rwSnapshotsDir.MountPath,
roLedgerDirs?.ConvertAll(d => d.MountPath),
roSnapshotsDirectory: null);
// Add any configuration files that the node storage provider needs during container start.
await nodeStorageProvider.AddNodeStorageProviderConfiguration(
nodeConfigDataDir,
rwLedgerDir,
rwSnapshotsDir,
roLedgerDirs,
roSnapshotsDir);
await cchostConfig.SaveConfig();
// Pack the contents of the directory into base64 encoded tar gzip string which then
// gets uncompressed and expanded in the container.
string tgzConfigData = await Utils.PackDirectory(nodeConfigDataDir);
var securityPolicy =
await this.GetContainerGroupSecurityPolicy(policyOption);
ContainerGroupData inputData = this.CreateContainerGroupData(
location,
networkName,
nodeName,
dnsNameLabel,
tgzConfigData,
instanceId,
securityPolicy);
await nodeStorageProvider.UpdateCreateContainerGroupParams(
rwLedgerDir,
rwSnapshotsDir,
roLedgerDirs,
roSnapshotsDir,
inputData);
if (providerConfig.JoinNodeSleep())
{
inputData.Containers.Single(
c => c.Name == AciConstants.ContainerName.CcHost).EnvironmentVariables.Add(
new ContainerEnvironmentVariable("TAIL_DEV_NULL")
{
Value = "true"
});
}
if (logsDir != null)
{
inputData.Containers.Single(
c => c.Name == AciConstants.ContainerName.CcHost).EnvironmentVariables.Add(
new ContainerEnvironmentVariable("LOGS_DIR")
{
Value = logsDir.MountPath
});
}
ContainerGroupData resourceData = await this.CreateContainerGroup(
containerGroupName,
inputData,
providerConfig);
return AciUtils.ToNodeEndpoint(resourceData);
}