in src/ccf/virtual-ccf-provider/docker/DockerNodeProvider.cs [205:382]
public async Task<NodeEndpoint> CreateJoinNode(
string nodeName,
string networkName,
string serviceCertPem,
string targetNodeName,
string targetRpcAddress,
string? nodeLogLevel,
SecurityPolicyConfiguration policyOption,
NodeData nodeData,
List<string> san,
JsonObject? providerConfig)
{
var nodeStorageProvider = NodeStorageProviderFactory.Create(
networkName,
providerConfig,
this.InfraType,
this.logger);
List<IDirectory>? roLedgerDirs = null;
IDirectory? roSnapshotsDir = null;
if (providerConfig.FastJoin(nodeStorageProvider.GetNodeStorageType()))
{
this.logger.LogInformation($"Will look for snapshots to use for joining {nodeName}.");
(roLedgerDirs, roSnapshotsDir) =
await nodeStorageProvider.GetReadonlyLedgerSnapshotsDir();
}
string nodeConfigDataDir = WorkspaceDirectories.GetConfigurationDirectory(
nodeName,
networkName,
this.InfraType);
Directory.CreateDirectory(nodeConfigDataDir);
var cchostConfig = await CCHostConfig.InitConfig(
"templates/virtual/join-config.json",
outDir: nodeConfigDataDir);
string containerName = "cchost-nw-" + nodeName;
cchostConfig.SetPublishedAddress(fqdn: containerName);
cchostConfig.SetNodeLogLevel(nodeLogLevel);
await cchostConfig.SetNodeData(nodeData);
cchostConfig.SetSubjectAltNames(san);
await cchostConfig.SetJoinConfiguration(targetRpcAddress, serviceCertPem);
// Set the ledger and snapshots directory mount paths that are mapped to the docker host.
if (!await this.client.ContainerExists(containerName) &&
await nodeStorageProvider.NodeStorageDirectoryExists(nodeName))
{
this.logger.LogWarning(
$"{nodeName} node storage folder already exists from a previous run and " +
$"will be mounted as-is for creating the join node container.");
await nodeStorageProvider.DeleteUncommittedLedgerFiles(nodeName);
}
await nodeStorageProvider.CreateNodeStorageDirectory(nodeName);
(var rwLedgerDir, var rwSnapshotsDir, var logsDir) =
await nodeStorageProvider.GetReadWriteLedgerSnapshotsDir(nodeName);
// Copy the latest snapshot from RO snapshots into the joining node's storage or else
// we may not have access to the snapshot when creating new nodes in join mode which use
// this node as the target for joining and then can get StartupSeqnoIsOld error if
// new nodes start w/o a snapshot while this node acting as primary started from a
// snapshot. We need to point the joining nodes to at least the snapshot
// from which the recovery node started.
await nodeStorageProvider.CopyLatestSnapshot(roSnapshotsDir, rwSnapshotsDir);
// Not specifying roSNapshotsDirectory as roSnapshotsDir?.MountPath since we copied the
// latest snapshot into the snapshots rw dir above.
cchostConfig.SetLedgerSnapshotsDirectory(
rwLedgerDir.MountPath,
rwSnapshotsDir.MountPath,
roLedgerDirs?.ConvertAll(d => d.MountPath),
roSnapshotsDirectory: null);
await cchostConfig.SaveConfig();
// Add any configuration files that the node storage provider needs during container start.
await nodeStorageProvider.AddNodeStorageProviderConfiguration(
nodeConfigDataDir,
rwLedgerDir,
rwSnapshotsDir,
roLedgerDirs,
roSnapshotsDir);
// Pack the contents of the directory into base64 encoded tar gzip string which then
// gets uncompressed and expanded in the container.
string tgzConfigData = await Utils.PackDirectory(nodeConfigDataDir);
var createContainerParams = new CreateContainerParameters
{
Labels = new Dictionary<string, string>
{
{
DockerConstants.CcfNetworkNameTag,
networkName
},
{
DockerConstants.CcfNetworkTypeTag,
"node"
},
{
DockerConstants.CcfNetworkResourceNameTag,
nodeName
}
},
Name = containerName,
Image = $"{ImageUtils.CcfRunJsAppVirtualImage()}:{ImageUtils.CcfRunJsAppVirtualTag()}",
Env =
[
$"CONFIG_DATA_TGZ={tgzConfigData}"
],
ExposedPorts = new Dictionary<string, EmptyStruct>
{
{
$"{Ports.RpcMainPort}/tcp", new EmptyStruct()
},
{
$"{Ports.NodeToNodePort}/tcp", new EmptyStruct()
},
{
$"{Ports.RpcDebugPort}/tcp", new EmptyStruct()
}
},
HostConfig = new HostConfig
{
Devices = new List<DeviceMapping>(),
CapAdd = new List<string>(),
NetworkMode = networkName,
Binds = new List<string>(),
PortBindings = new Dictionary<string, IList<PortBinding>>
{
{
$"{Ports.RpcMainPort}/tcp", new List<PortBinding>
{
new()
{
// Dynamic assignment.
HostPort = null
}
}
},
{
$"{Ports.RpcDebugPort}/tcp", new List<PortBinding>
{
new()
{
// Dynamic assignment.
HostPort = null
}
}
}
}
}
};
if (providerConfig.JoinNodeSleep())
{
createContainerParams.Env.Add("TAIL_DEV_NULL=true");
}
if (logsDir != null)
{
createContainerParams.Env.Add($"LOGS_DIR={logsDir.MountPath}");
}
await nodeStorageProvider.UpdateCreateContainerParams(
nodeName,
roLedgerDirs,
roSnapshotsDir,
createContainerParams);
NodeEndpoint nodeEndpoint = await this.CreateAndStartNodeContainer(createContainerParams);
await this.CreateAndStartRecoveryAgentContainer(networkName, nodeEndpoint);
return nodeEndpoint;
}