in src/tooling/docs-assembler/Sourcing/RepositorySourcesFetcher.cs [65:226]
public class RepositorySourcer(ILoggerFactory logger, IDirectoryInfo checkoutDirectory, IFileSystem readFileSystem, DiagnosticsCollector collector)
{
private readonly ILogger<RepositorySourcer> _logger = logger.CreateLogger<RepositorySourcer>();
public async Task<IReadOnlyCollection<Checkout>> AcquireAllLatest(Dictionary<string, Repository> repositories, ContentSource source, Cancel ctx = default)
{
var dict = new ConcurrentDictionary<string, Stopwatch>();
var checkouts = new ConcurrentBag<Checkout>();
await Parallel.ForEachAsync(repositories,
new ParallelOptions
{
CancellationToken = ctx,
MaxDegreeOfParallelism = Environment.ProcessorCount
}, async (kv, c) =>
{
await Task.Run(() =>
{
var name = kv.Key.Trim();
var repo = kv.Value;
var clone = CloneOrUpdateRepository(kv.Value, name, repo.GetBranch(source), dict);
checkouts.Add(clone);
}, c);
}).ConfigureAwait(false);
return checkouts.ToList().AsReadOnly();
}
public Checkout CloneOrUpdateRepository(Repository repository, string name, string branch, ConcurrentDictionary<string, Stopwatch> dict)
{
var fs = readFileSystem;
var checkoutFolder = fs.DirectoryInfo.New(Path.Combine(checkoutDirectory.FullName, name));
var relativePath = Path.GetRelativePath(Paths.WorkingDirectoryRoot.FullName, checkoutFolder.FullName);
var sw = Stopwatch.StartNew();
_ = dict.AddOrUpdate($"{name} ({branch})", sw, (_, _) => sw);
string? head;
if (checkoutFolder.Exists)
{
if (!TryUpdateSource(name, branch, relativePath, checkoutFolder, out head))
head = CheckoutFromScratch(repository, name, branch, relativePath, checkoutFolder);
}
else
head = CheckoutFromScratch(repository, name, branch, relativePath, checkoutFolder);
sw.Stop();
return new Checkout
{
Repository = repository,
Directory = checkoutFolder,
HeadReference = head
};
}
private bool TryUpdateSource(string name, string branch, string relativePath, IDirectoryInfo checkoutFolder, [NotNullWhen(true)] out string? head)
{
head = null;
try
{
_logger.LogInformation("Pull: {Name}\t{Branch}\t{RelativePath}", name, branch, relativePath);
// --allow-unrelated-histories due to shallow clones not finding a common ancestor
ExecIn(checkoutFolder, "git", "pull", "--depth", "1", "--allow-unrelated-histories", "--no-ff");
}
catch (Exception e)
{
_logger.LogError(e, "Failed to update {Name} from {RelativePath}, falling back to recreating from scratch", name, relativePath);
if (checkoutFolder.Exists)
{
checkoutFolder.Delete(true);
checkoutFolder.Refresh();
}
return false;
}
head = Capture(checkoutFolder, "git", "rev-parse", "HEAD");
return true;
}
private string CheckoutFromScratch(Repository repository, string name, string branch, string relativePath,
IDirectoryInfo checkoutFolder)
{
_logger.LogInformation("Checkout: {Name}\t{Branch}\t{RelativePath}", name, branch, relativePath);
switch (repository.CheckoutStrategy)
{
case "full":
Exec("git", "clone", repository.Origin, checkoutFolder.FullName,
"--depth", "1", "--single-branch",
"--branch", branch
);
break;
case "partial":
Exec(
"git", "clone", "--filter=blob:none", "--no-checkout", repository.Origin, checkoutFolder.FullName
);
ExecIn(checkoutFolder, "git", "sparse-checkout", "set", "--cone");
ExecIn(checkoutFolder, "git", "checkout", branch);
ExecIn(checkoutFolder, "git", "sparse-checkout", "set", "docs");
break;
}
return Capture(checkoutFolder, "git", "rev-parse", "HEAD");
}
private void Exec(string binary, params string[] args) => ExecIn(null, binary, args);
private void ExecIn(IDirectoryInfo? workingDirectory, string binary, params string[] args)
{
var arguments = new ExecArguments(binary, args)
{
WorkingDirectory = workingDirectory?.FullName
};
var result = Proc.Exec(arguments);
if (result != 0)
collector.EmitError("", $"Exit code: {result} while executing {binary} {string.Join(" ", args)} in {workingDirectory}");
}
// ReSharper disable once UnusedMember.Local
private string Capture(IDirectoryInfo? workingDirectory, string binary, params string[] args)
{
// Try 10 times to capture the output of the command, if it fails, we'll throw an exception on the last try
Exception? e = null;
for (var i = 0; i <= 9; i++)
{
try
{
return CaptureOutput();
}
catch (Exception ex)
{
if (ex is not null)
e = ex;
}
}
if (e is not null)
collector.EmitError("", "failure capturing stdout", e);
return string.Empty;
string CaptureOutput()
{
var arguments = new StartArguments(binary, args)
{
WorkingDirectory = workingDirectory?.FullName,
//WaitForStreamReadersTimeout = TimeSpan.FromSeconds(3),
Timeout = TimeSpan.FromSeconds(3),
WaitForExit = TimeSpan.FromSeconds(3),
ConsoleOutWriter = NoopConsoleWriter.Instance
};
var result = Proc.Start(arguments);
var line = result.ExitCode != 0
? throw new Exception($"Exit code is not 0. Received {result.ExitCode} from {binary}: {workingDirectory}")
: result.ConsoleOut.FirstOrDefault()?.Line ?? throw new Exception($"No output captured for {binary}: {workingDirectory}");
return line;
}
}
}