in src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Harness/IdeTestAssemblyRunner.cs [47:142]
protected override async Task<RunSummary> RunTestCollectionAsync(IMessageBus messageBus, ITestCollection testCollection, IEnumerable<IXunitTestCase> testCases, CancellationTokenSource cancellationTokenSource)
{
var result = new RunSummary();
var testAssemblyFinishedMessages = new List<ITestAssemblyFinished?>();
var completedTestCaseIds = new HashSet<string>();
try
{
// Handle [Fact], and also handle IdeSkippedDataRowTestCase that doesn't run inside Visual Studio
var nonIdeTestCases = testCases.Where(testCase => testCase is not IdeTestCaseBase).ToArray();
if (nonIdeTestCases.Any())
{
var summary = await RunTestCollectionForUnspecifiedVersionAsync(completedTestCaseIds, messageBus, testCollection, nonIdeTestCases, cancellationTokenSource);
result.Aggregate(summary.Item1);
testAssemblyFinishedMessages.Add(summary.Item2);
}
var ideTestCases = testCases.OfType<IdeTestCaseBase>().Where(testCase => testCase is not IdeInstanceTestCase).ToArray();
foreach (var testCasesByTargetVersion in ideTestCases.GroupBy(GetVisualStudioVersionForTestCase))
{
_ideInstancesInTests!.Add(testCasesByTargetVersion.Key);
using var marshalledObjects = new MarshalledObjects();
using (var visualStudioInstanceFactory = new VisualStudioInstanceFactory())
{
marshalledObjects.Add(visualStudioInstanceFactory);
var summary = await RunTestCollectionForVersionAsync(visualStudioInstanceFactory, testCasesByTargetVersion.Key, completedTestCaseIds, messageBus, testCollection, testCasesByTargetVersion, cancellationTokenSource);
result.Aggregate(summary.Item1);
testAssemblyFinishedMessages.Add(summary.Item2);
}
}
foreach (var ideInstanceTestCase in testCases.OfType<IdeInstanceTestCase>())
{
if (_ideInstancesInTests!.Contains(ideInstanceTestCase.VisualStudioInstanceKey))
{
// Already had at least one test run in this version, so no need to launch it separately.
// Report it as passed and continue.
ExecutionMessageSink.OnMessage(new TestClassStarting(new[] { ideInstanceTestCase }, ideInstanceTestCase.TestMethod.TestClass));
ExecutionMessageSink.OnMessage(new TestMethodStarting(new[] { ideInstanceTestCase }, ideInstanceTestCase.TestMethod));
ExecutionMessageSink.OnMessage(new TestCaseStarting(ideInstanceTestCase));
var test = new XunitTest(ideInstanceTestCase, ideInstanceTestCase.DisplayName);
ExecutionMessageSink.OnMessage(new TestStarting(test));
ExecutionMessageSink.OnMessage(new TestPassed(test, 0, output: null));
ExecutionMessageSink.OnMessage(new TestFinished(test, 0, output: null));
ExecutionMessageSink.OnMessage(new TestCaseFinished(ideInstanceTestCase, 0, 1, 0, 0));
ExecutionMessageSink.OnMessage(new TestMethodFinished(new[] { ideInstanceTestCase }, ideInstanceTestCase.TestMethod, 0, 1, 0, 0));
ExecutionMessageSink.OnMessage(new TestClassFinished(new[] { ideInstanceTestCase }, ideInstanceTestCase.TestMethod.TestClass, 0, 1, 0, 0));
continue;
}
using var marshalledObjects = new MarshalledObjects();
using (var visualStudioInstanceFactory = new VisualStudioInstanceFactory(leaveRunning: true))
{
marshalledObjects.Add(visualStudioInstanceFactory);
var summary = await RunTestCollectionForVersionAsync(visualStudioInstanceFactory, ideInstanceTestCase.VisualStudioInstanceKey, completedTestCaseIds, messageBus, testCollection, new[] { ideInstanceTestCase }, cancellationTokenSource);
result.Aggregate(summary.Item1);
testAssemblyFinishedMessages.Add(summary.Item2);
}
}
}
catch (Exception ex)
{
var completedTestCases = testCases.Where(testCase => completedTestCaseIds.Contains(testCase.UniqueID));
var remainingTestCases = testCases.Except(completedTestCases);
foreach (var casesByTestClass in remainingTestCases.GroupBy(testCase => testCase.TestMethod.TestClass))
{
ExecutionMessageSink.OnMessage(new TestClassStarting(casesByTestClass.ToArray(), casesByTestClass.Key));
foreach (var casesByTestMethod in casesByTestClass.GroupBy(testCase => testCase.TestMethod))
{
ExecutionMessageSink.OnMessage(new TestMethodStarting(casesByTestMethod.ToArray(), casesByTestMethod.Key));
foreach (var testCase in casesByTestMethod)
{
ExecutionMessageSink.OnMessage(new TestCaseStarting(testCase));
var test = new XunitTest(testCase, testCase.DisplayName);
ExecutionMessageSink.OnMessage(new TestStarting(test));
ExecutionMessageSink.OnMessage(new TestFailed(test, 0, null, new InvalidOperationException("Test did not run due to a harness failure.", ex)));
result.Failed++;
ExecutionMessageSink.OnMessage(new TestFinished(test, 0, null));
ExecutionMessageSink.OnMessage(new TestCaseFinished(testCase, 0, 1, 1, 0));
}
ExecutionMessageSink.OnMessage(new TestMethodFinished(casesByTestMethod.ToArray(), casesByTestMethod.Key, 0, casesByTestMethod.Count(), casesByTestMethod.Count(), 0));
}
ExecutionMessageSink.OnMessage(new TestClassFinished(casesByTestClass.ToArray(), casesByTestClass.Key, 0, casesByTestClass.Count(), casesByTestClass.Count(), 0));
}
}
return result;
}