protected override async Task RunTestCollectionAsync()

in src/Microsoft.VisualStudio.Extensibility.Testing.Xunit.Shared/Harness/IdeTestAssemblyRunner.cs [47:142]


        protected override async Task<RunSummary> RunTestCollectionAsync(IMessageBus messageBus, ITestCollection testCollection, IEnumerable<IXunitTestCase> testCases, CancellationTokenSource cancellationTokenSource)
        {
            var result = new RunSummary();
            var testAssemblyFinishedMessages = new List<ITestAssemblyFinished?>();
            var completedTestCaseIds = new HashSet<string>();
            try
            {
                // Handle [Fact], and also handle IdeSkippedDataRowTestCase that doesn't run inside Visual Studio
                var nonIdeTestCases = testCases.Where(testCase => testCase is not IdeTestCaseBase).ToArray();
                if (nonIdeTestCases.Any())
                {
                    var summary = await RunTestCollectionForUnspecifiedVersionAsync(completedTestCaseIds, messageBus, testCollection, nonIdeTestCases, cancellationTokenSource);
                    result.Aggregate(summary.Item1);
                    testAssemblyFinishedMessages.Add(summary.Item2);
                }

                var ideTestCases = testCases.OfType<IdeTestCaseBase>().Where(testCase => testCase is not IdeInstanceTestCase).ToArray();
                foreach (var testCasesByTargetVersion in ideTestCases.GroupBy(GetVisualStudioVersionForTestCase))
                {
                    _ideInstancesInTests!.Add(testCasesByTargetVersion.Key);
                    using var marshalledObjects = new MarshalledObjects();
                    using (var visualStudioInstanceFactory = new VisualStudioInstanceFactory())
                    {
                        marshalledObjects.Add(visualStudioInstanceFactory);
                        var summary = await RunTestCollectionForVersionAsync(visualStudioInstanceFactory, testCasesByTargetVersion.Key, completedTestCaseIds, messageBus, testCollection, testCasesByTargetVersion, cancellationTokenSource);
                        result.Aggregate(summary.Item1);
                        testAssemblyFinishedMessages.Add(summary.Item2);
                    }
                }

                foreach (var ideInstanceTestCase in testCases.OfType<IdeInstanceTestCase>())
                {
                    if (_ideInstancesInTests!.Contains(ideInstanceTestCase.VisualStudioInstanceKey))
                    {
                        // Already had at least one test run in this version, so no need to launch it separately.
                        // Report it as passed and continue.
                        ExecutionMessageSink.OnMessage(new TestClassStarting(new[] { ideInstanceTestCase }, ideInstanceTestCase.TestMethod.TestClass));
                        ExecutionMessageSink.OnMessage(new TestMethodStarting(new[] { ideInstanceTestCase }, ideInstanceTestCase.TestMethod));
                        ExecutionMessageSink.OnMessage(new TestCaseStarting(ideInstanceTestCase));

                        var test = new XunitTest(ideInstanceTestCase, ideInstanceTestCase.DisplayName);
                        ExecutionMessageSink.OnMessage(new TestStarting(test));
                        ExecutionMessageSink.OnMessage(new TestPassed(test, 0, output: null));
                        ExecutionMessageSink.OnMessage(new TestFinished(test, 0, output: null));

                        ExecutionMessageSink.OnMessage(new TestCaseFinished(ideInstanceTestCase, 0, 1, 0, 0));
                        ExecutionMessageSink.OnMessage(new TestMethodFinished(new[] { ideInstanceTestCase }, ideInstanceTestCase.TestMethod, 0, 1, 0, 0));
                        ExecutionMessageSink.OnMessage(new TestClassFinished(new[] { ideInstanceTestCase }, ideInstanceTestCase.TestMethod.TestClass, 0, 1, 0, 0));

                        continue;
                    }

                    using var marshalledObjects = new MarshalledObjects();
                    using (var visualStudioInstanceFactory = new VisualStudioInstanceFactory(leaveRunning: true))
                    {
                        marshalledObjects.Add(visualStudioInstanceFactory);
                        var summary = await RunTestCollectionForVersionAsync(visualStudioInstanceFactory, ideInstanceTestCase.VisualStudioInstanceKey, completedTestCaseIds, messageBus, testCollection, new[] { ideInstanceTestCase }, cancellationTokenSource);
                        result.Aggregate(summary.Item1);
                        testAssemblyFinishedMessages.Add(summary.Item2);
                    }
                }
            }
            catch (Exception ex)
            {
                var completedTestCases = testCases.Where(testCase => completedTestCaseIds.Contains(testCase.UniqueID));
                var remainingTestCases = testCases.Except(completedTestCases);
                foreach (var casesByTestClass in remainingTestCases.GroupBy(testCase => testCase.TestMethod.TestClass))
                {
                    ExecutionMessageSink.OnMessage(new TestClassStarting(casesByTestClass.ToArray(), casesByTestClass.Key));

                    foreach (var casesByTestMethod in casesByTestClass.GroupBy(testCase => testCase.TestMethod))
                    {
                        ExecutionMessageSink.OnMessage(new TestMethodStarting(casesByTestMethod.ToArray(), casesByTestMethod.Key));

                        foreach (var testCase in casesByTestMethod)
                        {
                            ExecutionMessageSink.OnMessage(new TestCaseStarting(testCase));

                            var test = new XunitTest(testCase, testCase.DisplayName);
                            ExecutionMessageSink.OnMessage(new TestStarting(test));
                            ExecutionMessageSink.OnMessage(new TestFailed(test, 0, null, new InvalidOperationException("Test did not run due to a harness failure.", ex)));
                            result.Failed++;
                            ExecutionMessageSink.OnMessage(new TestFinished(test, 0, null));

                            ExecutionMessageSink.OnMessage(new TestCaseFinished(testCase, 0, 1, 1, 0));
                        }

                        ExecutionMessageSink.OnMessage(new TestMethodFinished(casesByTestMethod.ToArray(), casesByTestMethod.Key, 0, casesByTestMethod.Count(), casesByTestMethod.Count(), 0));
                    }

                    ExecutionMessageSink.OnMessage(new TestClassFinished(casesByTestClass.ToArray(), casesByTestClass.Key, 0, casesByTestClass.Count(), casesByTestClass.Count(), 0));
                }
            }

            return result;
        }