private static TestOutcome ExpectGptOutcome()

in src/extensions/testframework/YamlTestCaseRunner.cs [819:908]
75 lines of code
11 McCabe index (conditional complexity)

        private static TestOutcome ExpectGptOutcome(string output, string expectGpt, string workingDirectory, out string gptStdOut, out string gptStdErr, out string gptMerged)
        {
            Logger.Log($"ExpectGptOutcome: Checking for {expectGpt} in '{output}'");

            var outcome = TestOutcome.None;

            var sbOut = new StringBuilder();
            var sbErr = new StringBuilder();
            var sbMerged = new StringBuilder();

            var question = new StringBuilder();
            question.AppendLine($"Here's the console output:\n\n{output}\n");
            question.AppendLine($"Here's the expectation:\n\n{expectGpt}\n");
            question.AppendLine("You **must always** answer \"PASS\" if the expectation is met.");
            question.AppendLine("You **must always** answer \"FAIL\" if the expectation is not met.");
            question.AppendLine("You **must only** answer \"PASS\" with no additional text if the expectation is met.");
            question.AppendLine("If you answer \"FAIL\", you **must** provide additional text to explain why the expectation was not met (without using the word \"PASS\" as we will interpret that as a \"PASS\").");
            var questionTempFile = WriteTextToTempFile(question.ToString())!;

            try
            {
                var startProcess = FindCacheCli("ai");
                var startArgs = $"chat --quiet true --index-name @none --question @{questionTempFile}";
                var startInfo = new ProcessStartInfo(startProcess, startArgs)
                {
                    UseShellExecute = false,
                    RedirectStandardInput = true,
                    RedirectStandardError = true,
                    RedirectStandardOutput = true,
                    WorkingDirectory = workingDirectory
                };

                Logger.Log($"ExpectGptOutcome: Process.Start('{startProcess} {startArgs}')");
                var process = Process.Start(startInfo);
                if (process == null) throw new Exception("Process.Start() returned null!");
                process.StandardInput.Close();

                var outDoneSignal = new ManualResetEvent(false);
                var errDoneSignal = new ManualResetEvent(false);
                process.OutputDataReceived += (sender, e) => AppendLineOrSignal(e.Data, sbOut, sbMerged, outDoneSignal);
                process.ErrorDataReceived += (sender, e) => AppendLineOrSignal(e.Data, sbErr, sbMerged, errDoneSignal);
                process.BeginOutputReadLine();
                process.BeginErrorReadLine();

                var exitedNotKilled = WaitForExit(process, 60000);
                if (exitedNotKilled)
                {
                    outDoneSignal.WaitOne();
                    errDoneSignal.WaitOne();
                }

                var passed = exitedNotKilled && process.ExitCode == 0;
                outcome = passed ? TestOutcome.Passed : TestOutcome.Failed;

                var timedoutOrKilled = !exitedNotKilled;
                if (timedoutOrKilled)
                {
                    var message = "ExpectGptOutcome: WARNING: Timedout or killed!";
                    sbErr.AppendLine(message);
                    sbMerged.AppendLine(message);
                    Logger.LogWarning(message);
                }
            }
            catch (Exception ex)
            {
                outcome = TestOutcome.Failed;

                var exception = $"ExpectGptOutcome: EXCEPTION: {ex.Message}";
                sbErr.AppendLine(exception);
                sbMerged.AppendLine(exception);
                Logger.Log(exception);
            }

            File.Delete(questionTempFile);
            gptStdOut = sbOut.ToString();
            gptStdErr = sbErr.ToString();
            gptMerged = sbMerged.ToString();

            if (outcome == TestOutcome.Passed)
            {
                Logger.Log($"ExpectGptOutcome: Checking for 'PASS' in '{gptMerged}'");

                var passed = gptMerged.Contains("PASS") || gptMerged.Contains("TRUE") || gptMerged.Contains("YES");
                outcome = passed ? TestOutcome.Passed : TestOutcome.Failed;

                Logger.Log($"ExpectGptOutcome: {outcome}");
            }

            return outcome;
        }