static List ChunkText()

in src/Blazor.AI.SeedData/Program.cs [265:299]


static List<string> ChunkText(string text, int maxChunkSize)
{
    List<string> chunks = [];
    int start = 0;

    while (start < text.Length)
    {
        // Find the end of the chunk
        int end = Math.Min(start + maxChunkSize, text.Length);

        // Ensure we don't cut off mid-sentence
        if (end < text.Length)
        {
            int lastPeriod = text.LastIndexOf('.', end);
            int lastNewLine = text.LastIndexOf('\n', end);

            // Find the last sentence end within the chunk
            int lastSentenceEnd = Math.Max(lastPeriod, lastNewLine);

            // If we found a sentence end, adjust the end of the chunk
            if (lastSentenceEnd >= start)
            {
                end = lastSentenceEnd + 1;
            }
        }

        // Add the chunk to the list
        chunks.Add(text.Substring(start, end - start).Trim());

        // Move the start index to the end of the current chunk
        start = end;
    }

    return chunks;
}