in src/Blazor.AI.SeedData/Program.cs [265:299]
static List<string> ChunkText(string text, int maxChunkSize)
{
List<string> chunks = [];
int start = 0;
while (start < text.Length)
{
// Find the end of the chunk
int end = Math.Min(start + maxChunkSize, text.Length);
// Ensure we don't cut off mid-sentence
if (end < text.Length)
{
int lastPeriod = text.LastIndexOf('.', end);
int lastNewLine = text.LastIndexOf('\n', end);
// Find the last sentence end within the chunk
int lastSentenceEnd = Math.Max(lastPeriod, lastNewLine);
// If we found a sentence end, adjust the end of the chunk
if (lastSentenceEnd >= start)
{
end = lastSentenceEnd + 1;
}
}
// Add the chunk to the list
chunks.Add(text.Substring(start, end - start).Trim());
// Move the start index to the end of the current chunk
start = end;
}
return chunks;
}