in src/features/chat-page/chat-services/chat-document-service.ts [212:234]
export async function ChunkDocumentWithOverlap(
document: string
): Promise<string[]> {
const chunks: string[] = [];
if (document.length <= CHUNK_SIZE) {
// If the document is smaller than the desired chunk size, return it as a single chunk.
chunks.push(document);
return chunks;
}
let startIndex = 0;
// Split the document into chunks of the desired size, with overlap.
while (startIndex < document.length) {
const endIndex = startIndex + CHUNK_SIZE;
const chunk = document.substring(startIndex, endIndex);
chunks.push(chunk);
startIndex = endIndex - CHUNK_OVERLAP;
}
return chunks;
}