def split_ast_by_size()

in tablestore-java-mcp-server-rag/knowledge-data-generator/chunk.py [0:0]

• 16 lines of code
• 5 McCabe index (conditional complexity)

def split_ast_by_size(tokens, max_size):
    chunks = []
    current_chunk = []
    current_size = 0

    for token in tokens:
        token_text = get_token_text(token)
        token_size = len(token_text)

        if current_size + token_size > max_size and current_chunk:
            chunks.append(current_chunk)
            current_chunk = []
            current_size = 0

        current_chunk.append(token)
        current_size += token_size

    if current_chunk:
        chunks.append(current_chunk)

    return chunks