def get_token_text()

in tablestore-java-mcp-server-rag/knowledge-data-generator/chunk.py [0:0]


def get_token_text(token):
    if token.type == "inline":
        return "".join(child.content for child in token.children if hasattr(child, "content"))
    elif token.type in ["paragraph_open", "paragraph_close"]:
        return ""
    else:
        return token.content or ""