def _get_documents()

in genai-for-marketing/backend_apis/app/utils_trendspotting.py [0:0]

• 17 lines of code
• 5 McCabe index (conditional complexity)

def _get_documents(articles: dict) -> list[dict]:
    """Gets a list of documents from a list of articles.

    Args:
        articles: 
            A list of articles.

    Returns:
        A list of documents.
    """
    documents = []
    unique_docs = set()

    for article in articles['articles']:
        parsed_article = _parse_article(article['url'])
        if (parsed_article and parsed_article.text and 
            (article['title'] not in unique_docs)):
            unique_docs.add(article['title'])
            document = {
                'page_content': parsed_article.text,
                'title': article['title'],
                'url': article['url'],
                'domain': article['domain'],
                'date': article['seendate']
            }
            documents.append(document)
    return documents