def split_documents_into_equal_bytes()

in ees_microsoft_outlook/utils.py [0:0]

• 19 lines of code
• 5 McCabe index (conditional complexity)

def split_documents_into_equal_bytes(documents, allowed_size):
    """This method splits a list of dictionary into list based on allowed size limit.
    :param documents: List of dictionary to be partitioned into chunks
    :param allowed_size: Maximum size allowed for indexing per request.
    Returns:
        list_of_chunks: List of dictionary array that to be indexed.
    """
    list_of_chunks = []
    chunk = []
    current_size = allowed_size
    for document in documents:
        document_size = len(str(document))
        if document_size < current_size:
            chunk.append(document)
            current_size -= document_size
        else:
            if chunk:
                list_of_chunks.append(chunk)
            if document_size > allowed_size:
                document["body"] = None
                document_size = len(str(document))
            chunk = [document]
            current_size = allowed_size - document_size
    list_of_chunks.append(chunk)
    return list_of_chunks