def manifest_ingest()

in src/data_load/load.py [0:0]

• 18 lines of code
• 10 McCabe index (conditional complexity)

def manifest_ingest(is_wpc, batch_size, data_objects, data_type):
    batch_objects = []
    logger.debug(f"Data type: {data_type}, WPC mode: {is_wpc}, data_objects: {data_objects}")

    for data_object in data_objects:
        logger.debug(f"Manifest Ingestion - Splitting data into batches - Full data set size {len(data_object)}, splitting into batches of {batch_size} records")
        if data_type == "MasterData":
            for record in data_object:  # Iterate over the records inside each data_object
                batch_objects.append(record)

                # Process batch when the size limit is reached
                if len(batch_objects) == batch_size:
                    send_batch_request(batch_objects, is_wpc, data_type)
                    batch_objects = []  # Reset for the next batch

        else:
            # Handle non-MasterData types (if they don't need special handling)
            batch_objects.append(data_object)

            if len(batch_objects) == batch_size or data_object == data_objects[-1]:
                send_batch_request(batch_objects, is_wpc, data_type)
                batch_objects = []  # Reset for the next batch

    # Send any remaining records in the final batch
    if batch_objects:
        send_batch_request(batch_objects, is_wpc, data_type)