def delete_ingested_records()

in src/data_load/load.py [0:0]

• 24 lines of code
• 11 McCabe index (conditional complexity)

def delete_ingested_records(dir_name):
    # Recursive traversal of files and subdirectories of the root directory and files processing
    success = []
    failed = []
    for root, _, files in os.walk(dir_name):
        for file in files:
            filepath = os.path.join(root, file)
            if filepath.endswith(".json"):
                with open(filepath) as file:
                    data_object = json.load(file)

            if not data_object:
                logger.error(f"Error with file {filepath}. File is empty.")
            elif "ReferenceData" in data_object:
                ingested_data = data_object["ReferenceData"]
            elif "MasterData" in data_object:
                ingested_data = data_object["MasterData"]

            ids = []
            for ingested_datum in ingested_data:
                if "id" in ingested_datum:
                    ids.append(ingested_datum.get("id"))

            s, f = delete_ids(ids)
            success += s
            failed += f
            logger.info(f"deleted records of - {filepath}")

    return success, failed