perf_testing/scripts/highspeed_create.py

import os
import subprocess
import concurrent.futures
import time
import multiprocessing
import argparse
import json

def create_file(file_index, folder):
    timestamp = int(time.time())  # Get current timestamp
    filename = os.path.join(folder, f'20GFile_{timestamp}_{file_index}')
    command = f"dd if=/dev/zero of={filename} bs=16M count=1280 oflag=direct"
    start_time = time.time()
    subprocess.run(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    end_time = time.time()
    return (filename, end_time - start_time)

def main(folder, num_files):
    if not os.path.exists(folder):
        os.makedirs(folder)

    start_time = time.time()
    with concurrent.futures.ThreadPoolExecutor(max_workers=multiprocessing.cpu_count()) as executor:
        futures = [executor.submit(create_file, i, folder) for i in range(num_files)]
        results = [f.result() for f in concurrent.futures.as_completed(futures)]
    end_time = time.time()

    total_time = end_time - start_time
    total_data_written = num_files * 20  # in GB
    speed_gbps = (total_data_written * 8) / total_time  # converting GB to Gb and then calculating Gbps

    print(json.dumps({"name": "create_10_20GB_file", "total_time": total_time, "speed": speed_gbps / 8, "unit": "GiB/s"}))

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='Create multiple 20GB files in parallel.')
    parser.add_argument('folder', type=str, help='The folder where the files will be written.')
    parser.add_argument('num_files', type=int, help='The number of 20GB files to create.')

    args = parser.parse_args()
    main(args.folder, args.num_files)