in cosmos/scripts/aggregate_multiple_file_results.py [0:0]
def main():
if len(sys.argv) == 1:
raise Exception("Input log files directory not provided. Syntax = 'python aggregate_multiple_files_result.py "
"<FilesDirectory>'")
path = sys.argv[1]
total_read_count = 0
total_read_failed_count = 0
total_update_count = 0
total_update_failed_count = 0
total_write_count = 0
total_write_failed_count = 0
total_scan_count = 0
total_scan_failed_count = 0
total_throughput = 0
for filename in glob.glob(os.path.join(path, '*.log')):
with open(os.path.join(os.getcwd(), filename), 'r') as f: # open in readonly mode
current_file = open(filename, 'r')
lines = current_file.readlines()
for line in lines:
if '[READ], Operations,' in line:
total_read_count += int(line.replace('[READ], Operations, ', ''))
elif '[READ-FAILED], Operations,' in line:
total_read_failed_count += int(line.replace('[READ-FAILED], Operations, ', ''))
elif '[UPDATE], Operations,' in line:
total_update_count += int(line.replace('[UPDATE], Operations, ', ''))
elif '[UPDATE-FAILED], Operations,' in line:
total_update_failed_count += int(line.replace('[UPDATE-FAILED], Operations, ', ''))
elif '[INSERT], Operations,' in line:
total_write_count += int(line.replace('[INSERT], Operations, ', ''))
elif '[INSERT-FAILED], Operations,' in line:
total_write_failed_count += int(line.replace('[INSERT-FAILED], Operations, ', ''))
elif '[SCAN], Operations,' in line:
total_scan_count += int(line.replace('[SCAN], Operations, ', ''))
elif '[SCAN-FAILED], Operations,' in line:
total_scan_failed_count += int(line.replace('[SCAN-FAILED], Operations, ', ''))
elif '[OVERALL], Throughput(ops/sec), ' in line:
total_throughput += float(line.replace('[OVERALL], Throughput(ops/sec), ', ''))
total_read_avg = 0
total_read_p95 = 0
total_read_p99 = 0
total_read_min = sys.maxsize
total_read_max = 0
total_read_failed_avg = 0
total_read_failed_p95 = 0
total_read_failed_p99 = 0
total_read_failed_min = sys.maxsize
total_read_failed_max = 0
total_update_avg = 0
total_update_p95 = 0
total_update_p99 = 0
total_update_min = sys.maxsize
total_update_max = 0
total_update_failed_avg = 0
total_update_failed_p95 = 0
total_update_failed_p99 = 0
total_update_failed_min = sys.maxsize
total_update_failed_max = 0
total_write_avg = 0
total_write_p95 = 0
total_write_p99 = 0
total_write_min = sys.maxsize
total_write_max = 0
total_write_failed_avg = 0
total_write_failed_p95 = 0
total_write_failed_p99 = 0
total_write_failed_min = sys.maxsize
total_write_failed_max = 0
total_scan_avg = 0
total_scan_p95 = 0
total_scan_p99 = 0
total_scan_min = sys.maxsize
total_scan_max = 0
total_scan_failed_avg = 0
total_scan_failed_p95 = 0
total_scan_failed_p99 = 0
total_scan_failed_min = sys.maxsize
total_scan_failed_max = 0
for filename in glob.glob(os.path.join(path, '*.log')):
with open(os.path.join(os.getcwd(), filename), 'r') as f: # open in readonly mode
current_file = open(filename, 'r')
lines = current_file.readlines()
current_read_operation = 0
current_read_failed_operation = 0
current_update_operation = 0
current_update_failed_operation = 0
current_write_operation = 0
current_write_failed_operation = 0
current_scan_operation = 0
current_scan_failed_operation = 0
for line in lines:
if '[READ], ' in line and total_read_count == 0:
continue
if '[READ], Operations,' in line:
current_read_operation += int(line.replace('[READ], Operations, ', ''))
elif '[READ], AverageLatency(us), ' in line:
weighted_current_avg = (float(line.replace('[READ], AverageLatency(us), ',
'')) * current_read_operation) / total_read_count
total_read_avg += weighted_current_avg
elif '[READ], 95thPercentileLatency(us), ' in line:
weighted_current_p95 = (float(line.replace('[READ], 95thPercentileLatency(us), ',
'')) * current_read_operation) / total_read_count
total_read_p95 += weighted_current_p95
elif '[READ], 99thPercentileLatency(us), ' in line:
weighted_current_p99 = (float(line.replace('[READ], 99thPercentileLatency(us), ',
'')) * current_read_operation) / total_read_count
total_read_p99 += weighted_current_p99
elif '[READ], MinLatency(us), ' in line:
weighted_current_min = int(line.replace('[READ], MinLatency(us), ',
''))
if total_read_min > weighted_current_min:
total_read_min = weighted_current_min
elif '[READ], MaxLatency(us), ' in line:
weighted_current_max = int(line.replace('[READ], MaxLatency(us), ',
''))
if total_read_max < weighted_current_max:
total_read_max = weighted_current_max
elif '[READ-FAILED], ' in line and total_read_failed_count == 0:
continue
elif '[READ-FAILED], Operations,' in line:
current_read_failed_operation += int(line.replace('[READ-FAILED], Operations, ', ''))
elif '[READ-FAILED], AverageLatency(us), ' in line:
weighted_current_avg = (float(line.replace('[READ-FAILED], AverageLatency(us), ',
'')) * current_read_failed_operation) / total_read_failed_count
total_read_failed_avg += weighted_current_avg
elif '[READ-FAILED], 95thPercentileLatency(us), ' in line:
weighted_current_p95 = (float(line.replace('[READ-FAILED], 95thPercentileLatency(us), ',
'')) * current_read_failed_operation) / total_read_failed_count
total_read_failed_p95 += weighted_current_p95
elif '[READ-FAILED], 99thPercentileLatency(us), ' in line:
weighted_current_p99 = (float(line.replace('[READ-FAILED], 99thPercentileLatency(us), ',
'')) * current_read_failed_operation) / total_read_failed_count
total_read_failed_p99 += weighted_current_p99
elif '[READ-FAILED], MinLatency(us), ' in line:
weighted_current_min = int(line.replace('[READ-FAILED], MinLatency(us), ',
''))
if total_read_failed_min > weighted_current_min:
total_read_failed_min = weighted_current_min
elif '[READ-FAILED], MaxLatency(us), ' in line:
weighted_current_max = int(line.replace('[READ-FAILED], MaxLatency(us), ',
''))
if total_read_failed_max < weighted_current_max:
total_read_failed_max = weighted_current_max
elif '[UPDATE], ' in line and total_update_count == 0:
continue
elif '[UPDATE], Operations,' in line:
current_update_operation += int(line.replace('[UPDATE], Operations, ', ''))
elif '[UPDATE], AverageLatency(us), ' in line:
weighted_current_avg = (float(line.replace('[UPDATE], AverageLatency(us), ',
'')) * current_update_operation) / total_update_count
total_update_avg += weighted_current_avg
elif '[UPDATE], 95thPercentileLatency(us), ' in line:
weighted_current_p95 = (float(line.replace('[UPDATE], 95thPercentileLatency(us), ',
'')) * current_update_operation) / total_update_count
total_update_p95 += weighted_current_p95
elif '[UPDATE], 99thPercentileLatency(us), ' in line:
weighted_current_p99 = (float(line.replace('[UPDATE], 99thPercentileLatency(us), ',
'')) * current_update_operation) / total_update_count
total_update_p99 += weighted_current_p99
elif '[UPDATE], MinLatency(us), ' in line:
weighted_current_min = int(line.replace('[UPDATE], MinLatency(us), ',
''))
if total_update_min > weighted_current_min:
total_update_min = weighted_current_min
elif '[UPDATE], MaxLatency(us), ' in line:
weighted_current_max = int(line.replace('[UPDATE], MaxLatency(us), ',
''))
if total_update_max < weighted_current_max:
total_update_max = weighted_current_max
elif '[UPDATE-FAILED], ' in line and total_update_failed_count == 0:
continue
elif '[UPDATE-FAILED], Operations,' in line:
current_update_failed_operation += int(line.replace('[UPDATE-FAILED], Operations, ', ''))
elif '[UPDATE-FAILED], AverageLatency(us), ' in line:
weighted_current_avg = (float(line.replace('[UPDATE-FAILED], AverageLatency(us), ',
'')) * current_update_failed_operation) / total_update_failed_count
total_update_failed_avg += weighted_current_avg
elif '[UPDATE-FAILED], 95thPercentileLatency(us), ' in line:
weighted_current_p95 = (float(line.replace('[UPDATE-FAILED], 95thPercentileLatency(us), ',
'')) * current_update_failed_operation) / total_update_failed_count
total_update_failed_p95 += weighted_current_p95
elif '[UPDATE-FAILED], 99thPercentileLatency(us), ' in line:
weighted_current_p99 = (float(line.replace('[UPDATE-FAILED], 99thPercentileLatency(us), ',
'')) * current_update_failed_operation) / total_update_failed_count
total_update_failed_p99 += weighted_current_p99
elif '[UPDATE-FAILED], MinLatency(us), ' in line:
weighted_current_min = int(line.replace('[UPDATE-FAILED], MinLatency(us), ',
''))
if total_update_failed_min > weighted_current_min:
total_update_failed_min = weighted_current_min
elif '[UPDATE-FAILED], MaxLatency(us), ' in line:
weighted_current_max = int(line.replace('[UPDATE-FAILED], MaxLatency(us), ',
''))
if total_update_failed_max < weighted_current_max:
total_update_failed_max = weighted_current_max
elif '[INSERT], ' in line and total_write_count == 0:
continue
elif '[INSERT], Operations,' in line:
current_write_operation += int(line.replace('[INSERT], Operations, ', ''))
elif '[INSERT], AverageLatency(us), ' in line:
weighted_current_avg = (float(line.replace('[INSERT], AverageLatency(us), ',
'')) * current_write_operation) / total_write_count
total_write_avg += weighted_current_avg
elif '[INSERT], 95thPercentileLatency(us), ' in line:
weighted_current_p95 = (float(line.replace('[INSERT], 95thPercentileLatency(us), ',
'')) * current_write_operation) / total_write_count
total_write_p95 += weighted_current_p95
elif '[INSERT], 99thPercentileLatency(us), ' in line:
weighted_current_p99 = (float(line.replace('[INSERT], 99thPercentileLatency(us), ',
'')) * current_write_operation) / total_write_count
total_write_p99 += weighted_current_p99
elif '[INSERT], MinLatency(us), ' in line:
weighted_current_min = int(line.replace('[INSERT], MinLatency(us), ',
''))
if total_write_min > weighted_current_min:
total_write_min = weighted_current_min
elif '[INSERT], MaxLatency(us), ' in line:
weighted_current_max = int(line.replace('[INSERT], MaxLatency(us), ',
''))
if total_write_max < weighted_current_max:
total_write_max = weighted_current_max
elif '[INSERT-FAILED], ' in line and total_write_failed_count == 0:
continue
elif '[INSERT-FAILED], Operations,' in line:
current_write_failed_operation += int(line.replace('[INSERT-FAILED], Operations, ', ''))
elif '[INSERT-FAILED], AverageLatency(us), ' in line:
weighted_current_avg = (float(line.replace('[INSERT-FAILED], AverageLatency(us), ',
'')) * current_write_failed_operation) / total_write_failed_count
total_write_failed_avg += weighted_current_avg
elif '[INSERT-FAILED], 95thPercentileLatency(us), ' in line:
weighted_current_p95 = (float(line.replace('[INSERT-FAILED], 95thPercentileLatency(us), ',
'')) * current_write_failed_operation) / total_write_failed_count
total_write_failed_p95 += weighted_current_p95
elif '[INSERT-FAILED], 99thPercentileLatency(us), ' in line:
weighted_current_p99 = (float(line.replace('[INSERT-FAILED], 99thPercentileLatency(us), ',
'')) * current_write_failed_operation) / total_write_failed_count
total_write_failed_p99 += weighted_current_p99
elif '[INSERT-FAILED], MinLatency(us), ' in line:
weighted_current_min = int(line.replace('[INSERT-FAILED], MinLatency(us), ',
''))
if total_write_failed_min > weighted_current_min:
total_write_failed_min = weighted_current_min
elif '[INSERT-FAILED], MaxLatency(us), ' in line:
weighted_current_max = int(line.replace('[INSERT-FAILED], MaxLatency(us), ',
''))
if total_write_failed_max < weighted_current_max:
total_write_failed_max = weighted_current_max
elif '[SCAN], ' in line and total_scan_count == 0:
continue
elif '[SCAN], Operations,' in line:
current_scan_operation += int(line.replace('[SCAN], Operations, ', ''))
elif '[SCAN], AverageLatency(us), ' in line:
weighted_current_avg = (float(line.replace('[SCAN], AverageLatency(us), ',
'')) * current_scan_operation) / total_scan_count
total_scan_avg += weighted_current_avg
elif '[SCAN], 95thPercentileLatency(us), ' in line:
weighted_current_p95 = (float(line.replace('[SCAN], 95thPercentileLatency(us), ',
'')) * current_scan_operation) / total_scan_count
total_scan_p95 += weighted_current_p95
elif '[SCAN], 99thPercentileLatency(us), ' in line:
weighted_current_p99 = (float(line.replace('[SCAN], 99thPercentileLatency(us), ',
'')) * current_scan_operation) / total_scan_count
total_scan_p99 += weighted_current_p99
elif '[SCAN], MinLatency(us), ' in line:
weighted_current_min = int(line.replace('[SCAN], MinLatency(us), ',
''))
if total_scan_min > weighted_current_min:
total_scan_min = weighted_current_min
elif '[SCAN], MaxLatency(us), ' in line:
weighted_current_max = int(line.replace('[SCAN], MaxLatency(us), ',
''))
if total_scan_max < weighted_current_max:
total_scan_max = weighted_current_max
elif '[SCAN-FAILED], ' in line and total_scan_failed_count == 0:
continue
elif '[SCAN-FAILED], Operations,' in line:
current_scan_failed_operation += int(line.replace('[SCAN-FAILED], Operations, ', ''))
elif '[SCAN-FAILED], AverageLatency(us), ' in line:
weighted_current_avg = (float(line.replace('[SCAN-FAILED], AverageLatency(us), ',
'')) * current_scan_failed_operation) / total_scan_failed_count
total_scan_failed_avg += weighted_current_avg
elif '[SCAN-FAILED], 95thPercentileLatency(us), ' in line:
weighted_current_p95 = (float(line.replace('[SCAN-FAILED], 95thPercentileLatency(us), ',
'')) * current_scan_failed_operation) / total_scan_failed_count
total_scan_failed_p95 += weighted_current_p95
elif '[SCAN-FAILED], 99thPercentileLatency(us), ' in line:
weighted_current_p99 = (float(line.replace('[SCAN-FAILED], 99thPercentileLatency(us), ',
'')) * current_scan_failed_operation) / total_scan_failed_count
total_scan_failed_p99 += weighted_current_p99
elif '[SCAN-FAILED], MinLatency(us), ' in line:
weighted_current_min = int(line.replace('[SCAN-FAILED], MinLatency(us), ',
''))
if total_scan_failed_min > weighted_current_min:
total_scan_failed_min = weighted_current_min
elif '[SCAN-FAILED], MaxLatency(us), ' in line:
weighted_current_max = int(line.replace('[SCAN-FAILED], MaxLatency(us), ',
''))
if total_scan_failed_max < weighted_current_max:
total_scan_failed_max = weighted_current_max
# create the csv writer
output_csv = open("aggregation" + ".csv", 'w', newline='')
writer = csv.writer(output_csv)
header = ['Operation', 'Count', 'Throughput', 'Min(microsecond)', 'Max(microsecond)', 'Avg(microsecond)',
'P95(microsecond)', 'P99(microsecond)']
writer.writerow(header)
if total_read_count > 0:
row_in_csv = ['READ', total_read_count, int(total_throughput), total_read_min, total_read_max,
int(total_read_avg), int(total_read_p95),
int(total_read_p99)]
writer.writerow(row_in_csv)
if total_read_failed_count > 0:
row_in_csv = ['READ-FAILED', total_read_failed_count, int(total_throughput), total_read_failed_min,
total_read_failed_max,
int(total_read_failed_avg),
int(total_read_failed_p95), int(total_read_failed_p99)]
writer.writerow(row_in_csv)
if total_update_count > 0:
row_in_csv = ['UPDATE', total_update_count, int(total_throughput), total_update_min, total_update_max,
int(total_update_avg), int(total_update_p95),
int(total_update_p99)]
writer.writerow(row_in_csv)
if total_update_failed_count > 0:
row_in_csv = ['UPDATE-FAILED', total_update_failed_count, int(total_throughput), total_update_failed_min,
total_update_failed_max, int(total_update_failed_avg),
int(total_update_failed_p95), int(total_update_failed_p99)]
writer.writerow(row_in_csv)
if total_write_count > 0:
row_in_csv = ['WRITE', total_write_count, int(total_throughput), total_write_min, total_write_max,
int(total_write_avg), int(total_write_p95),
int(total_write_p99)]
writer.writerow(row_in_csv)
if total_write_failed_count > 0:
row_in_csv = ['WRITE-FAILED', total_write_failed_count, int(total_throughput), total_write_failed_min,
total_write_failed_max, int(total_write_failed_avg),
int(total_write_failed_p95), int(total_write_failed_p99)]
writer.writerow(row_in_csv)
if total_scan_count > 0:
row_in_csv = ['SCAN', total_scan_count, int(total_throughput), total_scan_min, total_scan_max,
int(total_scan_avg), int(total_scan_p95),
int(total_scan_p99)]
writer.writerow(row_in_csv)
if total_scan_failed_count > 0:
row_in_csv = ['SCAN-FAILED', total_scan_failed_count, int(total_throughput), total_scan_failed_min,
total_scan_failed_max, int(total_scan_failed_avg),
int(total_scan_failed_p95), int(total_scan_failed_p99)]
writer.writerow(row_in_csv)
output_csv.close()
print("Successfully created " + output_csv.name)