def process_log_file()

in compat-tool/compat/compat.py [0:0]

• 74 lines of code
• 19 McCabe index (conditional complexity)

def process_log_file(ver, fname, unsupported_fname, unsupported_query_fname, skipped_line_fname):
    unsupported_file = open(unsupported_fname, "w")
    unsupported_query_file = open(unsupported_query_fname, "w")
    skipped_line_file = open(skipped_line_fname, "w")
    usage_map = {}
    cmd_map = {}
    line_ct = 0
    unsupported_ct = 0
    truncated_line_ct = 0
    unrecognized_line_ct = 0
    parse_exception_ct = 0
    totalLines = 0
    fileArray = []
    if os.path.isfile(fname):
        fileArray.append(fname)
    else:
        for fileName in os.listdir(fname):
            fileArray.append(os.path.join(fname,fileName))
    for thisFile in fileArray:
        print("processing file {}".format(thisFile))
        lineNum = 0
        with open(thisFile) as log_file:
            for line in log_file:
                totalLines += 1
                lineNum += 1
                if (lineNum % 10000 == 0):
                    # display log file progress
                    print(".. line {}".format(lineNum))
                if ("warning: log line attempted" in line) and ("over max size" in line) and ("printing beginning and end" in line):
                    truncated_line_ct += 1
                    skipped_line_file.write("fname = {} | line number = {} | reason = {}\n".format(thisFile,lineNum,"truncated line"))
                else:
                    le = logevent.LogEvent(line)
                    if (le.datetime is None):
                        unrecognized_line_ct += 1
                        skipped_line_file.write("fname = {} | line number = {} | reason = {}\n".format(thisFile,lineNum,"unrecognized line"))
                        skipped_line_file.write("   Actual Line | {}\n".format(le))
                        #raise SystemExit("Error: <%s> does not appear to be a supported MongoDB log file format" % thisFile)
                    else:
                        pl = process_line(le, usage_map, ver, cmd_map, lineNum)
                        line_ct += pl["processed"]
                        #print("{} | {}".format(pl,le))
                        if ("exception" in pl and pl["exception"]):
                            parse_exception_ct += 1
                            skipped_line_file.write("fname = {} | line number = {} | reason = {}\n".format(thisFile,lineNum,"invalid JSON"))
                        elif (pl["unsupported"]):
                            unsupported_file.write(pl["logevent"].line_str)
                            unsupported_file.write("\n")
                            unsupported_query_file.write(f'{pl["actual_query"]}  // {pl["unsupported_keys"]}\n')
                            unsupported_ct += 1
    unsupported_file.close()
    unsupported_query_file.close()
    skipped_line_file.close()

    print("")
    print('Results:')
    print("  Total lines processed = {}".format(totalLines))
    print("")
    if (truncated_line_ct > 0 or unrecognized_line_ct > 0 or parse_exception_ct > 0):
        print("**NOTE** - portions of the log file(s) processed were truncated or incorrectly formatted and excluded from the compatibility assessment")
        print("  Skipped {} log lines due to log line truncation (default 10KB, consider increasing maxLogSizeKB)".format(truncated_line_ct))
        print("  Skipped {} log lines due to unrecognized log format (missing timestamp)".format(unrecognized_line_ct))
        print("  Skipped {} log lines due to unusable log format (invalid JSON)".format(parse_exception_ct))
    print("")

    if (unsupported_ct > 0):
        print(f'{unsupported_ct} out of {line_ct} queries unsupported')
        print(f'Unsupported operators (and number of queries used):')
        for k,v in sorted(usage_map.items(), key=lambda x: (-x[1],x[0])):
            print(f'\t{k:20}  {v}')
    else:
        print('All queries are supported')
    print("")

    print('Query Types:')
    for k,v in sorted(cmd_map.items(), key=lambda x: (-x[1],x[0])):
        print(f'\t{k:10}  {v}')
    print(f'Log lines of unsupported operators logged here: {unsupported_fname}')
    print(f'Queries of unsupported operators logged here: {unsupported_query_fname}')
    print(f'Skipped line information logged here: {skipped_line_fname}')