in tools/scraper/parser.py [0:0]
def parse_log_file(fn, log_index, comn):
"""
Given a file name, return an array of Routers that hold the parsed lines.
Lines that don't parse are identified on stderr and then discarded.
:param fn: file name
:param log_index: router id 0 for 'A', 1 for 'B', ...
:param comn: common data
:return: list of Routers
"""
instance = 0
lineno = 0
search_for_in_progress = True
rtrs = []
rtr = None
key1 = "SERVER (trace) [" # AMQP traffic
key2 = "SERVER (info) Container Name:" # Normal 'router is starting' restart discovery line
key3 = "ROUTER_LS (info)" # a log line placed in separate pool of lines
keys = [key1, key3]
key4 = "ROUTER (info) Version:" # router version line
key5 = "ROUTER (info) Router started in " # router mode
with open(fn, 'r') as infile:
for line in infile:
if search_for_in_progress:
# What if the log file has no record of the router starting?
# This is an in_progress router and it is a pre-existing router instance
# and not one found by restart discovery.
# Any key or AMQP line indicates a router in-progress
if any(s in line for s in keys) or ("[" in line and "]" in line):
assert rtr is None
rtr = router.Router(fn, log_index, instance)
rtrs.append(rtr)
search_for_in_progress = False
rtr.restart_rec = router.RestartRecord(rtr, line, lineno + 1)
lineno += 1
verbatim_module = None
if len(comn.verbatim_include_list) > 0:
for modx in comn.verbatim_include_list:
if comn.module_key_in_line(modx, line):
verbatim_module = modx
break
if key2 in line:
# This line closes the current router, if any, and opens a new one
if rtr is not None:
instance += 1
rtr = router.Router(fn, log_index, instance)
rtrs.append(rtr)
rtr.restart_rec = router.RestartRecord(rtr, line, lineno)
search_for_in_progress = False
rtr.container_name = line[(line.find(key2) + len(key2)):].strip().split()[0]
elif key3 in line:
pl = None
try:
pl = ParsedLogLine(log_index, instance, lineno, line, comn, rtr)
except ValueError as ve:
pass
except Exception as e:
# t, v, tb = sys.exc_info()
if hasattr(e, 'message'):
traceback.print_exc()
sys.stderr.write("Failed to parse file '%s', line %d : %s. Analysis continuing...\n" % (fn, lineno, e.message))
else:
traceback.print_exc()
sys.stderr.write("Failed to parse file '%s', line %d : %s. Analysis continuing...\n" % (fn, lineno, e))
if pl is not None:
if pl.data.is_router_ls:
rtr.router_ls.append(pl)
elif key4 in line:
rtr.version = line[(line.find(key4) + len(key4)):].strip().split()[0]
elif key5 in line:
rtr.mode = line[(line.find(key5) + len(key5)):].strip().split()[0].lower()
elif verbatim_module is not None:
pl = ParsedLogLine(log_index, instance, lineno, line, comn, rtr)
rtr.lines.append(pl)
elif "[" in line and "]" in line:
try:
do_this = True if not hasattr(comn.args, 'skip_all_data') else not comn.args.skip_all_data
if not do_this:
# not indexing data. maybe do this line anyway
do_this = not any(s in line for s in [' @transfer', ' @disposition', ' @flow', 'EMPTY FRAME'])
if do_this:
pl = ParsedLogLine(log_index, instance, lineno, line, comn, rtr)
if pl is not None:
rtr.lines.append(pl)
else:
comn.data_skipped += 1
except ValueError as ve:
pass
except Exception as e:
# t, v, tb = sys.exc_info()
if hasattr(e, 'message'):
traceback.print_exc()
sys.stderr.write("Failed to parse file '%s', line %d : %s. Analysis continuing...\n" % (fn, lineno, e.message))
else:
traceback.print_exc()
sys.stderr.write("Failed to parse file '%s', line %d : %s. Analysis continuing...\n" % (fn, lineno, e))
# raise t, v, tb
else:
# ignore this log line
pass
return rtrs