in crashes.py [0:0]
def generateTopCrashReport(reports, stats, totalCrashesProcessed, parameters, ipcActorName, outputFilename, annoFilename, reportLowerClientLimit):
processType = parameters['process_type']
channel = parameters['channel']
queryFxVersion = parameters['version']
templateFile = open("template.html", "r")
template = templateFile.read()
templateFile.close()
# <!-- start of crash template -->
# <!-- end of crash template -->
innerTemplate, mainPage = extractAndTokenizeTemplate('crash', template, 'main')
annotationTemplate, mainPage = extractAndTokenizeTemplate('annotation', mainPage, 'annotations')
annotationReport, annotationTemplate = extractAndTokenizeTemplate('annotation report', annotationTemplate, 'annreports')
# <!-- start of signature template -->
# <!-- end of signature template -->
innerSigTemplate, outerSigTemplate = extractAndTokenizeTemplate('signature', innerTemplate, 'signature')
# Main inner block
# <!-- start of signature meta template -->
# <!-- end of signature meta template -->
innerSigMetaTemplate, outerSigMetaTemplate = extractAndTokenizeTemplate('signature meta', innerSigTemplate, 'reports')
# Report meta plus stack info
# <!-- start of report template -->
# <!-- end of report template -->
innerReportTemplate, outerReportTemplate = extractAndTokenizeTemplate('report', innerSigMetaTemplate, 'report')
# <!-- start of stackline template -->
# <!-- end of stackline template -->
innerStackTemplate, outerStackTemplate = extractAndTokenizeTemplate('stackline', innerReportTemplate, 'stackline')
outerStackTemplate = stripWhitespace(outerStackTemplate)
innerStackTemplate = stripWhitespace(innerStackTemplate)
outerReportTemplate = stripWhitespace(outerReportTemplate)
outerSigMetaTemplate = stripWhitespace(outerSigMetaTemplate)
outerSigTemplate = stripWhitespace(outerSigTemplate)
annotationTemplate = stripWhitespace(annotationTemplate)
annotationReport = stripWhitespace(annotationReport)
# mainPage = stripWhitespace(mainPage) # mucks with js
annDb = loadAnnotations(annoFilename)
#resultFile = open(("%s.html" % outputFilename), "w", encoding="utf-8")
resultFile = open(("%s.html" % outputFilename), "w", errors="replace")
signatureHtml = str()
sigMetaHtml = str()
annotationsHtml = str()
signatureIndex = 0
sigCount, reportCount = getDatasetStats(reports)
# generate a top crash list
sigCounter = Counter()
for hash in reports:
if reports[hash]['clientcount'] < reportLowerClientLimit:
continue
sigCounter[hash] = len(reports[hash]['reportList'])
collection = sigCounter.most_common(MostCommonLength)
sparklineJS = ''
for hash, crashcount in collection:
try:
sigRecord = reports[hash] # reports data vs. stats
except KeyError:
continue
signature = sigRecord['signature']
statsCrashData = stats[hash]['crashdata']
prettyOperatingSystems, prettyOperatingSystemVers, prettyArchs = getPrettyPlatformLists(statsCrashData)
prettyFirefoxVers = getPrettyFirefoxVersionList(statsCrashData, channel)
operatingSystemsList, operatingSystemVersList, archsList = getPlatformDataFromStatsRec(statsCrashData)
firefoxVersList = getFxVersionsFromStatsRec(statsCrashData)
crashcount = len(sigRecord['reportList'])
percent = (crashcount / reportCount)*100.0
if crashcount < MinCrashCount: # Skip small crash count reports
continue
isNewCrash = False
newIcon = 'noicon'
if testIfNewCrash(statsCrashData, queryFxVersion):
isNewCrash = True
newIcon = 'icon'
signatureIndex += 1
crashStatsHashQuery = 'https://crash-stats.mozilla.org/search/?'
crashStatsQuery = 'https://crash-stats.mozilla.org/search/?signature=~%s&product=Firefox&_facets=signature&process_type=%s' % (signature, processType)
# sort reports in this signature based on common crash reasons, so the most common
# is at the top of the list.
reportsToReport = generateTopReportsList(reports[hash]['reportList'])
#fissionIcon = 'noicon'
#if isFissionRelated(reports[hash]['reportList']):
# fissionIcon = 'icon'
#if crashcount < 10 and fissionIcon == 'icon':
# fissionIcon = 'grayicon'
lockdownIcon = 'noicon'
if isLockdownRelated(reports[hash]['reportList']):
lockdownIcon = 'icon'
reportHtml = str()
idx = 0
hashTotal= 0
oomIcon = 'noicon'
for report in reportsToReport:
idx = idx + 1
if idx > MaxReportCount:
break
oombytes = report['oomsize']
if report['oomsize'] is not None:
oomIcon = 'icon'
else:
oombytes = ''
crashReason = report['crashreason']
if (crashReason == None):
crashReason = ''
crashType = report['type']
crashType = crashType.replace('EXCEPTION_', '')
appendAmp = False
if hashTotal < 30: # This is all crash stats can hande (414 Request-URI Too Large)
try:
crashStatsHashQuery += 'minidump_sha256_hash=~' + report['minidumphash']
hashTotal += 1
appendAmp = True
except:
pass
# Redash meta data dump for a particular crash id
infoLink = "https://sql.telemetry.mozilla.org/queries/{query_id}?p_channel={channel}&p_process_type={process_type}&p_version={version}&p_crash_id={crash_id}".format(query_id=79462, channel=channel, process_type=processType, version=queryFxVersion, crash_id=report['crashid'])
startupStyle = 'noicon'
if report['startup'] != 0:
startupStyle = 'icon'
stackHtml = str()
for frameData in report['stack']:
# [idx] = { 'index': n, 'frame': '(frame)', 'srcUrl': '(url)', 'module': '(module)' }
frameIndex = frameData['index']
frame = frameData['frame']
srcUrl = frameData['srcUrl']
moduleName = frameData['module']
linkStyle = 'inline-block'
srcLink = srcUrl
if len(srcUrl) == 0:
linkStyle = 'none'
srcLink = ''
stackHtml += Template(innerStackTemplate).substitute(frameindex=frameIndex,
frame=escape(frame),
srcurl=srcLink,
module=moduleName,
style=linkStyle)
compositor = report['compositor']
if compositor == 'webrender_software_d3d11':
compositor = 'd3d11'
elif compositor == 'webrender':
compositor = 'webrender'
elif compositor == 'webrender_software':
compositor = 'swiggle'
elif compositor == 'none':
compositor = ''
reportHtml += Template(outerStackTemplate).substitute(expandostack=('st'+str(signatureIndex)+'-'+str(idx)),
rindex=idx,
type=crashType,
oomsize=oombytes,
devvendor=report['devvendor'],
devgen=report['devgen'],
devchipset=report['devchipset'],
description=report['devdescription'],
drvver=report['driverversion'],
drvdate=report['driverdate'],
compositor=compositor,
reason=crashReason,
infolink=infoLink,
startupiconclass=startupStyle,
stackline=stackHtml)
if appendAmp:
crashStatsHashQuery += '&'
# class="svg-$expandosig"
sparklineJS += generateSparklineJS(stats[hash], operatingSystemsList, operatingSystemVersList, firefoxVersList, archsList, 'svg-'+stringToHtmlId(hash)) + '\n'
# svg element
sigHtml = Template(outerReportTemplate).substitute(expandosig=stringToHtmlId(hash),
os=prettyOperatingSystems,
fxver=prettyFirefoxVers,
osver=prettyOperatingSystemVers,
arch=prettyArchs,
report=reportHtml)
crashStatsHashQuery = crashStatsHashQuery.rstrip('&')
searchIconClass = 'icon'
if hashTotal == 0:
crashStatsHashQuery = ''
searchIconClass = 'lticon'
# ann$expandosig - view signature meta parameter
annIconClass = 'lticon'
if signature in annDb:
record = annDb[signature]
# record['annotations'] { date: 'date', 'annotation': 'notes' }
sigAnnotations = str()
# record['fixedby'] (list of tables, { date: 'date', 'version': 87, 'bug': 1234567 }
for fb in record['fixedby']:
sigAnnotations += Template(annotationReport).substitute(annotations=escape(fb['annotation']),
fixedbybug=createBugLink(str(fb['bug'])),
fixedbyversion=fb['version'])
for annotation in record['annotations']:
annotation = escape(annotation['annotation'])
annotation = escapeBugLinks(annotation)
sigAnnotations += Template(annotationReport).substitute(annotations=annotation, fixedbybug='', fixedbyversion='')
annotationsHtml += Template(annotationTemplate).substitute(expandosig=('sig'+str(signatureIndex)),
annreports=sigAnnotations)
annIconClass = 'icon'
sigMetaHtml += Template(outerSigMetaTemplate).substitute(rank=signatureIndex,
percent=("%.00f%%" % percent),
# expandosig=('sig'+str(signatureIndex)),
expandosig=stringToHtmlId(hash),
annexpandosig=('sig'+str(signatureIndex)),
signature=(html.escape(signature)),
newicon=newIcon,
fissionicon='noicon',
lockicon=lockdownIcon,
oomicon=oomIcon,
iconclass=searchIconClass,
anniconclass=annIconClass,
cslink=crashStatsHashQuery,
cssearchlink=crashStatsQuery,
clientcount=sigRecord['clientcount'],
count=crashcount,
reports=sigHtml)
if ipcActorName:
ipcActorHdr = '<div class="header-elements">IPC Actor - {}</div>'.format(ipcActorName)
else:
ipcActorHdr = ""
signatureHtml += Template(outerSigTemplate).substitute(channel=channel,
version=queryFxVersion,
process=processType,
sigcount=sigCount,
ipcActorHdr=ipcActorHdr,
repcount=reportCount,
sparkline=sparklineJS,
signature=sigMetaHtml)
# Add processed date to the footer
dateTime = datetime.now().isoformat()
processHead = "{}".format(processType.capitalize())
if ipcActorName:
processHead += " ({})".format(ipcActorName)
resultFile.write(Template(mainPage).substitute(main=signatureHtml,
annotations=annotationsHtml,
process=processHead,
processeddate=dateTime))
resultFile.close()