in foundation_security_advisories/update_hof.py [0:0]
def main():
args = command_line()
# Do this first so we error immediately if the file isn't there.
with open(os.path.abspath(args.output), 'r') as f:
file_data = f.read()
f.close()
hmackey = get_hmac_key(args.apikey)
debuglogfilename = 'debuglog.' + str(int(time.time())) + '.log'
debuglog = open(debuglogfilename, 'w')
print("Okay, we're going to start. Have you assigned hof+ to the eligible bugs with bounty- and CVEs?")
print("The link for those for you to review is:")
print(" ", CVE_REVIEW.replace("YEAR_REPLACEME", args.year))
print("If you forgot to do this, kill the script, do it, and start it again.")
print("I'm logging to", debuglogfilename)
bugs = gather_bug_list(args.apikey)
hof_entries = []
(begin_date, end_date) = define_dates(args.quarter, args.year)
print("Generating Bug Data from " + str(len(bugs["bugs"])) + " bugs")
num_processed = 0
for bug in bugs["bugs"]:
bugid = str(bug["id"])
debuglog.write(bugid + ",")
debuglog.write(bug['classification'] + ",")
debuglog.write(bug['product'] + ",")
debuglog.write(bug['component'] + ",")
num_processed += 1
#if num_processed % 100 == 0:
print("Processed", num_processed, "of", len(bugs["bugs"]), "currently on", bugid)
if args.client and not is_client_bug(bug):
debuglog.write("wrong product: " + bug['product'] + "\n")
continue
elif args.web and is_client_bug(bug):
debuglog.write("wrong product: " + bug['product'] + "\n")
continue
data ={'bugid':bugid}
# ==========================================================================================
# Look for the bug bounty attachment first
attachment_url = BASE_URL + 'bug/' + bugid + '/attachment'
try:
attachments = http.get(attachment_url, headers=HEADERS, params={'api_key' : args.apikey}).json()['bugs'][bugid]
except Exception as e:
print ("Error in " + bugid)
print (e)
continue
# attachment_breakout[0] = email
# attachment_breakout[1] = paid
# attachment_breakout[2] = reported date
# attachment_breakout[3] = fixed date
# attachment_breakout[4] = awareded date
# attachment_breakout[5] = publish (true/false)
# attachment_breakout[6] = credit
# attachment_breakout[7] = twitter
# attachment_breakout[8] = url
foundAttachment = None
for attachment in attachments:
if attachment['file_name'] == 'bugbounty.data' and attachment['is_private'] == 1:
if foundAttachment and bugid not in ["913805"]:
raise Exception("Two bug bounty attachments were found for " + bugid)
foundAttachment = attachment
if foundAttachment:
# Old bugs with weird formats
if bugid in ['1074642']:
continue
try:
attachment = foundAttachment
attachment_breakout = [x.strip() for x in attachment['description'].split(',')]
data["email"] = attachment_breakout[0]
data["email_hmac"] = hmac_email(hmackey, attachment_breakout[0])
data["date_raw"] = attachment_breakout[4] or attachment_breakout[3] or attachment_breakout[2]
data["date"] = datetime.strptime(data["date_raw"], '%Y-%m-%d')
debuglog.write("bounty+," + data["date_raw"] + "," + data["email"] + "," + data["email_hmac"] + ",")
if begin_date < data["date"] < end_date:
if "@mozilla.com" in data["email"] and mozilla_email_was_employed(data["email"], data["date"]):
debuglog.write("Mozilla Employee in a Bug Bounty Attachment??\n")
# Don't add Mozilla employees filing bugs under their work email to the HOF
continue
# print("Generating Data For Bug %s - %s" % (bugid, data["email"]))
numFields = len(attachment_breakout)
if not bool(attachment_breakout[5]):
debuglog.write("Publish was blank,")
elif "no" == attachment_breakout[5].lower() or "false" == attachment_breakout[5].lower():
debuglog.write("Do Not Publish\n")
continue
else:
debuglog.write("publish,")
data["name"] = ""
if numFields > 6 and attachment_breakout[6]:
data["name"] = attachment_breakout[6]
if "[paid]" in data["name"]:
data["name"] = ""
elif "no response" in data["name"]:
data["name"] = ""
elif data["name"][0] == '"' and data["name"][-1] == '"':
data["name"] = data["name"][1:-1]
if data["name"] and data["email_hmac"] not in credit_entries and \
check_add_credit_to_script(data["email"], data["name"]):
add_credit_to_script(hmackey, data["email"], data["name"])
if not data["name"] and data["email_hmac"] in credit_entries:
data["name"] = credit_entries[data["email_hmac"]]
elif not data["name"]:
user_url = BASE_URL + 'user?names=' + data["email"]
try:
user_response = http.get(user_url, headers=HEADERS)
user_response_data = user_response.json()
except requests.exceptions.RequestException as e:
print("Could not get user data for " + user_url)
print(e)
sys.exit(1)
if user_response.status_code == 200 and user_response_data['users'][0]["real_name"]:
data["name"] = user_response_data['users'][0]["real_name"]
else:
data["name"] = data["email"].split('@', 1)[0]
print("Had to use fallback name for", bugid, data["email"], data["name"])
data["quarter-string"] = data["date_raw"][0:4] + month_to_quarter(data["date_raw"][5:7])
if data["email_hmac"] in twitter_entries:
data["twitter"] = twitter_entries[data["email_hmac"]]
elif numFields > 7 and attachment_breakout[7]:
data["twitter"] = attachment_breakout[7]
add_twitter_to_script(hmackey, data["email"], data["twitter"])
if data["email_hmac"] in url_entries:
data["url"] = url_entries[data["email_hmac"]]
elif numFields > 8 and attachment_breakout[8]:
data["url"] = attachment_breakout[8]
add_url_to_script(hmackey, data["email"], data["url"])
if not "url" in data and "twitter" in data:
data["url"] = "https://twitter.com/" + data["twitter"]
hof_entries.append(data)
debuglog.write(data["name"] + "," + (data["url"] if 'url' in data else "") + "\n")
continue # Go to next bug
else:
debuglog.write("Date wasn't in range\n")
continue
except:
debuglog.write("Printed Exception\n")
import traceback
print("--------------------------------------------------------")
print("Could not process %s" % bugid)
print("Attachment field: %s" % attachment['description'])
print("Split fields: %s" % attachment['description'].split(','))
print(traceback.format_exc())
print("--------------------------------------------------------")
continue
# ==========================================================================================
# If we didn't find a bounty attachment, then it's a Hall of Fame Entry
if 'cf_last_resolved' not in bug or not bug['cf_last_resolved']:
# No resolved date, so look for a flag date
hof_date = None
history_url = BASE_URL + 'bug/' + bugid + '/history'
try:
history = http.get(history_url, headers=HEADERS, params={'api_key' : args.apikey}).json()
for h in history['bugs'][0]['history'][::-1]:
for c in h['changes']:
if c['field_name'] == "flagtypes.name" and "sec-bounty-hof+" in c['added']:
hof_date = h['when'].split("T")[0]
if not hof_date:
raise Exception("Could not find a bounty+ date")
data["date_raw"] = hof_date.split("T")[0]
except Exception as e:
print ("Error in " + bugid)
print (e)
continue
else:
# Normal case
data["date_raw"] = bug['cf_last_resolved'].split("T")[0]
data["date"] = datetime.strptime(data["date_raw"], '%Y-%m-%d')
debuglog.write("bounty-," + data["date_raw"] + ",")
if begin_date < data["date"] < end_date:
data["name"] = ""
data["email"] = bug['creator_detail']['email']
data["email_hmac"] = hmac_email(hmackey, data["email"])
debuglog.write(data["email"] + "," + data["email_hmac"] + ",")
if "@mozilla.com" in data["email"] and mozilla_email_was_employed(data["email"], data["date"]):
debuglog.write("Mozilla Employee??\n")
# Don't add Mozilla employees filing bugs under their work email to the HOF
continue
# print("Generating Data For Bug %s - %s" % (bugid, data["email"]))
if data["email_hmac"] in credit_entries:
data["name"] = credit_entries[data["email_hmac"]]
else:
user_url = BASE_URL + 'user?names=' + data["email"]
try:
user_response = http.get(user_url, headers=HEADERS)
user_response_data = user_response.json()
except requests.exceptions.RequestException as e:
print("Could not get user data for " + user_url)
print(e)
sys.exit(1)
if user_response.status_code == 200 and user_response_data['users'][0]["real_name"]:
data["name"] = user_response_data['users'][0]["real_name"]
else:
data["name"] = data["email"].split('@', 1)[0]
print("Had to use fallback name:", bugid, data["email"], data["name"])
data["quarter-string"] = data["date_raw"][0:4] + month_to_quarter(data["date_raw"][5:7])
if data["email_hmac"] in twitter_entries:
data["twitter"] = twitter_entries[data["email_hmac"]]
if data["email_hmac"] in url_entries:
data["url"] = url_entries[data["email_hmac"]]
if not "url" in data and "twitter" in data:
data["url"] = "https://twitter.com/" + data["twitter"]
hof_entries.append(data)
debuglog.write(data["name"] + "," + (data["url"] if 'url' in data else "") + "\n")
continue # Go to next bug (although we're already at the end of the loop.)
else:
debuglog.write("Date wasn't in range\n")
continue
def soryByDate(val):
return val["date"]
hof_entries.sort(key=soryByDate, reverse=True)
oneEntryPerQuarter = set()
hof_output = ""
hof_bugzilla_queryarg = ""
for data in hof_entries:
try:
hof_bugzilla_queryarg += data['bugid'] + ","
thisData = data["name"] + " " + data["quarter-string"]
if thisData in oneEntryPerQuarter:
continue
oneEntryPerQuarter.add(thisData)
hof_output = hof_output + "- name: {}\n".format(data["name"])
hof_output = hof_output + " date: {}\n".format(data["date_raw"])
if "twitter" in data:
hof_output = hof_output + " twitter: \"{}\"\n".format(data["twitter"])
if "url" in data:
hof_output = hof_output + " url: {}\n".format(data["url"])
except Exception as e:
print("Could not write hof entry for ", data["name"])
final_output = file_data[:6] +'\n' + hof_output.rstrip() + file_data[6:]
with open(os.path.abspath(args.output), 'w') as output_file:
output_file.write(final_output)
print("I logged to", debuglogfilename)
print("Here is a bugzilla query showing all the bugs I processed for this quarter. Please note that a reporter _may_ have multiple bugs, so don't remove them from the HOF for a misplaced bug without double checking.")
print("https://bugzilla.mozilla.org/buglist.cgi?quicksearch={0}".format(hof_bugzilla_queryarg))