in mozci/console/commands/push.py [0:0]
def handle(self) -> None:
branch = self.argument("branch")
self.line("<comment>Loading pushes...</comment>")
self.pushes = classify_commands_pushes(
branch,
self.option("from-date"),
self.option("to-date"),
self.option("rev"),
)
option_names = [
name.replace("_", "-")
for name, _ in signature(Push.classify).parameters.items()
if name != "self"
]
if self.option("recalculate"):
classify_parameters = retrieve_classify_parameters(self.option)
elif any(self.option(name) for name in option_names):
self.line(
f"<error>--recalculate isn't set, you shouldn't provide --{', --'.join(option_names)} CLI options.</error>"
)
return
# Progress bar will display time stats & messages
progress = self.progress_bar(len(self.pushes))
progress.set_format(
" %current%/%max% [%bar%] %percent:3s%% %elapsed:6s% %message%"
)
# Setup specific route prefix for existing tasks, according to environment
environment = self.option("environment")
route_prefix = (
"project.mozci.classification"
if environment == "production"
else f"project.mozci.{environment}.classification"
)
self.errors = {}
self.classifications = {}
self.failures = {}
for push in self.pushes:
if self.option("recalculate"):
progress.set_message(f"Calc. {branch} {push.id}")
(
all_pushes,
removed_tasks,
backedoutby,
old_classifications,
) = prepare_for_analysis(push)
try:
self.classifications[push], regressions, _ = push.classify(
**classify_parameters
)
self.failures[push] = {
"real": regressions.real,
"intermittent": regressions.intermittent,
"unknown": regressions.unknown,
}
except Exception as e:
self.line(
f"<error>Classification failed on {branch} {push.rev}: {e}</error>"
)
self.errors[push] = e
for p in all_pushes:
# Once the Mozci algorithm has run, restore Sheriffs classifications to be able to properly compare failures classifications.
for task in p.tasks:
task.classification = old_classifications[p.id][task.id][
"classification"
]
task.classification_note = old_classifications[p.id][task.id][
"note"
]
# Restore backout information.
p.backedoutby = backedoutby[p.id]
# And also restore tasks marked as a backfill or a retrigger.
p.tasks = p.tasks + removed_tasks[p.id]
else:
progress.set_message(f"Fetch {branch} {push.id}")
try:
index = f"{route_prefix}.{branch}.revision.{push.rev}"
task = Task.create(
index=index, root_url=COMMUNITY_TASKCLUSTER_ROOT_URL
)
artifact = task.get_artifact(
"public/classification.json",
root_url=COMMUNITY_TASKCLUSTER_ROOT_URL,
)
self.classifications[push] = PushStatus[
artifact["push"]["classification"]
]
self.failures[push] = artifact["failures"]
except TaskNotFound as e:
self.line(
f"<comment>Taskcluster task missing for {branch} {push.rev}</comment>"
)
self.errors[push] = e
except Exception as e:
self.line(
f"<error>Fetch failed on {branch} {push.rev}: {e}</error>"
)
self.errors[push] = e
warnings = []
# Warn about pushes that are backed-out and where all failures on the push itself and its children are marked as intermittent
if push.backedout or push.bustage_fixed_by:
ever_classified_as_cause = check_ever_classified_as_cause(push, "label")
if not ever_classified_as_cause:
ever_classified_as_cause = check_ever_classified_as_cause(
push, "group"
)
if not ever_classified_as_cause:
warnings.append(
{
"message": f"Push {push.branch}/{push.rev} was backedout and all of its failures and the ones of its children were marked as intermittent or marked as caused by another push.",
"type": "error",
"notify": config.get("warnings", {}).get(
"ever_classified_as_cause", False
),
}
)
for task in push.tasks:
if task.classification != "fixed by commit":
continue
# Warn if there is a classification that references a revision that does not exist
fix_hgmo = HgRev.create(
task.classification_note[:12], branch=push.branch
)
try:
fix_hgmo.changesets
except PushNotFound:
warnings.append(
{
"message": f"Task {task.id} on push {push.branch}/{push.rev} contains a classification that references a non-existent revision: {task.classification_note}.",
"type": "error",
"notify": config.get("warnings", {}).get(
"non_existent_fix", False
),
}
)
continue
if fix_hgmo.pushid <= push.id:
warnings.append(
{
"message": f"Task {task.label} on push {push.branch}/{push.rev} is classified as fixed by {task.classification_note}, which is older than the push itself.",
"type": "error",
"notify": config.get("warnings", {}).get(
"fix_older_than_push", False
),
}
)
continue
# Warn when a failure is classified as fixed by a backout of a push that is newer than the failure itself
all_backedouts = set(
backedout
for backedouts in fix_hgmo.backouts.values()
for backedout in backedouts
)
all_bustagefixed = set()
for child in push._iterate_children():
if child.rev == fix_hgmo.node:
break
for bug in child.bugs:
if bug in fix_hgmo.bugs_without_backouts:
all_bustagefixed.add(child.rev)
all_fixed = all_backedouts | all_bustagefixed
if len(all_fixed) > 0 and all(
HgRev.create(backedout, branch=push.branch).pushid > push.id
for backedout in all_fixed
):
warnings.append(
{
"message": f"Task {task.label} on push {push.branch}/{push.rev} is classified as fixed by a backout/bustage fix ({fix_hgmo.node}) of pushes ({all_fixed}) that come after the failure itself.",
"type": "error",
"notify": config.get("warnings", {}).get(
"backout_of_newer_pushes", False
),
}
)
# Warn when there are inconsistent classifications for a given group
if push.backedout or push.bustage_fixed_by:
group_classifications: dict[
str, dict[tuple[str, str], set[str]]
] = collections.defaultdict(lambda: collections.defaultdict(set))
for other in push._iterate_children():
if (
push.backedoutby in other.revs
or push.bustage_fixed_by in other.revs
):
break
for name, summary in other.group_summaries.items():
for classification in summary.classifications:
group_classifications[name][classification].add(other.rev)
for name, classification_to_revs in group_classifications.items():
if len(classification_to_revs) > 1:
inconsistent_list = [
f" - {classification} in pushes {', '.join(revs)}"
for classification, revs in classification_to_revs.items()
]
inconsistent = "\n" + ",\n".join(inconsistent_list)
warnings.append(
{
"message": f"Group {name} has inconsistent classifications: {inconsistent}.",
"type": "comment",
"notify": config.get("warnings", {}).get(
"inconsistent", False
),
}
)
# Output all warnings and also send them to the Matrix room if defined
matrix_room = config.get("matrix-room-id")
for warning in warnings:
warn_type = warning["type"]
warn_message = warning["message"]
do_notify = warning["notify"]
self.line(f"<{warn_type}>{warn_message}</{warn_type}>")
if matrix_room and do_notify:
notify_matrix(room=matrix_room, body=warn_message)
if not matrix_room and warnings:
self.line(
"<comment>Some warning notifications should have been sent but no matrix room was provided in the secret.</comment>"
)
# Advance the overall progress bar
progress.advance()
# Conclude the progress bar
progress.finish()
print("\n")
error_line = ""
if self.errors:
if self.option("recalculate"):
error_line = "Failed to recalculate classification"
else:
error_line = "Failed to fetch classification"
error_line += f" for {len(self.errors)} out of {len(self.pushes)} pushes."
if not self.option("recalculate") and not self.option("send-email"):
error_line += " Use the '--recalculate' option if you want to generate them yourself."
self.line(f"<error>{error_line}</error>")
stats = [
self.log_pushes(PushStatus.BAD, False),
self.log_pushes(PushStatus.BAD, True),
self.log_pushes(PushStatus.GOOD, False),
self.log_pushes(PushStatus.GOOD, True),
self.log_pushes(PushStatus.UNKNOWN, False),
self.log_pushes(PushStatus.UNKNOWN, True),
]
if self.option("detailed-classifications"):
self.line("\n")
pushes_group_summaries = {}
real_stats = intermittent_stats = {
"total": 0,
"correct": 0,
"wrong": 0,
"pending": 0,
"conflicting": 0,
"missed": 0,
}
for push in self.pushes:
self.line(
f"<comment>Printing detailed classifications comparison for push {push.branch}/{push.rev}</comment>"
)
if push.push_uuid not in pushes_group_summaries:
pushes_group_summaries[push.push_uuid] = push.group_summaries
# Compare real failures that were predicted by mozci with the ones classified by Sheriffs
try:
pushes_group_summaries, sheriff_reals = retrieve_sheriff_reals(
pushes_group_summaries, push
)
except Exception:
self.line(
"<error>Failed to retrieve Sheriff classifications for the real failures of this push.</error>"
)
try:
push_real_stats, to_print = parse_and_log_details(
pushes_group_summaries[push.push_uuid],
sheriff_reals,
{"fixed by commit"},
push=push,
failures=self.failures,
state="real",
)
for line in to_print:
self.line(line)
real_stats = {
key: value + push_real_stats[key]
for key, value in real_stats.items()
}
except Exception:
self.line(
"<error>Failed to compare true and predicted real failures of this push.</error>"
)
# Compare intermittent failures that were predicted by mozci with the ones classified by Sheriffs
try:
(
pushes_group_summaries,
sheriff_intermittents,
) = retrieve_sheriff_intermittents(pushes_group_summaries, push)
except Exception:
self.line(
"<error>Failed to retrieve Sheriff classifications for the intermittent failures of this push.</error>"
)
try:
push_intermittent_stats, to_print = parse_and_log_details(
pushes_group_summaries[push.push_uuid],
sheriff_intermittents,
set(INTERMITTENT_CLASSES),
push=push,
failures=self.failures,
state="intermittent",
)
for line in to_print:
self.line(line)
intermittent_stats = {
key: value + push_intermittent_stats[key]
for key, value in intermittent_stats.items()
}
except Exception:
self.line(
"<error>Failed to compare true and predicted intermittent failures of this push.</error>"
)
self.line(
f"\n<comment>Printing overall detailed classifications comparison for {len(self.pushes)} pushes</comment>"
)
detailed_stats = [
f"{real_stats['correct']} out of {real_stats['total']} failures were correctly classified as real ('fixed by commit' by Sheriffs).",
f"{real_stats['wrong']} out of {real_stats['total']} failures were wrongly classified as real ('intermittent' by Sheriffs).",
f"{real_stats['pending']} out of {real_stats['total']} failures classified as real are waiting to be classified by Sheriffs.",
f"{real_stats['conflicting']} out of {real_stats['total']} failures classified as real have conflicting classifications applied by Sheriffs.",
f"{real_stats['missed']} real failures were missed or classified as unknown by Mozci.",
f"{intermittent_stats['correct']} out of {intermittent_stats['total']} failures were correctly classified as intermittent ('intermittent' by Sheriffs).",
f"{intermittent_stats['wrong']} out of {intermittent_stats['total']} failures were wrongly classified as intermittent ('fixed by commit' by Sheriffs).",
f"{intermittent_stats['pending']} out of {intermittent_stats['total']} failures classified as intermittent are waiting to be classified by Sheriffs.",
f"{intermittent_stats['conflicting']} out of {intermittent_stats['total']} failures classified as intermittent have conflicting classifications applied by Sheriffs.",
f"{intermittent_stats['missed']} intermittent failures were missed or classified as unknown by Mozci.",
]
for line in detailed_stats:
self.line(line)
stats += detailed_stats
if self.option("send-email"):
self.send_emails(len(self.pushes), stats, error_line)
output = self.option("output")
if output:
# Build stats for CSV
with open(output, "w") as csvfile:
writer = csv.DictWriter(
csvfile,
fieldnames=[
"revision",
"date",
"classification",
"backedout",
"error_type",
"error_message",
],
)
writer.writeheader()
writer.writerows([self.build_stats(push) for push in self.pushes])
self.line(
f"<info>Written stats for {len(self.pushes)} pushes in {output}</info>"
)