#!/usr/bin/python3 # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at https://mozilla.org/MPL/2.0/. """ Used to produce comparisons of browsertime videos between a base and a new revision. """ import argparse import cv2 import gc import numpy as np import os import pathlib import json import shutil import subprocess from matplotlib import pyplot as plt from scipy.stats import spearmanr from sys import stdout from time import sleep try: from urllib.parse import urlencode from urllib.request import urlopen, urlretrieve except ImportError: from urllib import urlencode, urlretrieve from urllib2 import urlopen from artifact_downloader import artifact_downloader from task_processor import get_task_data_paths, match_vismets_with_videos, sorted_nicely TASK_IDS = ( "https://firefox-ci-tc.services.mozilla.com/api/index/v1/tasks/" + "gecko.v2.{}.revision.{}.taskgraph" ) TASK_INFO = "https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/{}" def side_by_side_parser(): parser = argparse.ArgumentParser( "This tool can be used to generate a side-by-side visualization of two videos. " "When using this tool, make sure that the `--test-name` is an exact match, i.e. " "if you are comparing the task `test-linux64-shippable-qr/opt-browsertime-tp6-firefox-linkedin-e10s` " "between two revisions, then use `browsertime-tp6-firefox-linkedin-e10s` as the suite name " "and `test-linux64-shippable-qr/opt` as the platform." ) parser.add_argument( "--base-revision", type=str, required=True, help="The base revision to compare a new revision to.", ) parser.add_argument( "--base-branch", type=str, default="autoland", help="Branch to search for the base revision.", ) parser.add_argument( "--new-revision", type=str, required=True, help="The base revision to compare a new revision to.", ) parser.add_argument( "--new-branch", type=str, default="autoland", help="Branch to search for the new revision.", ) parser.add_argument( "--test-name", "--base-test-name", type=str, required=True, dest="test_name", help="The name of the test task to get videos from.", ) parser.add_argument( "--new-test-name", type=str, default=None, help="The name of the test task to get videos from in the new revision.", ) parser.add_argument( "--platform", "--base-platform", type=str, required=True, dest="platform", help="Platform to return results for.", ) parser.add_argument( "--new-platform", type=str, default=None, help="Platform to return results for in the new revision.", ) parser.add_argument( "--overwrite", action="store_true", default=False, help="If set, the downloaded task group data will be deleted before " + "it gets re-downloaded.", ) parser.add_argument( "--cold", action="store_true", default=False, help="If set, we'll only look at cold pageload tests.", ) parser.add_argument( "--warm", action="store_true", default=False, help="If set, we'll only look at warm pageload tests.", ) parser.add_argument( "--most-similar", action="store_true", default=False, help="If set, we'll search for a video pairing that is the most similar.", ) parser.add_argument( "--search-crons", action="store_true", default=False, help="If set, we will search for the tasks within the cron jobs as well. ", ) parser.add_argument( "--skip-download", action="store_true", default=False, help="If set, we won't try to download artifacts again and we'll " + "try using what already exists in the output folder.", ) parser.add_argument( "--output", type=str, default=os.getcwd(), help="This is where the data will be saved. Defaults to CWD. " + "You can include a name for the file here, otherwise it will " + "default to side-by-side.mp4.", ) parser.add_argument( "--metric", type=str, default="speedindex", help="Metric to use for side-by-side comparison.", ) parser.add_argument( "--vismetPath", type=str, default=False, help="Paths to visualmetrics.py for step chart generation.", ) parser.add_argument( "--original", action="store_true", default=False, help="If set, use the original videos in the side-by-side instead " + "of the postprocessed videos.", ) parser.add_argument( "--skip-slow-gif", action="store_true", default=False, help="If set, the slow-motion GIFs won't be produced.", ) return parser def write_same_line(x, sleep_time=0.0001): stdout.write("\r%s" % str(x)) stdout.flush() sleep(sleep_time) def finish_same_line(): stdout.write("\r \r\n") def get_json(url, params=None): if params is not None: url += "?" + urlencode(params) r = urlopen(url).read().decode("utf-8") return json.loads(r) def find_task_group_id(revision, branch, search_crons=False): # Find the task IDs from this revision first task_ids_url = TASK_IDS.format(branch, revision) print("Downloading task ids from: %s" % task_ids_url) task_ids_data = get_json(task_ids_url) if "tasks" not in task_ids_data or len(task_ids_data["tasks"]) == 0: raise Exception("Cannot find any task IDs for %s!" % revision) task_group_ids = [] for task in task_ids_data["tasks"]: # Only find the task group ID for the decision task if we # don't need to search for cron tasks if not search_crons and not task["namespace"].endswith("decision"): continue task_group_url = TASK_INFO.format(task["taskId"]) print("Downloading task group id from: %s" % task_group_url) task_info = get_json(task_group_url) task_group_ids.append(task_info["taskGroupId"]) return task_group_ids def find_videos(artifact_dir, original=False): # Find the cold/warm browsertime.json files cold_path = "" warm_path = "" for path in pathlib.Path(artifact_dir).rglob("*-browsertime.json"): if "cold" in str(path): cold_path = path elif "warm" in str(path): warm_path = path if not cold_path: raise Exception("Cannot find a browsertime.json file for the cold pageloads.") if not warm_path: raise Exception("Cannot find a browsertime.json file for the warm pageloads.") with cold_path.open() as f: cold_data = json.load(f) with warm_path.open() as f: warm_data = json.load(f) return { "cold": [ str(pathlib.Path(cold_path.parents[0], file)).replace( ".mp4", "-original.mp4" ) if original else str(pathlib.Path(cold_path.parents[0], file)) for file in cold_data[0]["files"]["video"] ], "warm": [ str(pathlib.Path(warm_path.parents[0], file)).replace( ".mp4", "-original.mp4" ) if original else str(pathlib.Path(warm_path.parents[0], file)) for file in warm_data[0]["files"]["video"] ], } def find_videos_with_retriggers(artifact_dirs, original=False): results = {"cold": [], "warm": []} for artifact_dir in artifact_dirs: videos = find_videos(artifact_dir, original=original) results["cold"].extend(videos["cold"]) results["warm"].extend(videos["warm"]) return results def get_similarity( old_videos_info, new_videos_info, output, prefix="", most_similar=False ): """Calculates a similarity score for two groupings of videos. The technique works as follows: 2. For each UxV video pairings, build a cross-correlation matrix: 1. Get each of the videos and calculate their histograms across the full videos. 2. Calculate the correlation coefficient between these two. 3. Average the cross-correlation matrix to obtain the score. Args: old_videos: List of old videos. new_videos: List of new videos (from this task). output: Location to output videos with low similarity scores. prefix: Prefix a string to the output. Returns: A dictionary containing the worst pairing and the 3D similarity score. """ def _get_frames(video): """Gets all frames from a video into a list.""" allframes = [] orange_pixind = 0 orange_frameind = 0 frame_count = 0 check_for_orange = True while video.isOpened(): ret, frame = video.read() if ret: # Convert to gray to simplify the process allframes.append(cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)) # Check if it's orange still if check_for_orange: frame = allframes[-1] histo, _, _ = plt.hist(np.asarray(frame).flatten(), bins=255) maxi = np.argmax(histo) if not orange_pixind: if maxi > 130: continue orange_pixind = maxi elif maxi == orange_pixind: orange_frameind = frame_count else: check_for_orange = False frame_count += 1 else: video.release() break return allframes[orange_frameind:], orange_frameind nhists = [] old_videos = [entry["data"] for entry in old_videos_info] new_videos = [entry["data"] for entry in new_videos_info] new_orange_frameinds = [] old_orange_frameinds = [] total_vids = min(len(old_videos), len(new_videos)) xcorr = np.zeros((total_vids, total_vids)) for i in range(total_vids): datao, old_orange_frameind = _get_frames(old_videos[i]) datao = np.asarray(datao) old_orange_frameinds.append(old_orange_frameind) histo, _, _ = plt.hist(datao.flatten(), bins=255) plt.clf() gc.collect() for j in range(total_vids): write_same_line("Comparing old video %s to new video %s" % (i + 1, j + 1)) if i == 0: # Only calculate the histograms once; it takes time datan, new_orange_frameind = _get_frames(new_videos[j]) datan = np.asarray(datan) new_orange_frameinds.append(new_orange_frameind) histn, _, _ = plt.hist(datan.flatten(), bins=255) plt.clf() gc.collect() nhists.append(histn) else: histn = nhists[j] rho, _ = spearmanr(histo, histn) xcorr[i, j] = rho finish_same_line() similarity = np.nanmean(xcorr) print("Average 3D similarity: %s" % str(np.round(similarity, 5))) if most_similar: inds = np.unravel_index(np.argmax(xcorr, axis=None), xcorr.shape) else: inds = np.unravel_index(np.argmin(xcorr, axis=None), xcorr.shape) oldvid = old_videos_info[inds[0]]["path"] oldvidnewpath = str(pathlib.Path(output, "%sold_video.mp4" % prefix)) shutil.copyfile(oldvid, oldvidnewpath) newvid = new_videos_info[inds[1]]["path"] newvidnewpath = str(pathlib.Path(output, "%snew_video.mp4" % prefix)) shutil.copyfile(newvid, newvidnewpath) return { "sim3": np.round(similarity, 5), "oldvid": oldvidnewpath, "oldvid_ind": old_orange_frameinds[inds[0]], "newvid": newvidnewpath, "newvid_ind": new_orange_frameinds[inds[1]], } def find_lowest_similarity(base_videos, new_videos, output, prefix, most_similar=False): def _open_data(file): return cv2.VideoCapture(str(file)) return get_similarity( [{"data": _open_data(str(f)), "path": str(f)} for f in base_videos], [{"data": _open_data(str(f)), "path": str(f)} for f in new_videos], output, prefix, most_similar=most_similar, ) def open_and_organize_perfherder(files, metric): def _open_perfherder(filen): with open(filen) as f: return json.load(f) res = {"cold": [], "warm": []} for filen in files: data = _open_perfherder(filen) for suite in data["suites"]: pl_type = "warm" if "cold" in suite["extraOptions"]: pl_type = "cold" for subtest in suite["subtests"]: if subtest["name"].lower() != metric.lower(): continue # Each entry here will be a single retrigger of # the test for the requested metric (ordered # based on the `files` ordering) res[pl_type].append(subtest) return res def generate_step_chart(oldvid, newvid, vismetPath, prefix, metric, output): print("Generating step chart for %s" % metric) oldvid_metrics = json.loads( subprocess.check_output( [ "python", vismetPath, "--orange", "--perceptual", "--contentful", "--force", "--renderignore", "5", "--json", "--viewport", "--video", oldvid, ] ) ) newvid_metrics = json.loads( subprocess.check_output( [ "python", vismetPath, "--orange", "--perceptual", "--contentful", "--force", "--renderignore", "5", "--json", "--viewport", "--video", newvid, ] ) ) if metric.lower() == "perceptualspeedindex": progress = "PerceptualSpeedIndexProgress" metricName = "PerceptualSpeedIndex" elif metric.lower() == "contentfulspeedindex": progress = "ContentfulSpeedIndexProgress" metricName = "ContentfulSpeedIndex" else: progress = "VisualProgress" metricName = "SpeedIndex" x = [] y = [] for pt in oldvid_metrics[progress].split(","): x_val, y_val = pt.split("=") x.append(int(x_val)) y.append(int(y_val)) plt.step(x, y, label="Before (%d)" % oldvid_metrics[metricName]) x = [] y = [] for pt in newvid_metrics[progress].split(","): x_val, y_val = pt.split("=") x.append(int(x_val)) y.append(int(y_val)) plt.step(x, y, label="After (%d)" % newvid_metrics[metricName]) plt.legend(loc="lower right") plt.title("%s %s" % (prefix.rstrip("_"), metricName)) plt.savefig(str(output / "%s-%s-step.png" % (prefix.rstrip("_"), metric))) plt.clf() def find_closest_videos( base_videos, base_vismet, new_videos, new_vismet, output, prefix, metric ): base_btime_id = "" base_min_idx = None # Recalculate median for all values, then find the new video # by searching in the list for it (use index) to determine # the matching video. replicates = [] for retrigger in base_vismet: replicates.extend(retrigger["replicates"]) median_value = np.median(replicates) # Find the video which most closely matches the average diff = [abs(replicate - median_value) for replicate in replicates] min_diff = min(diff) base_min_idx = diff.index(min_diff) print( "BASE: metric=%s prefix=%s mean=%d closest=%d index=%d" % ( metric, prefix, median_value, min_diff, base_min_idx, ) ) replicates = [] for retrigger in new_vismet: replicates.extend(retrigger["replicates"]) median_value = np.median(replicates) # Find the video which most closely matches the average diff = [abs(replicate - median_value) for replicate in replicates] min_diff = min(diff) new_min_idx = diff.index(min_diff) print( "NEW: metric=%s prefix=%s mean=%d closest=%d index=%d" % ( metric, prefix, median_value, min_diff, new_min_idx, ) ) oldvid = base_videos[base_min_idx] oldvidnewpath = str(pathlib.Path(output, "%sold_video.mp4" % prefix)) shutil.copyfile(oldvid, oldvidnewpath) newvid = new_videos[new_min_idx] newvidnewpath = str(pathlib.Path(output, "%snew_video.mp4" % prefix)) shutil.copyfile(newvid, newvidnewpath) if args.vismetPath: generate_step_chart(oldvid, newvid, args.vismetPath, prefix, metric, output) # The index values used here are for frame selection during video editing. # We use 0 to select all frames. return { "oldvid": oldvidnewpath, "oldvid_ind": 0, "newvid": newvidnewpath, "newvid_ind": 0, } def build_side_by_side(base_video, new_video, base_ind, new_ind, output_dir, filename): before_vid = pathlib.Path(output_dir, "before.mp4") after_vid = pathlib.Path(output_dir, "after.mp4") before_cut_vid = pathlib.Path(output_dir, "before-cut.mp4") after_cut_vid = pathlib.Path(output_dir, "after-cut.mp4") before_rs_vid = pathlib.Path(output_dir, "before-rs.mp4") after_rs_vid = pathlib.Path(output_dir, "after-rs.mp4") for apath in ( before_vid, after_vid, before_cut_vid, after_cut_vid, before_rs_vid, after_rs_vid, ): if apath.exists(): apath.unlink() overlay_text = ( "fps=fps=60,drawtext=text={}\\\\ :fontsize=(h/20):fontcolor=black:y=10:" + "timecode=00\\\\:00\\\\:00\\\\:00:rate=60*1000/1001:fontcolor=white:x=(w-tw)/2:" + "y=10:box=1:boxcolor=0x00000000@1[vid]" ) common_options = [ "-map", "[vid]", "-c:v", "libx264", "-crf", "18", "-preset", "veryfast", ] # Cut the videos subprocess.check_output( ["ffmpeg", "-i", str(base_video), "-vf", "select=gt(n\\,%s)" % base_ind] + [str(before_cut_vid)] ) subprocess.check_output( ["ffmpeg", "-i", str(new_video), "-vf", "select=gt(n\\,%s)" % new_ind] + [str(after_cut_vid)] ) # Resample subprocess.check_output( ["ffmpeg", "-i", str(before_cut_vid), "-filter:v", "fps=fps=60"] + [str(before_rs_vid)] ) subprocess.check_output( ["ffmpeg", "-i", str(after_cut_vid), "-filter:v", "fps=fps=60"] + [str(after_rs_vid)] ) # Generate the before and after videos subprocess.check_output( [ "ffmpeg", "-i", str(before_rs_vid), "-filter_complex", overlay_text.format("BEFORE"), ] + common_options + [str(before_vid)] ) subprocess.check_output( [ "ffmpeg", "-i", str(after_rs_vid), "-filter_complex", overlay_text.format("AFTER"), ] + common_options + [str(after_vid)] ) subprocess.check_output( [ "ffmpeg", "-i", str(before_vid), "-i", str(after_vid), "-filter_complex", "[0:v]pad=iw*2:ih[int];[int][1:v]overlay=W/2:0[vid]", ] + common_options + [str(pathlib.Path(output_dir, filename))] ) def convert_mp4_to_gif(path_to_mp4, path_to_gif, slow_motion=False): path_to_gif = str(path_to_gif) fps = 30 # Use slow motion for more subtle differences if slow_motion: fps = 100 path_to_gif = path_to_gif.replace(".gif", "-slow-motion.gif") # Generate palette for a better quality subprocess.check_output( [ "ffmpeg", "-i", str(path_to_mp4), "-vf", f"fps={fps},scale=1024:-1:flags=lanczos,palettegen", "-y", ] + [path_to_gif.replace(".gif", "-palette.gif")] ) subprocess.check_output( [ "ffmpeg", "-i", str(path_to_mp4), "-i", path_to_gif.replace(".gif", "-palette.gif"), "-filter_complex", f"fps={fps},scale=1024:-1:flags=lanczos[x];[x][1:v]paletteuse", "-loop", "-1", ] + [str(path_to_gif)] ) subprocess.check_output(["rm", path_to_gif.replace(".gif", "-palette.gif")]) return str(path_to_gif) if __name__ == "__main__": args = side_by_side_parser().parse_args() overwrite = args.overwrite if shutil.which("ffmpeg") is None: raise Exception( "Cannot find ffmpeg in path! Please install it before continuing." ) if "vismet-" in args.platform: args.platform = args.platform.replace("vismet-", "") if not args.test_name.endswith("-e10s"): args.test_name += "-e10s" print( "Vismet tasks do not contain browsertime video recordings." + "We'll assume you meant this platform: %s" % args.platform ) if args.vismetPath and not pathlib.Path(args.vismetPath).exists(): raise Exception("Cannot find the vismet script at: %s" % args.vismetPath) if args.metric != "similarity" and args.skip_download: print( "WARNING: Downloads will not be skipped as you are using something other " "than the similarity metric (only supported for this metric)." ) # Parse the given output argument filename = "side-by-side.mp4" output = pathlib.Path(args.output) if output.exists() and output.is_file(): print("Deleting existing output file...") output.unlink() elif not output.suffixes: output.mkdir(parents=True, exist_ok=True) else: filename = output.name output = output.parents[0] output.mkdir(parents=True, exist_ok=True) # Make sure we remove the old side-by-side visualization # for the FFMPEG operations cold_path = pathlib.Path(output, "cold-" + filename) warm_path = pathlib.Path(output, "warm-" + filename) if cold_path.exists(): cold_path.unlink() if warm_path.exists(): warm_path.unlink() # Get the task group IDs for the revisions base_revision_ids = find_task_group_id( args.base_revision, args.base_branch, search_crons=args.search_crons ) new_revision_ids = find_task_group_id( args.new_revision, args.new_branch, search_crons=args.search_crons ) base_task_dirs = [pathlib.Path(output, revid) for revid in base_revision_ids] new_task_dirs = [pathlib.Path(output, revid) for revid in new_revision_ids] if overwrite: for task_dir in base_task_dirs + new_task_dirs: if task_dir.exists(): print("Removing existing task group folder: %s" % str(task_dir)) shutil.rmtree(str(task_dir)) def _search_for_paths(rev_ids, artifact, open_data=False): found_paths = [] for rev_id in rev_ids: if found_paths: break # Get the paths to the directory holding the artifacts, the 0 # index is because we are only looking at one suite here. found_paths = list( get_task_data_paths(rev_id, str(output), artifact=artifact).values() )[0] return found_paths # Setup the vismet version of the platform and test names vismet_platform = args.platform.replace("test-", "test-vismet-") test_no_e10s = args.test_name.replace("-e10s", "") # Download the artifacts if not args.skip_download: base_paths = [] for base_revision_id in base_revision_ids: if base_paths: break artifact_downloader( base_revision_id, output_dir=str(output), test_suites=[args.test_name], platform=args.platform, artifact_to_get=["browsertime-results", "perfherder-data"], unzip_artifact=True, download_failures=False, ingest_continue=False, ) base_paths = _search_for_paths([base_revision_id], "browsertime-results") base_vismet = _search_for_paths([base_revision_id], "perfherder-data") new_paths = [] for new_revision_id in new_revision_ids: if new_paths: break artifact_downloader( new_revision_id, output_dir=str(output), test_suites=[args.new_test_name or args.test_name], platform=args.new_platform or args.platform, artifact_to_get=["browsertime-results", "perfherder-data"], unzip_artifact=True, download_failures=False, ingest_continue=False, ) new_paths = _search_for_paths([new_revision_id], "browsertime-results") new_vismet = _search_for_paths([new_revision_id], "perfherder-data") else: base_paths = _search_for_paths(base_revision_ids, "browsertime-results") base_vismet = _search_for_paths(base_revision_ids, "perfherder-data") new_paths = _search_for_paths(new_revision_ids, "browsertime-results") new_vismet = _search_for_paths(new_revision_ids, "perfherder-data") # Make sure we only downloaded one task failure_msg = ( "Not enough artifacts downloaded for %s, can't compare! " + "Found paths: %s \nTry using --search-crons if you are sure the task exists." ) if not base_paths: raise Exception(failure_msg % (args.base_revision, base_paths)) if not new_paths: raise Exception(failure_msg % (args.new_revision, new_paths)) # Gather the videos and split them between warm and cold base_videos = find_videos_with_retriggers(base_paths, original=args.original) new_videos = find_videos_with_retriggers(new_paths, original=args.original) # If we are looking at something other than similarity, # prepare the data for this (open, and split between # cold and warm) if args.metric != "similarity": print("Opening, and organizing perfherder data...") org_base_vismet = open_and_organize_perfherder(base_vismet, args.metric) org_new_vismet = open_and_organize_perfherder(new_vismet, args.metric) if (not org_new_vismet["cold"] and not org_new_vismet["warm"]) or ( not org_base_vismet["cold"] and not org_base_vismet["warm"] ): raise Exception("Could not find any data with the metric: %s" % args.metric) run_cold = args.cold run_warm = args.warm if not args.cold and not args.warm: run_cold = True run_warm = True # Find the worst video pairing for cold and warm print("Starting comparisons, this may take a few minutes") if run_cold: print("Running comparison for cold pageloads...") if args.metric == "similarity": cold_pairing = find_lowest_similarity( base_videos["cold"], new_videos["cold"], str(output), "cold_", most_similar=args.most_similar, ) else: cold_pairing = find_closest_videos( base_videos["cold"], org_base_vismet["cold"], new_videos["cold"], org_new_vismet["cold"], str(output), "cold_", args.metric, ) if run_warm: gc.collect() print("Running comparison for warm pageloads...") if args.metric == "similarity": warm_pairing = find_lowest_similarity( base_videos["warm"], new_videos["warm"], str(output), "warm_", most_similar=args.most_similar, ) else: warm_pairing = find_closest_videos( base_videos["warm"], org_base_vismet["warm"], new_videos["warm"], org_new_vismet["warm"], str(output), "warm_", args.metric, ) # Build up the side-by-side comparisons now if run_cold: output_name = str(pathlib.Path(output, "cold-" + filename)) build_side_by_side( cold_pairing["oldvid"], cold_pairing["newvid"], cold_pairing["oldvid_ind"], cold_pairing["newvid_ind"], output, "cold-" + filename, ) print("Successfully built a side-by-side cold comparison: %s" % output_name) gif_output_name = pathlib.Path( output, "cold-" + filename.replace(".mp4", ".gif") ) gif_output_name = convert_mp4_to_gif(output_name, gif_output_name) print( "Successfully converted the side-by-side cold comparison to gif: %s" % gif_output_name ) if not args.skip_slow_gif: gif_output_name = convert_mp4_to_gif( output_name, gif_output_name, slow_motion=True ) print( "Successfully converted the side-by-side cold comparison to slow motion gif: %s" % gif_output_name ) if run_warm: output_name = str(pathlib.Path(output, "warm-" + filename)) build_side_by_side( warm_pairing["oldvid"], warm_pairing["newvid"], warm_pairing["oldvid_ind"], warm_pairing["newvid_ind"], output, "warm-" + filename, ) print("Successfully built a side-by-side warm comparison: %s" % output_name) gif_output_name = pathlib.Path( output, "warm-" + filename.replace(".mp4", ".gif") ) gif_output_name = convert_mp4_to_gif(output_name, gif_output_name) print( "Successfully converted the side-by-side warm comparison to gif: %s" % gif_output_name ) if not args.skip_slow_gif: gif_output_name = convert_mp4_to_gif( output_name, gif_output_name, slow_motion=True ) print( "Successfully converted the side-by-side warm comparison to slow motion gif: %s" % gif_output_name )