ui/perfherder/perf-helpers/helpers.js (751 lines of code) (raw):

import moment from 'moment'; import numeral from 'numeral'; import queryString from 'query-string'; import { getApiUrl, getJobsUrl } from '../../helpers/url'; import { update, processResponse } from '../../helpers/http'; import PerfSeriesModel, { getSeriesName, getTestName, } from '../../models/perfSeries'; import RepositoryModel from '../../models/repository'; import JobModel from '../../models/job'; import { sxsTaskName } from '../../helpers/constants'; import { endpoints, tValueCareMin, tValueConfidence, noiseMetricTitle, availablePlatforms, summaryStatusMap, alertStatusMap, phFrameworksWithRelatedBranches, phTimeRanges, phDefaultTimeRangeValue, unknownFrameworkMessage, permaLinkPrefix, } from './constants'; const numberFormat = new Intl.NumberFormat('en-US', { maximumFractionDigits: 2, }); export const formatNumber = (input) => numberFormat.format(input); export const abbreviatedNumber = (num) => num.toString().length <= 5 ? num : numeral(num).format('0.0a'); export const displayNumber = (input) => Number.isNaN(input) ? 'N/A' : Number(input).toFixed(2); export const calcPercentOf = function calcPercentOf(a, b) { return b ? (100 * a) / b : 0; }; export const calcAverage = function calcAverage(values) { if (!values.length) { return 0; } return values.reduce((a, b) => a + b, 0) / values.length; }; export const getStdDev = function getStandardDeviation(values, avg) { if (values.length < 2) { return undefined; } if (!avg) avg = calcAverage(values); return Math.sqrt( values.map((v) => (v - avg) ** 2).reduce((a, b) => a + b) / (values.length - 1), ); }; // If a set has only one value, assume average-ish-plus standard deviation, which // will manifest as smaller t-value the less items there are at the group // (so quite small for 1 value). This default value is a parameter. // C/T mean control/test group (in our case original/new data). export const getTTest = function getTTest( valuesC, valuesT, stddevDefaultFactor, ) { const lenC = valuesC.length; const lenT = valuesT.length; if (!lenC || !lenT) { return 0; } const avgC = calcAverage(valuesC); const avgT = calcAverage(valuesT); let stddevC = lenC > 1 ? getStdDev(valuesC, avgC) : stddevDefaultFactor * avgC; let stddevT = lenT > 1 ? getStdDev(valuesT, avgT) : stddevDefaultFactor * avgT; if (lenC === 1) { stddevC = (valuesC[0] * stddevT) / avgT; } else if (lenT === 1) { stddevT = (valuesT[0] * stddevC) / avgC; } const delta = avgT - avgC; const stdDiffErr = Math.sqrt( (stddevC * stddevC) / lenC + // control-variance / control-size (stddevT * stddevT) / lenT, ); return delta / stdDiffErr; }; const numericCompare = (a, b) => { if (a < b) { return -1; } if (a > b) { return 1; } return 0; }; const analyzeSet = (values, testName) => { let average; let stddev = 1; if (testName === noiseMetricTitle) { average = Math.sqrt(values.map((x) => x ** 2).reduce((a, b) => a + b, 0)); } else { average = calcAverage(values); stddev = getStdDev(values, average); } return { average, stddev, stddevPct: Math.round(calcPercentOf(stddev, average) * 100) / 100, // We use slice to keep the original values at their original order // in case the order is important elsewhere. runs: values.slice().sort(numericCompare), }; }; const getClassName = (newIsBetter, oldVal, newVal, absTValue) => { // Returns a class name, if any, based on a relative change in the absolute value if (!oldVal || !newVal) { return ''; } let ratio = newVal / oldVal; if (ratio < 1) { ratio = 1 / ratio; // Direction agnostic and always >= 1. } if (ratio < 1.02 || absTValue < tValueCareMin) { return ''; } if (absTValue < tValueConfidence) { return newIsBetter ? '' : 'warning'; } return newIsBetter ? 'success' : 'danger'; }; // Aggregates two sets of values into a "comparison object" which is later used // to display a single line of comparison. // The result object has the following properties: // - .isEmpty: true if no data for either side. // If !isEmpty, for originalData/newData (if the data exists) // - .[original|new]Value // Average of the values // - .[original|new]Stddev // stddev // - .[original|new]StddevPct // stddev as percentage of the average // - .[original|new]Runs // Display data: number of runs and their values // If both originalData/newData exist, comparison data: // - .newIsBetter // is new result better or worse (even if unsure) // - .isImprovement // is new result better + we're confident about it // - .isRegression // is new result worse + we're confident about it // - .delta // - .deltaPercentage // - .confidence // t-test value // - .confidenceText // 'low'/'med'/'high' // - .confidenceTextLong // more explanation on what confidenceText means // - .isMeaningful // for highlighting - bool over t-test threshold // And some data to help formatting of the comparison: // - .className // - .magnitude // - .marginDirection export const getCounterMap = function getCounterMap( testName, originalData, newData, ) { const cmap = { isEmpty: false }; const hasOrig = originalData && originalData.values.length; const hasNew = newData && newData.values.length; if (!hasOrig && !hasNew) { cmap.isEmpty = true; return cmap; } cmap.originalRetriggerableJobId = null; cmap.newRetriggerableJobId = null; if (hasOrig) { const orig = analyzeSet(originalData.values, testName); cmap.originalValue = orig.average; cmap.originalRuns = orig.runs; cmap.originalStddev = orig.stddev; cmap.originalStddevPct = orig.stddevPct; cmap.originalRepoName = originalData.repository_name; if (originalData.job_ids && originalData.job_ids.length) { [cmap.originalRetriggerableJobId] = originalData.job_ids; } } else { cmap.originalRuns = []; } if (hasNew) { const newd = analyzeSet(newData.values, testName); cmap.newValue = newd.average; cmap.newRuns = newd.runs; cmap.newStddev = newd.stddev; cmap.newStddevPct = newd.stddevPct; cmap.newRepoName = newData.repository_name; if (newData.job_ids && newData.job_ids.length) { [cmap.newRetriggerableJobId] = newData.job_ids; } } else { cmap.newRuns = []; } if (!hasOrig || !hasNew) { return cmap; // No comparison, just display for one side. } cmap.frameworkId = originalData.framework_id; // Normally tests are "lower is better", can be over-ridden with a series option cmap.delta = cmap.newValue - cmap.originalValue; cmap.newIsBetter = (originalData.lower_is_better && cmap.delta < 0) || (!originalData.lower_is_better && cmap.delta > 0); cmap.deltaPercentage = calcPercentOf(cmap.delta, cmap.originalValue); // arbitrary scale from 0-20% multiplied by 5, capped // at 100 (so 20% regression === 100% bad) cmap.magnitude = Math.min(Math.abs(cmap.deltaPercentage) * 5, 100); // 0.15 is used for getTTest: default stddev if both sets have only a single value - 15%. // Should be rare case and it's unreliable, but at least have something. const absTValue = Math.abs( getTTest(originalData.values, newData.values, 0.15), ); cmap.className = getClassName( cmap.newIsBetter, cmap.originalValue, cmap.newValue, absTValue, ); cmap.confidence = absTValue; cmap.confidenceTextLong = 'Result of running t-test on base versus new result distribution: '; if (absTValue < tValueCareMin) { cmap.confidenceText = 'low'; cmap.confidenceTextLong += "A value of 'low' suggests less confidence that there is a sustained, significant change between the two revisions."; } else if (absTValue < tValueConfidence) { cmap.confidenceText = 'med'; cmap.confidenceTextLong += "A value of 'med' indicates uncertainty that there is a significant change. If you haven't already, consider retriggering the job to be more sure."; } else { cmap.confidenceText = 'high'; cmap.confidenceTextLong += "A value of 'high' indicates more confidence that there is a significant change, however you should check the historical record for the test by looking at the graph to be more sure (some noisy tests can provide inconsistent results)."; } cmap.isRegression = cmap.className === 'danger'; cmap.isImprovement = cmap.className === 'success'; cmap.isMeaningful = cmap.className !== ''; cmap.isComplete = cmap.originalRuns.length && cmap.newRuns.length; cmap.isConfident = (cmap.originalRuns.length > 1 && cmap.newRuns.length > 1 && absTValue >= tValueConfidence) || (cmap.originalRuns.length >= 6 && cmap.newRuns.length >= 6 && absTValue >= tValueCareMin); cmap.needsMoreRuns = cmap.isComplete && !cmap.isConfident && cmap.originalRuns.length < 6; cmap.isNoiseMetric = false; return cmap; }; // TODO change usage of signature_hash to signature.id export const getGraphsLink = function getGraphsLink( seriesList, resultSets, timeRange, ) { const params = { series: seriesList.map((series) => [ series.projectName, series.signature, 1, series.frameworkId, ]), highlightedRevisions: resultSets.map((resultSet) => resultSet.revision.slice(0, 12), ), }; if (resultSets && !timeRange) { params.timerange = Math.max( ...resultSets.map((resultSet) => phTimeRanges .map((range) => range.value) .find((t) => Date.now() / 1000.0 - resultSet.push_timestamp < t), ), ); } if (timeRange) { params.timerange = timeRange; } return `./graphs?${queryString.stringify(params)}`; }; export const createNoiseMetric = function createNoiseMetric( cmap, name, compareResults, ) { cmap.name = name; cmap.isNoiseMetric = true; if (compareResults.has(noiseMetricTitle)) { compareResults.get(noiseMetricTitle).push(cmap); } else { compareResults.set(noiseMetricTitle, [cmap]); } return compareResults; }; export const createGraphsLinks = ( validatedProps, links, framework, timeRange, signature, app, ) => { const { originalProject, newProject, originalRevision, newResultSet, originalResultSet, } = validatedProps; const graphsParams = [...new Set([originalProject, newProject])].map( (projectName) => ({ projectName, signature, frameworkId: framework.id, application: app, }), ); let graphsLink; if (originalRevision) { graphsLink = getGraphsLink(graphsParams, [originalResultSet, newResultSet]); } else { graphsLink = getGraphsLink(graphsParams, [newResultSet], timeRange.value); } links.push({ title: 'graph', to: graphsLink, }); return links; }; export const getSideBySideLink = ( repository, baseRevision, newRevision, platform, testName, ) => { const revisions = `${baseRevision.slice(0, 12)} ${newRevision.slice(0, 12)}`; const jobUrl = getJobsUrl({ repo: repository, tochange: newRevision, fromchange: baseRevision, searchStr: `${platform} ${testName} ${revisions} ${sxsTaskName}`, group_state: 'expanded', }); return jobUrl; }; // TODO change usage of signature_hash to signature.id // for originalSignature and newSignature query params const Alert = (alertData, optionCollectionMap) => ({ ...alertData, title: getSeriesName(alertData.series_signature, optionCollectionMap, { includePlatformInName: true, }), }); export const getTimeRange = (alertSummary) => { const defaultTimeRange = alertSummary.repository === 'mozilla-beta' ? 7776000 : phDefaultTimeRangeValue; const timeRange = Math.max( defaultTimeRange, phTimeRanges .map((time) => time.value) .find( (value) => Date.now() / 1000.0 - alertSummary.push_timestamp <= value, ), ); // default value of one year, for one a push_timestamp exceeds the one year value slightly return timeRange || 31536000; }; // TODO change usage of signature_hash to signature.id export const getGraphsURL = ( alert, timeRange, alertRepository, performanceFrameworkId, ) => { let url = `./graphs?timerange=${timeRange}&series=${alertRepository},${alert.series_signature.id},1,${alert.series_signature.framework_id}`; // automatically add related branches (we take advantage of // the otherwise rather useless signature hash to avoid having to fetch this // information from the server) if (phFrameworksWithRelatedBranches.includes(performanceFrameworkId)) { const branches = alertRepository === 'mozilla-beta' ? ['autoland'] : []; url += branches .map( (branch) => `&series=${branch},${alert.series_signature.signature_hash},1,${alert.series_signature.framework_id}`, ) .join(''); } return url; }; export const modifyAlert = (alert, modification) => update(getApiUrl(`${endpoints.alert}${alert.id}/`), modification); export const modifyAlertSummary = (alertSummaryId) => { update(getApiUrl(`${endpoints.alertSummary}${alertSummaryId}/`)); }; export const getInitializedAlerts = (alertSummary, optionCollectionMap) => // this function converts the representation returned by the perfherder // api into a representation more suited for display in the UI // just treat related (reassigned or downstream) alerts as one // big block -- we'll display in the UI depending on their content alertSummary.alerts .concat(alertSummary.related_alerts) .map((alertData) => Alert(alertData, optionCollectionMap)); export const addResultsLink = (taskId) => { const taskLink = 'https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/'; const resultsPath = '/runs/0/artifacts/public/test_info/browsertime-results.tgz'; return `${taskLink}${taskId}${resultsPath}`; }; export const getFrameworkName = (frameworks, frameworkId) => { const framework = frameworks.find((item) => item.id === frameworkId); return framework ? framework.name : unknownFrameworkMessage; }; const getPlatformInfo = (platforms) => { const platformInfo = []; platforms.forEach((platform) => availablePlatforms.forEach((name) => { if (platform.includes(name.toLowerCase())) { if (!platformInfo.includes(name)) { platformInfo.push(name); } } }), ); return platformInfo; }; export const getFilledBugSummary = (alertSummary) => { let filledBugSummary; let maxMagnitudeAlert; let minMagnitudeAlert; // we should never include downstream alerts in the description let alertsInSummary = alertSummary.alerts.filter( (alert) => alert.status !== alertStatusMap.downstream || alert.summary_id === alertSummary.id, ); // figure out if there are any regressions -- if there are, // the summary should only incorporate those. if there // aren't, then use all of them (that aren't downstream, // see above) const regressions = alertsInSummary.filter((alert) => alert.is_regression); if (regressions.length > 0) { alertsInSummary = regressions; } // reassigned and invalid alerts are excluded alertsInSummary = [ ...new Set( alertsInSummary.filter( (alert) => (alert.related_summary_id === alertSummary.id || alert.related_summary_id === null) && alert.status !== alertStatusMap.invalid, ), ), ]; if (alertsInSummary.length > 1) { const maxMagnitude = Math.max( ...alertsInSummary.map((alert) => alert.amount_pct), ); const minMagnitude = Math.min( ...alertsInSummary.map((alert) => alert.amount_pct), ); maxMagnitudeAlert = alertsInSummary.find( (alert) => alert.amount_pct === maxMagnitude, ); minMagnitudeAlert = alertsInSummary.find( (alert) => alert.amount_pct === minMagnitude, ); filledBugSummary = `${maxMagnitude} - ${minMagnitude}%`; } else if (alertsInSummary.length === 1) { filledBugSummary = `${alertsInSummary[0].amount_pct}%`; } else { filledBugSummary = 'Empty alert'; } // add test info let testInfo = `${getTestName(alertsInSummary[0].series_signature)}`; if (maxMagnitudeAlert && minMagnitudeAlert) { testInfo = `${getTestName( maxMagnitudeAlert.series_signature, )} / ${getTestName(minMagnitudeAlert.series_signature)}`; } if (alertsInSummary.length > 2) { testInfo += ` + ${alertsInSummary.length - 2} more`; } filledBugSummary += ` ${testInfo}`; // add platform info const platforms = [ ...new Set( alertsInSummary.map((alert) => alert.series_signature.machine_platform), ), ]; const platformInfo = getPlatformInfo(platforms).sort().join(', '); filledBugSummary += ` (${platformInfo})`; // add push date info const pushDate = moment(alertSummary.push_timestamp * 1000).format( 'ddd MMMM D YYYY', ); filledBugSummary += ` regression on ${pushDate}`; return filledBugSummary; }; export const getTitle = (alertSummary) => { let title; // we should never include downstream alerts in the description let alertsInSummary = alertSummary.alerts.filter( (alert) => alert.status !== alertStatusMap.downstream || alert.summary_id === alertSummary.id, ); // figure out if there are any regressions -- if there are, // the summary should only incorporate those. if there // aren't, then use all of them (that aren't downstream, // see above) const regressions = alertsInSummary.filter((alert) => alert.is_regression); if (regressions.length > 0) { alertsInSummary = regressions; } if (alertsInSummary.length > 1) { title = `${Math.min( ...alertsInSummary.map((alert) => alert.amount_pct), )} - ${Math.max(...alertsInSummary.map((alert) => alert.amount_pct))}%`; } else if (alertsInSummary.length === 1) { title = `${alertsInSummary[0].amount_pct}%`; } else { title = 'Empty alert'; } // add test info const testInfo = [ ...new Set(alertsInSummary.map((a) => getTestName(a.series_signature))), ] .sort() .join(' / '); title += ` ${testInfo}`; // add platform info const platformInfo = [ ...new Set(alertsInSummary.map((a) => a.series_signature.machine_platform)), ] .sort() .join(', '); title += ` (${platformInfo})`; return title; }; export const updateAlertSummary = async (alertSummaryId, params) => update(getApiUrl(`${endpoints.alertSummary}${alertSummaryId}/`), params); export const convertParams = (params, value) => Boolean(params[value] !== undefined && parseInt(params[value], 10)); export const getFrameworkData = (props) => { const { validated, frameworks } = props; if (validated.framework) { const frameworkObject = frameworks.find( (item) => item.id === parseInt(validated.framework, 10), ); return frameworkObject; } return { id: 1, name: 'talos' }; }; export const getStatus = (statusNum, statusMap = summaryStatusMap) => { const status = Object.entries(statusMap).find( (item) => statusNum === item[1], ); return status[0]; }; export const containsText = (string, text) => { const words = text .split(' ') .map((word) => `(?=.*${word})`) .join(''); const regex = RegExp(words, 'gi'); return regex.test(string); }; export const processSelectedParam = (tooltipArray) => ({ signature_id: parseInt(tooltipArray[0], 10), dataPointId: parseInt(tooltipArray[1], 10), }); export const getInitialData = async ( errorMessages, repositoryName, framework, timeRange, ) => { const params = { interval: timeRange.value, framework: framework.id }; const platforms = await PerfSeriesModel.getPlatformList( repositoryName.name, params, ); const updates = { ...processResponse(platforms, 'platforms', errorMessages), }; return updates; }; export const updateSeriesData = (origSeriesData, testData) => origSeriesData.filter( (item) => testData.findIndex((test) => item.id === test.signature_id) === -1, ); export const getSeriesData = async ( params, errorMessages, repositoryName, testData, ) => { let updates = { filteredData: [], relatedTests: [], showNoRelatedTests: false, loading: false, }; const response = await PerfSeriesModel.getSeriesList( repositoryName.name, params, ); updates = { ...updates, ...processResponse(response, 'origSeriesData', errorMessages), }; if (updates.origSeriesData) { updates.seriesData = updateSeriesData(updates.origSeriesData, testData); } return updates; }; export const scrollWithOffset = function scrollWithOffset(el) { // solution from https://github.com/rafrex/react-router-hash-link/issues/25#issuecomment-536688104 const yCoordinate = el.getBoundingClientRect().top + window.pageYOffset; const yOffset = -35; window.scrollTo({ top: yCoordinate + yOffset, behavior: 'smooth' }); }; export const onPermalinkClick = (hashBasedValue, history, element) => { scrollWithOffset(element); history.replace( `${history.location.pathname}${history.location.search}#${hashBasedValue}`, ); }; // human readable signature name const getSignatureName = (testName, platformName) => [testName, platformName].filter((item) => item !== null).join(' '); export const getHashBasedId = function getHashBasedId( testName, hashFunction, platformName = null, ) { const tableSection = platformName === null ? 'header' : 'row'; const hashValue = hashFunction(getSignatureName(testName, platformName)); return `${permaLinkPrefix}-${tableSection}-${hashValue}`; }; const retriggerByRevision = async ( jobId, currentRepo, isBaseline, times, props, ) => { const { isBaseAggregate, notify } = props; // do not retrigger if the base is aggregate (there is a selected time range) if (isBaseline && isBaseAggregate) { return; } if (jobId) { const job = await JobModel.get(currentRepo.name, jobId); JobModel.retrigger([job], currentRepo, notify, times); } }; export const retriggerMultipleJobs = async ( results, baseRetriggerTimes, newRetriggerTimes, props, ) => { // retrigger base revision jobs const { projects } = props; retriggerByRevision( results.originalRetriggerableJobId, RepositoryModel.getRepo(results.originalRepoName, projects), true, baseRetriggerTimes, props, ); // retrigger new revision jobs retriggerByRevision( results.newRetriggerableJobId, RepositoryModel.getRepo(results.newRepoName, projects), false, newRetriggerTimes, props, ); }; export const reduceDictToKeys = function reduceDictToKeys(dict, keys) { const keysToKeep = keys || dict.keys(); if (!dict) { return false; } const reducedDict = {}; Object.entries(dict).forEach(([key, value]) => { if (keysToKeep.includes(key)) { reducedDict[key] = value; } }); return reducedDict; }; export const createGraphData = ( seriesData, alertSummaries, colors, symbols, commonAlerts, replicates, ) => seriesData.map((series) => { const color = colors.pop(); const symbol = symbols.pop(); // signature_id, framework_id and repository_name are // not renamed in camel case in order to match the fields // returned by the performance/summary API (since we only fetch // new data if a user adds additional tests to the graph) return { color: color || ['border-secondary', ''], symbol: symbol || ['circle', 'outline'], visible: Boolean(color), name: series.name, suite: series.suite, signature_id: series.signature_id, signatureHash: series.signature_hash, framework_id: series.framework_id, platform: series.platform, repository_name: series.repository_name, projectId: series.repository_id, id: `${series.repository_name} ${series.name}`, data: series.data.map((dataPoint) => ({ // Backend implicitly provides all dates as UTC. // Let's make this explicit, so frontend doesn't get confused. retrigger_time: new Date(`${dataPoint.submit_time}Z`), x: new Date(`${dataPoint.push_timestamp}Z`), y: dataPoint.value, z: color ? color[1] : '', _z: symbol || ['circle', 'outline'], revision: dataPoint.revision, alertSummary: alertSummaries.find( (item) => item.push_id === dataPoint.push_id, ), commonAlert: reduceDictToKeys( commonAlerts[0].find((alert) => alert.push_id === dataPoint.push_id), ['id', 'status'], ), signature_id: series.signature_id, pushId: dataPoint.push_id, jobId: dataPoint.job_id, dataPointId: dataPoint.id, application: series.application, })), application: series.application, measurementUnit: series.measurement_unit || '', lowerIsBetter: series.lower_is_better, resultSetData: series.data.map((dataPoint) => dataPoint.push_id), parentSignature: series.parent_signature, shouldAlert: series.should_alert, alertChangeType: series.alert_change_type, alertThreshold: series.alert_threshold, replicates, }; }); /** * This function will create a new Map object that holds keys composed by name and platform. * * @param {Array} results - The full list of signatures for a specific revision, framework and repo. */ export const getResultsMap = (results) => { const resultsMap = new Map(); const testNames = []; const names = []; const platforms = []; results.forEach((item) => { testNames.push(item.test); names.push(item.name); platforms.push(item.platform); const key = `${item.name} ${item.platform}`; if ( !resultsMap.has(key) || (resultsMap.has(key) && item.values.length !== 0) ) { resultsMap.set(key, item); } if (item.test !== '' && !resultsMap.has(item.test)) { resultsMap.set(item.test, item); } }); return { testNames, names, platforms, resultsMap }; };