in x-pack/platform/plugins/shared/ml/server/models/results_service/anomaly_charts.ts [460:660]
function processRecordsForDisplay(
combinedJobRecords: Record<string, MlJob>,
anomalyRecords: MlRecordForInfluencer[]
): { records: ChartRecord[]; errors: Record<string, Set<string>> | undefined } {
// Aggregate the anomaly data by detector, and entity (by/over/partition).
if (anomalyRecords.length === 0) {
return { records: [], errors: undefined };
}
// Aggregate by job, detector, and analysis fields (partition, by, over).
const aggregatedData: Record<string, any> = Object.create(null);
const jobsErrorMessage: Record<string, string> = Object.create(null);
each(anomalyRecords, (record) => {
// Check if we can plot a chart for this record, depending on whether the source data
// is chartable, and if model plot is enabled for the job.
const job = combinedJobRecords[record.job_id];
// if we already know this job has datafeed aggregations we cannot support
// no need to do more checks
if (jobsErrorMessage[record.job_id] !== undefined) {
return;
}
let isChartable =
isSourceDataChartableForDetector(job as CombinedJob, record.detector_index) ||
isMappableJob(job as CombinedJob, record.detector_index);
if (isChartable === false) {
if (isModelPlotChartableForDetector(job, record.detector_index)) {
// Check if model plot is enabled for this job.
// Need to check the entity fields for the record in case the model plot config has a terms list.
const entityFields = getEntityFieldList(record);
if (isModelPlotEnabled(job, record.detector_index, entityFields)) {
isChartable = true;
} else {
isChartable = false;
jobsErrorMessage[record.job_id] = i18n.translate(
'xpack.ml.timeSeriesJob.sourceDataNotChartableWithDisabledModelPlotMessage',
{
defaultMessage:
'source data is not viewable for this detector and model plot is disabled',
}
);
}
} else {
jobsErrorMessage[record.job_id] = i18n.translate(
'xpack.ml.timeSeriesJob.sourceDataModelPlotNotChartableMessage',
{
defaultMessage: 'both source data and model plot are not chartable for this detector',
}
);
}
}
if (isChartable === false) {
return;
}
const jobId = record.job_id;
if (aggregatedData[jobId] === undefined) {
aggregatedData[jobId] = Object.create(null);
}
const detectorsForJob = aggregatedData[jobId];
const detectorIndex = record.detector_index;
if (detectorsForJob[detectorIndex] === undefined) {
detectorsForJob[detectorIndex] = Object.create(null);
}
// TODO - work out how best to display results from detectors with just an over field.
const firstFieldName =
record.partition_field_name ?? record.by_field_name ?? record.over_field_name;
const firstFieldValue =
record.partition_field_value ?? record.by_field_value ?? record.over_field_value;
if (fieldsSafe([firstFieldName, firstFieldValue]) === false) {
return;
}
if (firstFieldName !== undefined && firstFieldValue !== undefined) {
const groupsForDetector = detectorsForJob[detectorIndex];
if (groupsForDetector[firstFieldName] === undefined) {
groupsForDetector[firstFieldName] = Object.create(null);
}
const valuesForGroup: Record<string, any> = groupsForDetector[firstFieldName];
if (valuesForGroup[firstFieldValue] === undefined) {
valuesForGroup[firstFieldValue] = Object.create(null);
}
const dataForGroupValue = valuesForGroup[firstFieldValue];
let isSecondSplit = false;
if (record.partition_field_name !== undefined) {
const splitFieldName = record.over_field_name ?? record.by_field_name;
if (splitFieldName !== undefined) {
isSecondSplit = true;
}
}
if (isSecondSplit === false) {
if (dataForGroupValue.maxScoreRecord === undefined) {
dataForGroupValue.maxScore = record.record_score;
dataForGroupValue.maxScoreRecord = record;
} else {
if (record.record_score > dataForGroupValue.maxScore) {
dataForGroupValue.maxScore = record.record_score;
dataForGroupValue.maxScoreRecord = record;
}
}
} else {
// Aggregate another level for the over or by field.
const secondFieldName = record.over_field_name ?? record.by_field_name;
const secondFieldValue = record.over_field_value ?? record.by_field_value;
if (
secondFieldName === undefined ||
secondFieldValue === undefined ||
fieldsSafe([secondFieldName, secondFieldValue]) === false
) {
return;
}
if (dataForGroupValue[secondFieldName] === undefined) {
dataForGroupValue[secondFieldName] = Object.create(null);
}
const splitsForGroup = dataForGroupValue[secondFieldName];
if (splitsForGroup[secondFieldValue] === undefined) {
splitsForGroup[secondFieldValue] = Object.create(null);
}
const dataForSplitValue = splitsForGroup[secondFieldValue];
if (dataForSplitValue.maxScoreRecord === undefined) {
dataForSplitValue.maxScore = record.record_score;
dataForSplitValue.maxScoreRecord = record;
} else {
if (record.record_score > dataForSplitValue.maxScore) {
dataForSplitValue.maxScore = record.record_score;
dataForSplitValue.maxScoreRecord = record;
}
}
}
} else {
// Detector with no partition or by field.
const dataForDetector = detectorsForJob[detectorIndex];
if (dataForDetector.maxScoreRecord === undefined) {
dataForDetector.maxScore = record.record_score;
dataForDetector.maxScoreRecord = record;
} else {
if (record.record_score > dataForDetector.maxScore) {
dataForDetector.maxScore = record.record_score;
dataForDetector.maxScoreRecord = record;
}
}
}
});
// Group job id by error message instead of by job:
const errorMessages: Record<string, Set<string>> = Object.create(null);
Object.keys(jobsErrorMessage).forEach((jobId) => {
const msg = jobsErrorMessage[jobId];
if (errorMessages[msg] === undefined) {
errorMessages[msg] = new Set([jobId]);
} else {
errorMessages[msg].add(jobId);
}
});
let recordsForSeries: ChartRecord[] = [];
// Convert to an array of the records with the highest record_score per unique series.
each(aggregatedData, (detectorsForJob) => {
each(detectorsForJob, (groupsForDetector) => {
if (groupsForDetector.errorMessage !== undefined) {
recordsForSeries.push(groupsForDetector.errorMessage);
} else {
if (groupsForDetector.maxScoreRecord !== undefined) {
// Detector with no partition / by field.
recordsForSeries.push(groupsForDetector.maxScoreRecord);
} else {
each(groupsForDetector, (valuesForGroup) => {
each(valuesForGroup, (dataForGroupValue) => {
if (dataForGroupValue.maxScoreRecord !== undefined) {
recordsForSeries.push(dataForGroupValue.maxScoreRecord);
} else {
// Second level of aggregation for partition and by/over.
each(dataForGroupValue, (splitsForGroup) => {
each(splitsForGroup, (dataForSplitValue) => {
recordsForSeries.push(dataForSplitValue.maxScoreRecord);
});
});
}
});
});
}
}
});
});
recordsForSeries = sortBy(recordsForSeries, 'record_score').reverse();
return { records: recordsForSeries, errors: errorMessages };
}