in public/pages/utils/anomalyResultUtils.ts [592:1023]
endTime: get(rawAnomaly, 'data_end_time'),
plotTime: get(rawAnomaly, 'data_end_time'),
});
});
}
}
});
return {
anomalies: anomalies,
featureData: featureData,
};
};
export const parseAnomalySummary = (
anomalySummaryResult: any
): AnomalySummary => {
const anomalyCount = get(
anomalySummaryResult,
'response.aggregations.count_anomalies.value',
0
);
return {
anomalyOccurrence: anomalyCount,
minAnomalyGrade: anomalyCount
? toFixedNumberForAnomaly(
get(
anomalySummaryResult,
'response.aggregations.min_anomaly_grade.value'
)
)
: 0,
maxAnomalyGrade: anomalyCount
? toFixedNumberForAnomaly(
get(
anomalySummaryResult,
'response.aggregations.max_anomaly_grade.value'
)
)
: 0,
avgAnomalyGrade: anomalyCount
? toFixedNumberForAnomaly(
get(
anomalySummaryResult,
'response.aggregations.avg_anomaly_grade.value'
)
)
: 0,
minConfidence: anomalyCount
? toFixedNumberForAnomaly(
get(
anomalySummaryResult,
'response.aggregations.min_confidence.value'
)
)
: 0,
maxConfidence: anomalyCount
? toFixedNumberForAnomaly(
get(
anomalySummaryResult,
'response.aggregations.max_confidence.value'
)
)
: 0,
lastAnomalyOccurrence: anomalyCount
? minuteDateFormatter(
get(
anomalySummaryResult,
'response.aggregations.max_data_end_time.value'
)
)
: '',
};
};
export const parsePureAnomalies = (
anomalySummaryResult: any
): AnomalyData[] => {
const anomaliesHits = get(anomalySummaryResult, 'response.hits.hits', []);
const anomalies = [] as AnomalyData[];
if (anomaliesHits.length > 0) {
anomaliesHits.forEach((item: any) => {
const rawAnomaly = get(item, '_source');
anomalies.push({
anomalyGrade: toFixedNumberForAnomaly(get(rawAnomaly, 'anomaly_grade')),
confidence: toFixedNumberForAnomaly(get(rawAnomaly, 'confidence')),
startTime: get(rawAnomaly, 'data_start_time'),
endTime: get(rawAnomaly, 'data_end_time'),
plotTime: get(rawAnomaly, 'data_end_time'),
entity: get(rawAnomaly, 'entity'),
});
});
}
return anomalies;
};
export type FeatureDataPoint = {
isMissing: boolean;
plotTime: number;
startTime: number;
endTime: number;
};
export const FEATURE_DATA_CHECK_WINDOW_OFFSET = 2;
export const getFeatureDataPoints = (
featureData: FeatureAggregationData[],
interval: number,
dateRange: DateRange | undefined
): FeatureDataPoint[] => {
const featureDataPoints = [] as FeatureDataPoint[];
if (!dateRange) {
return featureDataPoints;
}
const existingTimes = isEmpty(featureData)
? []
: featureData
.map((feature) => getRoundedTimeInMin(feature.startTime))
.filter((featureTime) => featureTime != undefined);
for (
let currentTime = getRoundedTimeInMin(dateRange.startDate);
currentTime <
// skip checking for latest interval as data point for it may not be delivered in time
getRoundedTimeInMin(
dateRange.endDate -
FEATURE_DATA_CHECK_WINDOW_OFFSET * interval * MIN_IN_MILLI_SECS
);
currentTime += interval * MIN_IN_MILLI_SECS
) {
const isExisting = findTimeExistsInWindow(
existingTimes,
getRoundedTimeInMin(currentTime),
getRoundedTimeInMin(currentTime) + interval * MIN_IN_MILLI_SECS
);
featureDataPoints.push({
isMissing: !isExisting,
plotTime: currentTime + interval * MIN_IN_MILLI_SECS,
startTime: currentTime,
endTime: currentTime + interval * MIN_IN_MILLI_SECS,
});
}
return featureDataPoints;
};
const findTimeExistsInWindow = (
timestamps: any[],
startTime: number,
endTime: number
): boolean => {
// timestamps is in desc order
let result = false;
if (isEmpty(timestamps)) {
return result;
}
let left = 0;
let right = timestamps.length - 1;
while (left <= right) {
let middle = Math.floor((right + left) / 2);
if (timestamps[middle] >= startTime && timestamps[middle] < endTime) {
result = true;
break;
}
if (timestamps[middle] < startTime) {
right = middle - 1;
}
if (timestamps[middle] >= endTime) {
left = middle + 1;
}
}
return result;
};
const getRoundedTimeInMin = (timestamp: number): number => {
return Math.round(timestamp / MIN_IN_MILLI_SECS) * MIN_IN_MILLI_SECS;
};
const sampleFeatureMissingDataPoints = (
featureMissingDataPoints: FeatureDataPoint[],
dateRange?: DateRange
): FeatureDataPoint[] => {
if (!dateRange) {
return featureMissingDataPoints;
}
const sampleTimeWindows = calculateTimeWindowsWithMaxDataPoints(
MAX_FEATURE_ANNOTATIONS,
dateRange
);
const sampledResults = [] as FeatureDataPoint[];
for (const timeWindow of sampleTimeWindows) {
const sampledDataPoint = getMiddleDataPoint(
getDataPointsInWindow(featureMissingDataPoints, timeWindow)
);
if (sampledDataPoint) {
sampledResults.push({
...sampledDataPoint,
startTime: Math.min(timeWindow.startDate, sampledDataPoint.startTime),
endTime: Math.max(timeWindow.endDate, sampledDataPoint.endTime),
} as FeatureDataPoint);
}
}
return sampledResults;
};
const getMiddleDataPoint = (arr: any[]) => {
if (arr && arr.length > 0) {
return arr[Math.floor(arr.length / 2)];
}
return undefined;
};
const getDataPointsInWindow = (
dataPoints: FeatureDataPoint[],
timeWindow: DateRange
) => {
return dataPoints.filter(
(dataPoint) =>
get(dataPoint, 'plotTime', 0) >= timeWindow.startDate &&
get(dataPoint, 'plotTime', 0) < timeWindow.endDate
);
};
const generateFeatureMissingAnnotations = (
featureMissingDataPoints: FeatureDataPoint[]
) => {
return featureMissingDataPoints.map((feature) => ({
dataValue: feature.plotTime,
details: `There is feature data point missing between ${moment(
feature.startTime
).format('MM/DD/YY h:mm A')} and ${moment(feature.endTime).format(
'MM/DD/YY h:mm A'
)}`,
header: dateFormatter(feature.plotTime),
}));
};
const finalizeFeatureMissingDataAnnotations = (
featureMissingDataPoints: any[],
dateRange?: DateRange
) => {
const sampledFeatureMissingDataPoints = sampleFeatureMissingDataPoints(
featureMissingDataPoints,
dateRange
);
return generateFeatureMissingAnnotations(sampledFeatureMissingDataPoints);
};
export const getFeatureMissingDataAnnotations = (
featureData: FeatureAggregationData[],
interval: number,
queryDateRange?: DateRange,
displayDateRange?: DateRange
) => {
const featureMissingDataPoints = getFeatureDataPoints(
featureData,
interval,
queryDateRange
).filter((dataPoint) => get(dataPoint, 'isMissing', false));
const featureMissingAnnotations = finalizeFeatureMissingDataAnnotations(
featureMissingDataPoints,
displayDateRange
);
return featureMissingAnnotations;
};
// returns feature data points(missing/existing both included) for detector in a map like
// {
// 'featureName': data points[]
// }
export const getFeatureDataPointsForDetector = (
detector: Detector,
featuresData: { [key: string]: FeatureAggregationData[] },
interval: number,
dateRange?: DateRange
) => {
let featureDataPointsForDetector = {} as {
[key: string]: FeatureDataPoint[];
};
const allFeatures = get(
detector,
'featureAttributes',
[] as FeatureAttributes[]
);
allFeatures.forEach((feature) => {
//@ts-ignore
const featureData = featuresData[feature.featureId];
const featureDataPoints = getFeatureDataPoints(
featureData,
interval,
dateRange
);
featureDataPointsForDetector = {
...featureDataPointsForDetector,
[feature.featureName]: featureDataPoints,
};
});
return featureDataPointsForDetector;
};
export const getFeatureMissingSeverities = (featuresDataPoint: {
[key: string]: FeatureDataPoint[];
}): Map<MISSING_FEATURE_DATA_SEVERITY, string[]> => {
const featureMissingSeverities = new Map();
for (const [featureName, featureDataPoints] of Object.entries(
featuresDataPoint
)) {
// all feature data points should have same length
let featuresWithMissingData = [] as string[];
if (featureDataPoints.length <= 1) {
// return empty map
return featureMissingSeverities;
}
if (
featureDataPoints.length === 2 &&
featureDataPoints[0].isMissing &&
featureDataPoints[1].isMissing
) {
if (featureMissingSeverities.has(MISSING_FEATURE_DATA_SEVERITY.YELLOW)) {
featuresWithMissingData = featureMissingSeverities.get(
MISSING_FEATURE_DATA_SEVERITY.YELLOW
);
}
featuresWithMissingData.push(featureName);
featureMissingSeverities.set(
MISSING_FEATURE_DATA_SEVERITY.YELLOW,
featuresWithMissingData
);
continue;
}
const orderedFeatureDataPoints = orderBy(
featureDataPoints,
// sort by plot time in desc order
(dataPoint) => get(dataPoint, 'plotTime', 0),
SORT_DIRECTION.DESC
);
// feature has >= 3 data points
if (
orderedFeatureDataPoints.length >= 3 &&
orderedFeatureDataPoints[0].isMissing &&
orderedFeatureDataPoints[1].isMissing
) {
// at least latest 2 ones are missing
let currentSeverity = MISSING_FEATURE_DATA_SEVERITY.YELLOW;
if (orderedFeatureDataPoints[2].isMissing) {
// all the latest 3 ones are missing
currentSeverity = MISSING_FEATURE_DATA_SEVERITY.RED;
}
if (featureMissingSeverities.has(currentSeverity)) {
featuresWithMissingData = featureMissingSeverities.get(currentSeverity);
}
featuresWithMissingData.push(featureName);
featureMissingSeverities.set(currentSeverity, featuresWithMissingData);
}
}
return featureMissingSeverities;
};
export const getFeatureDataMissingMessageAndActionItem = (
featureMissingSev: MISSING_FEATURE_DATA_SEVERITY | undefined,
featuresWithMissingData: string[],
hideFeatureMessage: boolean
) => {
switch (featureMissingSev) {
case MISSING_FEATURE_DATA_SEVERITY.YELLOW:
return {
message: `Recent data is missing for feature${
featuresWithMissingData.length > 1 ? 's' : ''
}: ${featuresWithMissingData.join(
', '
)}. So, anomaly result is missing during this time.`,
actionItem:
'Make sure your data is ingested correctly.' + hideFeatureMessage
? ''
: ' See the feature data shown below for more details.',
};
case MISSING_FEATURE_DATA_SEVERITY.RED:
return {
message: `Data is not being ingested correctly for feature${
featuresWithMissingData.length > 1 ? 's' : ''
}: ${featuresWithMissingData.join(
', '
)}. So, anomaly result is missing during this time.`,
actionItem:
`${DETECTOR_INIT_FAILURES.NO_TRAINING_DATA.actionItem}` +
hideFeatureMessage
? ''
: ' See the feature data shown below for more details.',
};
default:
return {
message: '',
actionItem: '',
};
}
};
// Generates query to get the top anomalous entities (or entity pairs)
// for some detector, sorting by severity or occurrence.
export const getTopAnomalousEntitiesQuery = (
startTime: number,
endTime: number,
detectorId: string,
size: number,
sortType: AnomalyHeatmapSortType,
isMultiCategory: boolean,
isHistorical?: boolean,
taskId?: string,
includedEntities?: Entity[]
) => {
const termField =
isHistorical && taskId ? { task_id: taskId } : { detector_id: detectorId };
// To handle BWC, we will return 2 possible queries based on the # of categorical fields:
// (1) legacy way (1 category field): bucket aggregate over the single, nested, 'entity.value' field
// (2) new way (>= 2 category fields): bucket aggregate over the new 'model_id' field
let requestBody = isMultiCategory
? {
size: 0,
query: {
bool: {
filter: [
{
range: {