in packages/orchestratorlib/src/utility.ts [3270:3482]
public static generateLabelObjectEvaluationReport(
scoringFunctionToPredictionStructureWithScoreLabelObject: (
utteranceLabelsPairArray: StructTextLabelObjects[],
labelArrayAndMap: ILabelArrayAndMap,
multiLabelPredictionThreshold: number,
unknownLabelPredictionThreshold: number) => PredictionStructureWithScoreLabelObject[],
dataSetLabels: string[],
utteranceLabelsMap: Map<string, Label[]>,
utteranceLabelDuplicateMap: Map<string, Label[]>,
ambiguousClosenessThreshold: number,
lowConfidenceScoreThreshold: number,
multiLabelPredictionThreshold: number,
unknownLabelPredictionThreshold: number,
unknownSpuriousEntityLabelsProcessed: {
'utteranceUnknownEntityLabelsMap': Map<string, Label[]>;
'utteranceUnknownEntityLabelDuplicateMap': Map<string, Label[]>;
'utteranceSpuriousEntityLabelsMap': Map<string, Label[]>;
'utteranceSpuriousEntityLabelDuplicateMap': Map<string, Label[]>;
'utteranceLabelMapSetAddedWithUnknownLabel': boolean;
'utteranceLabelDuplicateMapSetAddedWithUnknownLabel': boolean;}): {
'evaluationReportLabelUtteranceStatistics': {
'evaluationSummary': string;
'labelArrayAndMap': ILabelArrayAndMap;
'labelStatisticsAndHtmlTable': {
'labelUtterancesMap': Map<string, Set<string>>;
'labelUtterancesTotal': number;
'labelStatistics': string[][];
'labelStatisticsHtml': string;};
'utteranceStatisticsAndHtmlTable': {
'utteranceStatisticsMap': Map<number, number>;
'utteranceStatistics': StructTextNumber[];
'utteranceCount': number;
'utteranceStatisticsHtml': string;};
'spuriousLabelStatisticsAndHtmlTable': {
'spuriousLabelUtterancesMap': StructTextStringSet[];
'spuriousLabelUtterancesTotal': number;
'spuriousLabelStatistics': string[][];
'spuriousLabelStatisticsHtml': string; };
'utterancesMultiLabelArrays': StructTextText[];
'utterancesMultiLabelArraysHtml': string;
'utteranceLabelDuplicateHtml': string; };
'evaluationReportAnalyses': {
'evaluationSummary': string;
'ambiguousAnalysis': {
'scoringAmbiguousUtterancesArrays': string[][];
'scoringAmbiguousUtterancesArraysHtml': string;
'scoringAmbiguousUtteranceSimpleArrays': string[][];};
'misclassifiedAnalysis': {
'scoringMisclassifiedUtterancesArrays': string[][];
'scoringMisclassifiedUtterancesArraysHtml': string;
'scoringMisclassifiedUtterancesSimpleArrays': string[][];};
'lowConfidenceAnalysis': {
'scoringLowConfidenceUtterancesArrays': string[][];
'scoringLowConfidenceUtterancesArraysHtml': string;
'scoringLowConfidenceUtterancesSimpleArrays': string[][];};
'confusionMatrixAnalysis': {
'confusionMatrix': IConfusionMatrix;
'multiLabelObjectConfusionMatrixExact': MultiLabelObjectConfusionMatrixExact;
'multiLabelObjectConfusionMatrixSubset': MultiLabelObjectConfusionMatrixSubset;
'predictingConfusionMatrixOutputLines': string[][];
'confusionMatrixMetricsHtml': string;
'confusionMatrixAverageMetricsHtml': string;
'confusionMatrixAverageDescriptionMetricsHtml': string;};};
'predictionStructureWithScoreLabelObjectArray': PredictionStructureWithScoreLabelObject[];
'scoreOutputLines': string[][];
'groundTruthJsonContent': string;
'predictionJsonContent': string;
} {
// ---- NOTE ---- load the evaluation summary template.
const evaluationSummary: string = EvaluationSummaryTemplateHtml.html;
// ---- NOTE ---- generate evaluation report before calling the score() function.
Utility.debuggingLog('Utility.generateLabelObjectEvaluationReport(), ready to call Utility.generateLabelObjectEvaluationReportLabelUtteranceStatistics()');
const evaluationReportLabelUtteranceStatistics: {
'evaluationSummary': string;
'labelArrayAndMap': ILabelArrayAndMap;
'labelStatisticsAndHtmlTable': {
'labelUtterancesMap': Map<string, Set<string>>;
'labelUtterancesTotal': number;
'labelStatistics': string[][];
'labelStatisticsHtml': string;};
'utteranceStatisticsAndHtmlTable': {
'utteranceStatisticsMap': Map<number, number>;
'utteranceStatistics': StructTextNumber[];
'utteranceCount': number;
'utteranceStatisticsHtml': string;};
'spuriousLabelStatisticsAndHtmlTable': {
'spuriousLabelUtterancesMap': StructTextStringSet[];
'spuriousLabelUtterancesTotal': number;
'spuriousLabelStatistics': string[][];
'spuriousLabelStatisticsHtml': string; };
'utterancesMultiLabelArrays': StructTextText[];
'utterancesMultiLabelArraysHtml': string;
'utteranceLabelDuplicateHtml': string;
} = Utility.generateLabelObjectEvaluationReportLabelUtteranceStatistics(
evaluationSummary,
dataSetLabels,
utteranceLabelsMap,
utteranceLabelDuplicateMap,
unknownSpuriousEntityLabelsProcessed,
'{LABEL_TEXT_STATISTICS}',
'{TEXT_DUPLICATES}',
'{SPURIOUS_UTTERANCE_LABELS}',
(unknownLabelPredictionThreshold > 0)); // ---- NOTE ---- there is no UNKNOWN prediction unless the threshold is higher than 0.
Utility.debuggingLog('Utility.generateLabelObjectEvaluationReport(), finished calling Utility.generateLabelObjectEvaluationReportLabelUtteranceStatistics()');
// ---- NOTE ---- collect utterance prediction and scores.
Utility.debuggingLog('Utility.generateLabelObjectEvaluationReport(), ready to call scoringFunctionToPredictionStructureWithScoreLabelObject()');
const utteranceLabelsPairArray: StructTextLabelObjects[] = [...utteranceLabelsMap.keys()].map(
(x: string) => new StructTextLabelObjects(x, utteranceLabelsMap.get(x) as Label[]));
const predictionStructureWithScoreLabelObjectArray: PredictionStructureWithScoreLabelObject[] =
scoringFunctionToPredictionStructureWithScoreLabelObject(
utteranceLabelsPairArray,
evaluationReportLabelUtteranceStatistics.labelArrayAndMap,
multiLabelPredictionThreshold,
unknownLabelPredictionThreshold);
// ---- NOTE-REFACTORED ---- const predictionStructureWithScoreLabelObjectArray: PredictionStructureWithScoreLabelObject[] = UtilityLabelResolver.score(
// ---- NOTE-REFACTORED ---- utteranceLabelsPairArray,
// ---- NOTE-REFACTORED ---- evaluationReportLabelUtteranceStatistics.labelArrayAndMap,
// ---- NOTE-REFACTORED ---- multiLabelPredictionThreshold,
// ---- NOTE-REFACTORED ---- unknownLabelPredictionThreshold);
Utility.debuggingLog('Utility.generateLabelObjectEvaluationReport(), finished calling scoringFunctionToPredictionStructureWithScoreLabelObject()');
// ---- NOTE ---- generate evaluation report after calling the score() function.
Utility.debuggingLog('Utility.generateLabelObjectEvaluationReport(), ready to call Utility.generateLabelObjectEvaluationReportAnalyses()');
const evaluationReportAnalyses: {
'evaluationSummary': string;
'ambiguousAnalysis': {
'scoringAmbiguousUtterancesArrays': string[][];
'scoringAmbiguousUtterancesArraysHtml': string;
'scoringAmbiguousUtteranceSimpleArrays': string[][];};
'misclassifiedAnalysis': {
'scoringMisclassifiedUtterancesArrays': string[][];
'scoringMisclassifiedUtterancesArraysHtml': string;
'scoringMisclassifiedUtterancesSimpleArrays': string[][];};
'lowConfidenceAnalysis': {
'scoringLowConfidenceUtterancesArrays': string[][];
'scoringLowConfidenceUtterancesArraysHtml': string;
'scoringLowConfidenceUtterancesSimpleArrays': string[][];};
'confusionMatrixAnalysis': {
'confusionMatrix': IConfusionMatrix;
'multiLabelObjectConfusionMatrixExact': MultiLabelObjectConfusionMatrixExact;
'multiLabelObjectConfusionMatrixSubset': MultiLabelObjectConfusionMatrixSubset;
'predictingConfusionMatrixOutputLines': string[][];
'confusionMatrixMetricsHtml': string;
'confusionMatrixAverageMetricsHtml': string;
'confusionMatrixAverageDescriptionMetricsHtml': string;};
} = Utility.generateLabelObjectEvaluationReportAnalyses(
evaluationReportLabelUtteranceStatistics.evaluationSummary,
evaluationReportLabelUtteranceStatistics.labelArrayAndMap,
predictionStructureWithScoreLabelObjectArray,
ambiguousClosenessThreshold,
lowConfidenceScoreThreshold,
unknownLabelPredictionThreshold);
Utility.debuggingLog('Utility.generateLabelObjectEvaluationReport(), finished calling Utility.generateLabelObjectEvaluationReportAnalyses()');
// ---- NOTE ---- generate score output file lines.
Utility.debuggingLog('Utility.generateLabelObjectEvaluationReport(), ready to call Utility.generateLabelObjectScoreOutputLines()');
const scoreOutputLines: string[][] = Utility.generateLabelObjectScoreOutputLines(
predictionStructureWithScoreLabelObjectArray);
Utility.debuggingLog('Utility.generateLabelObjectEvaluationReport(), finished calling Utility.generateLabelObjectScoreOutputLines()');
// ---- NOTE ---- generate score output file lines.
Utility.debuggingLog('Utility.generateLabelObjectEvaluationReport(), ready to call Utility.generateLabelObjectGroundTruthJsons()');
const groundTruthJsons: Array<{
'text': string;
'intents': string[];
'entities': Array<{
'entity': string;
'startPos': number;
'endPos': number;
'text': string;
}>;
}> = Utility.generateLabelObjectGroundTruthJsons(
predictionStructureWithScoreLabelObjectArray);
const groundTruthJsonContent: string = Utility.jsonStringify(groundTruthJsons);
Utility.debuggingLog('Utility.generateLabelObjectEvaluationReport(), finished calling Utility.generateLabelObjectGroundTruthJsons()');
// ---- NOTE ---- generate score output file lines.
Utility.debuggingLog('Utility.generateLabelObjectEvaluationReport(), ready to call Utility.generateLabelObjectPredictionJsons()');
const predictionJsons: Array<{
'text': string;
'intents': string[];
'entities': Array<{
'entity': string;
'startPos': number;
'endPos': number;
'text': string;
}>;
'intent_scores': Array<{
'intent': string;
'score': number;
}>;
'entity_scores': Array<{
'entity': string;
'startPos': number;
'endPos': number;
'score': number;
}>;
}> = Utility.generateLabelObjectPredictionJsons(
predictionStructureWithScoreLabelObjectArray);
const predictionJsonContent: string = Utility.jsonStringify(predictionJsons);
Utility.debuggingLog('Utility.generateLabelObjectEvaluationReport(), finished calling Utility.generateLabelObjectPredictionJsons()');
// ---- NOTE ---- debugging ouput.
// if (Utility.toPrintDetailedDebuggingLogToConsole) {
// Utility.debuggingLog(`Utility.generateLabelObjectEvaluationReport(), labelArrayAndMap.stringArray=${Utility.jsonStringify(evaluationReportLabelUtteranceStatistics.labelArrayAndMap.stringArray)}`);
// Utility.debuggingLog(`Utility.generateLabelObjectEvaluationReport(), labelArrayAndMap.stringMap=${Utility.jsonStringify(evaluationReportLabelUtteranceStatistics.labelArrayAndMap.stringMap)}`);
// const labels: any = LabelResolver.getLabels(LabelType.Intent);
// Utility.debuggingLog(`Utility.generateLabelObjectEvaluationReport(), labels=${Utility.jsonStringify(labels)}`);
// }
// ---- NOTE ---- return
return {
evaluationReportLabelUtteranceStatistics,
evaluationReportAnalyses,
predictionStructureWithScoreLabelObjectArray,
scoreOutputLines,
groundTruthJsonContent,
predictionJsonContent};
}