in packages/orchestratorlib/src/utility.ts [2364:2577]
public static generateLabelStringEvaluationReport(
scoringFunctionToPredictionStructureWithScoreLabelString: (
utteranceLabelsPairArray: StructTextLabelStrings[],
labelArrayAndMap: ILabelArrayAndMap,
multiLabelPredictionThreshold: number,
unknownLabelPredictionThreshold: number) => PredictionStructureWithScoreLabelString[],
dataSetLabels: string[],
utteranceLabelsMap: Map<string, Set<string>>,
utteranceLabelDuplicateMap: Map<string, Set<string>>,
ambiguousClosenessThreshold: number,
lowConfidenceScoreThreshold: number,
multiLabelPredictionThreshold: number,
unknownLabelPredictionThreshold: number,
unknownSpuriousLabelsProcessed: {
'utteranceUnknownLabelsMap': Map<string, Set<string>>;
'utteranceUnknownLabelDuplicateMap': Map<string, Set<string>>;
'utteranceSpuriousLabelsMap': Map<string, Set<string>>;
'utteranceSpuriousLabelDuplicateMap': Map<string, Set<string>>;
'utteranceLabelMapSetAddedWithUnknownLabel': boolean;
'utteranceLabelDuplicateMapSetAddedWithUnknownLabel': boolean;}): {
'evaluationReportLabelUtteranceStatistics': {
'evaluationSummary': string;
'labelArrayAndMap': ILabelArrayAndMap;
'labelStatisticsAndHtmlTable': {
'labelUtterancesMap': Map<string, Set<string>>;
'labelUtterancesTotal': number;
'labelStatistics': string[][];
'labelStatisticsHtml': string;};
'utteranceStatisticsAndHtmlTable': {
'utteranceStatisticsMap': Map<number, number>;
'utteranceStatistics': StructTextNumber[];
'utteranceCount': number;
'utteranceStatisticsHtml': string;};
'spuriousLabelStatisticsAndHtmlTable': {
'spuriousLabelUtterancesMap': StructTextStringSet[];
'spuriousLabelUtterancesTotal': number;
'spuriousLabelStatistics': string[][];
'spuriousLabelStatisticsHtml': string; };
'utterancesMultiLabelArrays': StructTextText[];
'utterancesMultiLabelArraysHtml': string;
'utteranceLabelDuplicateHtml': string; };
'evaluationReportAnalyses': {
'evaluationSummary': string;
'ambiguousAnalysis': {
'scoringAmbiguousUtterancesArrays': string[][];
'scoringAmbiguousUtterancesArraysHtml': string;
'scoringAmbiguousUtteranceSimpleArrays': string[][];};
'misclassifiedAnalysis': {
'scoringMisclassifiedUtterancesArrays': string[][];
'scoringMisclassifiedUtterancesArraysHtml': string;
'scoringMisclassifiedUtterancesSimpleArrays': string[][];};
'lowConfidenceAnalysis': {
'scoringLowConfidenceUtterancesArrays': string[][];
'scoringLowConfidenceUtterancesArraysHtml': string;
'scoringLowConfidenceUtterancesSimpleArrays': string[][];};
'confusionMatrixAnalysis': {
'confusionMatrix': IConfusionMatrix;
'multiLabelObjectConfusionMatrixExact': MultiLabelObjectConfusionMatrixExact;
'multiLabelObjectConfusionMatrixSubset': MultiLabelObjectConfusionMatrixSubset;
'predictingConfusionMatrixOutputLines': string[][];
'confusionMatrixMetricsHtml': string;
'confusionMatrixAverageMetricsHtml': string;
'confusionMatrixAverageDescriptionMetricsHtml': string;};};
'predictionStructureWithScoreLabelStringArray': PredictionStructureWithScoreLabelString[];
'scoreOutputLines': string[][];
'groundTruthJsonContent': string;
'predictionJsonContent': string;
} {
// ---- NOTE ---- load the evaluation summary template.
const evaluationSummary: string = EvaluationSummaryTemplateHtml.html;
// ---- NOTE ---- generate evaluation report before calling the score() function.
Utility.debuggingLog('Utility.generateLabelStringEvaluationReport(), ready to call Utility.generateLabelStringEvaluationReportLabelUtteranceStatistics()');
const evaluationReportLabelUtteranceStatistics: {
'evaluationSummary': string;
'labelArrayAndMap': ILabelArrayAndMap;
'labelStatisticsAndHtmlTable': {
'labelUtterancesMap': Map<string, Set<string>>;
'labelUtterancesTotal': number;
'labelStatistics': string[][];
'labelStatisticsHtml': string;};
'utteranceStatisticsAndHtmlTable': {
'utteranceStatisticsMap': Map<number, number>;
'utteranceStatistics': StructTextNumber[];
'utteranceCount': number;
'utteranceStatisticsHtml': string;};
'spuriousLabelStatisticsAndHtmlTable': {
'spuriousLabelUtterancesMap': StructTextStringSet[];
'spuriousLabelUtterancesTotal': number;
'spuriousLabelStatistics': string[][];
'spuriousLabelStatisticsHtml': string; };
'utterancesMultiLabelArrays': StructTextText[];
'utterancesMultiLabelArraysHtml': string;
'utteranceLabelDuplicateHtml': string;
} = Utility.generateLabelStringEvaluationReportLabelUtteranceStatistics(
evaluationSummary,
dataSetLabels,
utteranceLabelsMap,
utteranceLabelDuplicateMap,
unknownSpuriousLabelsProcessed,
'{LABEL_TEXT_STATISTICS}',
'{TEXT_DUPLICATES}',
'{SPURIOUS_UTTERANCE_LABELS}',
(unknownLabelPredictionThreshold > 0)); // ---- NOTE ---- there is no UNKNOWN prediction unless the threshold is higher than 0.
Utility.debuggingLog('Utility.generateLabelStringEvaluationReport(), finished calling Utility.generateLabelStringEvaluationReportLabelUtteranceStatistics()');
// ---- NOTE ---- collect utterance prediction and scores.
Utility.debuggingLog('Utility.generateLabelStringEvaluationReport(), ready to call scoringFunctionToPredictionStructureWithScoreLabelString()');
const utteranceLabelsPairArray: StructTextLabelStrings[] = [...utteranceLabelsMap.keys()].map(
(key: string) => new StructTextStringSet(key, utteranceLabelsMap.get(key) as Set<string>)).map(
(x: StructTextStringSet) => new StructTextLabelStrings(x.text, [...x.stringSet]));
const predictionStructureWithScoreLabelStringArray: PredictionStructureWithScoreLabelString[] =
scoringFunctionToPredictionStructureWithScoreLabelString(
utteranceLabelsPairArray,
evaluationReportLabelUtteranceStatistics.labelArrayAndMap,
multiLabelPredictionThreshold,
unknownLabelPredictionThreshold);
// ---- NOTE-REFACTORED ---- const predictionStructureWithScoreLabelStringArray: PredictionStructureWithScoreLabelString[] = UtilityLabelResolver.score(
// ---- NOTE-REFACTORED ---- utteranceLabelsPairArray,
// ---- NOTE-REFACTORED ---- evaluationReportLabelUtteranceStatistics.labelArrayAndMap,
// ---- NOTE-REFACTORED ---- multiLabelPredictionThreshold,
// ---- NOTE-REFACTORED ---- unknownLabelPredictionThreshold);
Utility.debuggingLog('Utility.generateLabelStringEvaluationReport(), finished calling scoringFunctionToPredictionStructureWithScoreLabelString()');
// ---- NOTE ---- generate evaluation report after calling the score() function.
Utility.debuggingLog('Utility.generateLabelStringEvaluationReport(), ready to call Utility.generateLabelStringEvaluationReportAnalyses()');
const evaluationReportAnalyses: {
'evaluationSummary': string;
'ambiguousAnalysis': {
'scoringAmbiguousUtterancesArrays': string[][];
'scoringAmbiguousUtterancesArraysHtml': string;
'scoringAmbiguousUtteranceSimpleArrays': string[][];};
'misclassifiedAnalysis': {
'scoringMisclassifiedUtterancesArrays': string[][];
'scoringMisclassifiedUtterancesArraysHtml': string;
'scoringMisclassifiedUtterancesSimpleArrays': string[][];};
'lowConfidenceAnalysis': {
'scoringLowConfidenceUtterancesArrays': string[][];
'scoringLowConfidenceUtterancesArraysHtml': string;
'scoringLowConfidenceUtterancesSimpleArrays': string[][];};
'confusionMatrixAnalysis': {
'confusionMatrix': IConfusionMatrix;
'multiLabelObjectConfusionMatrixExact': MultiLabelObjectConfusionMatrixExact;
'multiLabelObjectConfusionMatrixSubset': MultiLabelObjectConfusionMatrixSubset;
'predictingConfusionMatrixOutputLines': string[][];
'confusionMatrixMetricsHtml': string;
'confusionMatrixAverageMetricsHtml': string;
'confusionMatrixAverageDescriptionMetricsHtml': string;};
} = Utility.generateLabelStringEvaluationReportAnalyses(
evaluationReportLabelUtteranceStatistics.evaluationSummary,
evaluationReportLabelUtteranceStatistics.labelArrayAndMap,
predictionStructureWithScoreLabelStringArray,
ambiguousClosenessThreshold,
lowConfidenceScoreThreshold,
unknownLabelPredictionThreshold);
Utility.debuggingLog('Utility.generateLabelStringEvaluationReport(), finished calling Utility.generateLabelStringEvaluationReportAnalyses()');
// ---- NOTE ---- generate score output file lines.
Utility.debuggingLog('Utility.generateLabelStringEvaluationReport(), ready to call Utility.generateLabelStringScoreOutputLines()');
const scoreOutputLines: string[][] = Utility.generateLabelStringScoreOutputLines(
predictionStructureWithScoreLabelStringArray);
Utility.debuggingLog('Utility.generateLabelStringEvaluationReport(), finished calling Utility.generateLabelStringScoreOutputLines()');
// ---- NOTE ---- generate score output file lines.
Utility.debuggingLog('Utility.generateLabelStringEvaluationReport(), ready to call Utility.generateLabelStringGroundTruthJsons()');
const groundTruthJsons: Array<{
'text': string;
'intents': string[];
'entities': Array<{
'entity': string;
'startPos': number;
'endPos': number;
'text': string;
}>;
}> = Utility.generateLabelStringGroundTruthJsons(
predictionStructureWithScoreLabelStringArray);
const groundTruthJsonContent: string = Utility.jsonStringify(groundTruthJsons);
Utility.debuggingLog('Utility.generateLabelStringEvaluationReport(), finished calling Utility.generateLabelStringGroundTruthJsons()');
// ---- NOTE ---- generate score output file lines.
Utility.debuggingLog('Utility.generateLabelStringEvaluationReport(), ready to call Utility.generateLabelStringPredictionJsons()');
const predictionJsons: Array<{
'text': string;
'intents': string[];
'entities': Array<{
'entity': string;
'startPos': number;
'endPos': number;
'text': string;
}>;
'intent_scores': Array<{
'intent': string;
'score': number;
}>;
'entity_scores': Array<{
'entity': string;
'startPos': number;
'endPos': number;
'score': number;
}>;
}> = Utility.generateLabelStringPredictionJsons(
predictionStructureWithScoreLabelStringArray);
const predictionJsonContent: string = Utility.jsonStringify(predictionJsons);
Utility.debuggingLog('Utility.generateLabelStringEvaluationReport(), finished calling Utility.generateLabelStringPredictionJsons()');
// ---- NOTE ---- debugging ouput.
// if (Utility.toPrintDetailedDebuggingLogToConsole) {
// Utility.debuggingLog(`Utility.generateLabelStringEvaluationReport(), labelArrayAndMap.stringArray=${Utility.jsonStringify(evaluationReportLabelUtteranceStatistics.labelArrayAndMap.stringArray)}`);
// Utility.debuggingLog(`Utility.generateLabelStringEvaluationReport(), labelArrayAndMap.stringMap=${Utility.jsonStringify(evaluationReportLabelUtteranceStatistics.labelArrayAndMap.stringMap)}`);
// const labels: any = LabelResolver.getLabels(LabelType.Intent);
// Utility.debuggingLog(`Utility.generateLabelStringEvaluationReport(), labels=${Utility.jsonStringify(labels)}`);
// }
// ---- NOTE ---- return
return {
evaluationReportLabelUtteranceStatistics,
evaluationReportAnalyses,
predictionStructureWithScoreLabelStringArray,
scoreOutputLines,
groundTruthJsonContent,
predictionJsonContent};
}