in packages/orchestratorlib/src/utilitylabelresolver.ts [84:308]
public static scoreBatchStringLabels(
utteranceLabelsPairArray: StructTextLabelStrings[],
labelArrayAndMap: ILabelArrayAndMap,
multiLabelPredictionThreshold: number,
unknownLabelPredictionThreshold: number): PredictionStructureWithScoreLabelString[] {
// -----------------------------------------------------------------------
Utility.debuggingLog('UtilityLabelResolver.scoreBatchStringLabels(), entering');
if (Utility.toPrintDetailedDebuggingLogToConsole) {
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchObjectLabels(), multiLabelPredictionThreshold="${multiLabelPredictionThreshold}"`);
}
if (Utility.toPrintDetailedDebuggingLogToConsole) {
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchObjectLabels(), unknownLabelPredictionThreshold="${unknownLabelPredictionThreshold}"`);
}
// -----------------------------------------------------------------------
// ---- NOTE-FOR-DEBUGGING-ONLY ---- Utility.resetToPrintDetailedDebuggingLogToConsole(true);
// ---- NOTE-FOR-FUTURE ---- const hasUnknownLabelInMapAlready: boolean = Utility.UnknownLabel in labelArrayAndMap.stringMap;
// -----------------------------------------------------------------------
const utterances: string[] = utteranceLabelsPairArray.map((x: StructTextLabelStrings) => x.text);
const predictionStructureWithScoreLabelStringArray: PredictionStructureWithScoreLabelString[] = [];
if (UtilityDispatcher.isEmptyStringArray(utterances)) {
return predictionStructureWithScoreLabelStringArray;
}
const scoreResultsBatch: any = LabelResolver.scoreBatch(utterances, LabelType.Intent);
if (utterances.length !== scoreResultsBatch.length) {
UtilityDispatcher.debuggingNamedThrow2(
'utterances.length !== scoreResultsBatch.length',
utterances.length,
scoreResultsBatch.length,
'utterances.length',
'scoreResultsBatch.length');
}
UtilityDispatcher.debuggingNamedLog1('UtilityLabelResolver.scoreBatchStringLabels()', utterances.length, 'utterances.length');
UtilityDispatcher.debuggingNamedLog1('UtilityLabelResolver.scoreBatchStringLabels()', scoreResultsBatch.length, 'scoreResultsBatch.length');
if (UtilityDispatcher.toPrintDetailedDebuggingLogToConsole) {
UtilityDispatcher.debuggingNamedLog1('UtilityLabelResolver.scoreBatchStringLabels()', utterances, 'utterances');
}
if (UtilityDispatcher.toPrintDetailedDebuggingLogToConsole) {
UtilityDispatcher.debuggingNamedLog1('UtilityLabelResolver.scoreBatchStringLabels()', scoreResultsBatch, 'scoreResultsBatch');
}
// -----------------------------------------------------------------------
for (let index: number = 0; index < scoreResultsBatch.length; index++) {
// ---------------------------------------------------------------------
const utteranceLabels: StructTextLabelStrings = utteranceLabelsPairArray[index];
const scoreResults: any = scoreResultsBatch[index];
if (UtilityDispatcher.toPrintDetailedDebuggingLogToConsole) {
UtilityDispatcher.debuggingNamedLog1('UtilityLabelResolver.scoreBatchStringLabels()', index, 'index');
UtilityDispatcher.debuggingNamedLog1('UtilityLabelResolver.scoreBatchStringLabels()', utteranceLabels, 'utteranceLabels');
UtilityDispatcher.debuggingNamedLog1('UtilityLabelResolver.scoreBatchStringLabels()', scoreResults.length, 'scoreResults.length');
UtilityDispatcher.debuggingNamedLog1('UtilityLabelResolver.scoreBatchStringLabels()', scoreResults, 'scoreResults');
}
// ---------------------------------------------------------------------
if (utteranceLabels) {
// -------------------------------------------------------------------
const utterance: string = utteranceLabels.text;
if (Utility.isEmptyString(utterance)) {
Utility.debuggingThrow('UtilityLabelResolver.scoreBatchStringLabels() failed to produce a prediction for an empty utterance');
}
if (Utility.toPrintDetailedDebuggingLogToConsole) {
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchStringLabels(), utterance=${utterance}`);
}
// -------------------------------------------------------------------
const labels: string[] =
utteranceLabels.labels;
const labelsIndexes: number[] =
labels.map((x: string) => Utility.carefullyAccessStringMap(
labelArrayAndMap.stringMap,
x));
const labelsStringArray: string[] =
labels.map((label: string) => Utility.outputString(
label,
UtilityLabelResolver.toObfuscateLabelTextInReportUtilityLabelResolver));
// -------------------------------------------------------------------
const labelsConcatenated: string = Utility.concatenateDataArrayToDelimitedString(
labelsStringArray);
const labelsConcatenatedToHtmlTable: string = Utility.concatenateDataArrayToHtmlTable(
'', // ---- 'Label',
labelsStringArray);
/** ---- NOTE-FOR-REFERENCE-ALTERNATIVE-LOGIC ----
* if (Utility.toPrintDetailedDebuggingLogToConsole) {
* Utility.debuggingLog(`UtilityLabelResolver.scoreBatchStringLabels(), before calling LabelResolver.score(), utterance=${utterance}`);
* }
* // Utility.debuggingLog(`UtilityLabelResolver.scoreBatchStringLabels(), before calling LabelResolver.score(), utterance=${utterance}`);
* const scoreResults: any = LabelResolver.score(utterance, LabelType.Intent);
* // Utility.debuggingLog(`UtilityLabelResolver.scoreBatchStringLabels(), after calling LabelResolver.LabelResolver.score(), utterance=${utterance}`);
*/
if (!scoreResults) {
Utility.debuggingThrow(`UtilityLabelResolver.scoreBatchStringLabels() failed to produce a prediction for utterance "${utterance}"`);
}
if (Utility.toPrintDetailedDebuggingLogToConsole) {
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchStringLabels(), scoreResults=${Utility.jsonStringify(scoreResults)}`);
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchStringLabels(), scoreResults.length=${scoreResults.length}`);
}
const scoreResultArray: Result[] = Utility.scoreResultsToArray(scoreResults, labelArrayAndMap.stringMap);
if (Utility.toPrintDetailedDebuggingLogToConsole) {
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchStringLabels(), scoreResultArray=${Utility.jsonStringify(scoreResultArray)}`);
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchStringLabels(), scoreResultArray.length=${scoreResultArray.length}`);
}
// -------------------------------------------------------------------
const scoreArray: number[] = scoreResultArray.map(
(x: Result) => x.score);
const argMax: { 'indexesMax': number[]; 'max': number } =
((multiLabelPredictionThreshold > 0) ?
Utility.getIndexesOnMaxOrEntriesOverThreshold(scoreArray, multiLabelPredictionThreshold) :
Utility.getIndexesOnMaxEntries(scoreArray));
if (Utility.toPrintDetailedDebuggingLogToConsole) {
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchStringLabels(), argMax.indexesMax=${Utility.jsonStringify(argMax.indexesMax)}`);
}
const labelsPredictedScore: number =
argMax.max;
let labelsPredictedIndexes: number[] =
argMax.indexesMax;
let labelsPredicted: string[] =
labelsPredictedIndexes.map((x: number) => scoreResultArray[x].label.name);
const labelsPredictedStringArray: string[] =
labelsPredicted.map((label: string) => Utility.outputString(
label,
UtilityLabelResolver.toObfuscateLabelTextInReportUtilityLabelResolver));
let labelsPredictedClosestText: string[] =
labelsPredictedIndexes.map((x: number) => scoreResultArray[x].closesttext);
const unknownPrediction: boolean = labelsPredictedScore < unknownLabelPredictionThreshold;
if (unknownPrediction) {
labelsPredictedIndexes = [Utility.carefullyAccessStringMap(labelArrayAndMap.stringMap, Utility.UnknownLabel)];
labelsPredicted = [Utility.UnknownLabel];
labelsPredictedClosestText = [];
}
// -------------------------------------------------------------------
if (Utility.toPrintDetailedDebuggingLogToConsole) {
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchStringLabels(), labelsPredictedIndexes=${Utility.jsonStringify(labelsPredictedIndexes)}`);
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchStringLabels(), labelsPredicted=${Utility.jsonStringify(labelsPredicted)}`);
}
// -------------------------------------------------------------------
const labelsPredictedConcatenated: string = Utility.concatenateDataArrayToDelimitedString(
labelsPredictedStringArray);
const labelsPredictedConcatenatedToHtmlTable: string = Utility.concatenateDataArrayToHtmlTable(
'', // ---- 'Label',
labelsPredictedStringArray);
if (Utility.toPrintDetailedDebuggingLogToConsole) {
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchStringLabels(), labelsPredictedConcatenated="${Utility.jsonStringify(labelsPredictedConcatenated)}"`);
}
const labelsPredictedEvaluation: number = Utility.evaluateMultiLabelSubsetPrediction(labels, labelsPredicted);
if (Utility.toPrintDetailedDebuggingLogToConsole) {
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchStringLabels(), labelsPredictedEvaluation="${labelsPredictedEvaluation}"`);
}
const predictedScoreStructureHtmlTable: string = Utility.selectedScoreResultsToHtmlTable(
scoreResultArray,
labelsPredictedIndexes,
unknownLabelPredictionThreshold,
UtilityLabelResolver.toObfuscateLabelTextInReportUtilityLabelResolver,
'',
['Label', 'Score', 'Closest Example'],
['30%', '10%', '60%']);
if (Utility.toPrintDetailedDebuggingLogToConsole) {
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchStringLabels(), predictedScoreStructureHtmlTable="${predictedScoreStructureHtmlTable}"`);
}
const labelsScoreStructureHtmlTable: string = Utility.selectedScoreResultsToHtmlTable(
scoreResultArray,
labelsIndexes,
unknownLabelPredictionThreshold,
UtilityLabelResolver.toObfuscateLabelTextInReportUtilityLabelResolver,
'',
['Label', 'Score', 'Closest Example'],
['30%', '10%', '60%']);
if (Utility.toPrintDetailedDebuggingLogToConsole) {
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchStringLabels(), labelsScoreStructureHtmlTable="${labelsScoreStructureHtmlTable}"`);
}
predictionStructureWithScoreLabelStringArray.push(new PredictionStructureWithScoreLabelString(
utterance,
labelsPredictedEvaluation,
labels,
labelsConcatenated,
labelsConcatenatedToHtmlTable,
labelsIndexes,
labelsPredicted,
labelsPredictedConcatenated,
labelsPredictedConcatenatedToHtmlTable,
labelsPredictedIndexes,
labelsPredictedScore,
labelsPredictedClosestText,
scoreResultArray,
scoreArray,
predictedScoreStructureHtmlTable,
labelsScoreStructureHtmlTable));
if (Utility.toPrintDetailedDebuggingLogToConsole) {
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchStringLabels(), finished scoring for utterance "${utterance}"`);
}
// ---- NOTE ---- debugging ouput.
if (Utility.toPrintDetailedDebuggingLogToConsole) {
for (const result of scoreResults) {
// eslint-disable-next-line max-depth
if (result) {
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchStringLabels(), result=${Utility.jsonStringify(result)}`);
const closesttext: string = result.closesttext;
const score: number = result.score;
const label: any = result.label;
const labelname: string = label.name;
const labeltype: LabelType = label.labeltype;
const span: any = label.span;
const offset: number = span.offset;
const length: number = span.length;
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchStringLabels(), closesttext=${closesttext}`);
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchStringLabels(), score=${score}`);
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchStringLabels(), label=${Utility.jsonStringify(label)}`);
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchStringLabels(), Object.keys(label)=${Object.keys(label)}`);
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchStringLabels(), label.name=${labelname}`);
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchStringLabels(), label.labeltype=${labeltype}`);
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchStringLabels(), span=${Utility.jsonStringify(span)}`);
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchStringLabels(), Object.keys(span)=${Object.keys(span)}`);
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchStringLabels(), label.span.offset=${offset}`);
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchStringLabels(), label.span.length=${length}`);
}
}
}
if ((predictionStructureWithScoreLabelStringArray.length % Utility.NumberOfInstancesPerProgressDisplayBatchForIntent) === 0) {
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchStringLabels(): Added predictionStructureWithScoreLabelStringArray.length=${predictionStructureWithScoreLabelStringArray.length}`);
}
// -------------------------------------------------------------------
}
// ---------------------------------------------------------------------
}
// -----------------------------------------------------------------------
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchStringLabels(): Total added predictionStructureWithScoreLabelStringArray.length=${predictionStructureWithScoreLabelStringArray.length}`);
// Utility.debuggingLog('UtilityLabelResolver.scoreBatchStringLabels(), leaving');
// -----------------------------------------------------------------------
return predictionStructureWithScoreLabelStringArray;
}