in packages/orchestratorlib/src/utilitylabelresolver.ts [773:1006]
public static scoreObjectLabels(
utteranceLabelsPairArray: StructTextLabelObjects[],
labelArrayAndMap: ILabelArrayAndMap,
multiLabelPredictionThreshold: number,
unknownLabelPredictionThreshold: number): PredictionStructureWithScoreLabelObject[] {
// -----------------------------------------------------------------------
// Utility.debuggingLog('UtilityLabelResolver.scoreObjectLabels(), entering');
// -----------------------------------------------------------------------
// ---- NOTE-FOR-DEBUGGING-ONLY ---- Utility.toPrintDetailedDebuggingLogToConsole = true;
// ---- NOTE-FOR-FUTURE ---- const hasUnknownLabelInMapAlready: boolean = Utility.UnknownLabel in labelArrayAndMap.stringMap;
const predictionStructureWithScoreLabelObjectArray: PredictionStructureWithScoreLabelObject[] = [];
if (Utility.toPrintDetailedDebuggingLogToConsole) {
Utility.debuggingLog(`UtilityLabelResolver.scoreObjectLabels(), unknownLabelPredictionThreshold="${unknownLabelPredictionThreshold}"`);
}
for (const utteranceLabels of utteranceLabelsPairArray) {
// ---------------------------------------------------------------------
if (utteranceLabels) {
const utterance: string = utteranceLabels.text;
if (Utility.isEmptyString(utterance)) {
Utility.debuggingThrow('UtilityLabelResolver.scoreObjectLabels() failed to produce a prediction for an empty utterance');
}
if (Utility.toPrintDetailedDebuggingLogToConsole) {
Utility.debuggingLog(`UtilityLabelResolver.scoreObjectLabels(), utterance=${utterance}`);
}
// -------------------------------------------------------------------
const labels: Label[] =
utteranceLabels.labels;
const labelsIndexes: number[] =
labels.map((x: Label) => Utility.carefullyAccessStringMap(
labelArrayAndMap.stringMap,
x.name));
const labelsStringArray: string[] =
labels.map((label: Label) => Utility.outputLabelString(
label,
UtilityLabelResolver.toObfuscateLabelTextInReportUtilityLabelResolver));
// -------------------------------------------------------------------
const labelsConcatenated: string = Utility.concatenateDataArrayToDelimitedString(
labelsStringArray);
if (Utility.toPrintDetailedDebuggingLogToConsole) {
Utility.debuggingLog(`UtilityLabelResolver.scoreObjectLabels(), labelsConcatenated=${labelsConcatenated}`);
}
const labelsConcatenatedToHtmlTable: string = Utility.concatenateDataArrayToHtmlTable(
'', // ---- 'Label',
labelsStringArray);
if (Utility.toPrintDetailedDebuggingLogToConsole) {
Utility.debuggingLog(`UtilityLabelResolver.scoreObjectLabels(), labelsConcatenatedToHtmlTable=${labelsConcatenatedToHtmlTable}`);
}
if (Utility.toPrintDetailedDebuggingLogToConsole) {
Utility.debuggingLog(`UtilityLabelResolver.scoreObjectLabels(), before calling LabelResolver.score(), utterance=${utterance}`);
}
// Utility.debuggingLog(`UtilityLabelResolver.scoreObjectLabels(), before calling LabelResolver.score(), utterance=${utterance}`);
const scoreResults: any = LabelResolver.score(utterance, LabelType.Entity);
// Utility.debuggingLog(`UtilityLabelResolver.scoreObjectLabels(), after calling LabelResolver.LabelResolver.score(), utterance=${utterance}`);
if (!scoreResults) {
Utility.debuggingThrow(`UtilityLabelResolver.scoreObjectLabels() failed to produce a prediction for utterance "${utterance}"`);
}
if (Utility.toPrintDetailedDebuggingLogToConsole) {
Utility.debuggingLog(`UtilityLabelResolver.scoreObjectLabels(), scoreResults=${Utility.jsonStringify(scoreResults)}`);
Utility.debuggingLog(`UtilityLabelResolver.scoreObjectLabels(), scoreResults.length=${scoreResults.length}`);
}
// Utility.debuggingLog(`UtilityLabelResolver.scoreObjectLabels(), before calling Utility.scoreResultsToArray(), utterance=${utterance}`);
const scoreResultArray: Result[] = Utility.scoreResultsToArray(scoreResults, labelArrayAndMap.stringMap);
// Utility.debuggingLog(`UtilityLabelResolver.scoreObjectLabels(), after calling Utility.scoreResultsToArray(), utterance=${utterance}`);
if (Utility.toPrintDetailedDebuggingLogToConsole) {
Utility.debuggingLog(`UtilityLabelResolver.scoreObjectLabels(), scoreResultArray.length=${scoreResultArray.length}`);
}
if (Utility.toPrintDetailedDebuggingLogToConsole) {
Utility.debuggingLog(`UtilityLabelResolver.scoreObjectLabels(), scoreResultArray)=${Utility.jsonStringify(scoreResultArray)}`);
Utility.debuggingLog(`UtilityLabelResolver.scoreObjectLabels(), scoreResultArray.length=${scoreResultArray.length}`);
}
// -------------------------------------------------------------------
const scoreResultArrayFiltered: Result[] =
scoreResultArray.filter((x: Result) => (x !== undefined));
// -------------------------------------------------------------------
const scoreArrayFiltered: number[] = scoreResultArrayFiltered.map(
(x: Result) => ((x === undefined) ? 0 : x.score));
let argMax: { 'indexesMax': number[]; 'max': number } =
{indexesMax: [], max: 0};
if (!Utility.isEmptyNumberArray(scoreArrayFiltered)) {
argMax =
((multiLabelPredictionThreshold > 0) ?
Utility.getIndexesOnMaxOrEntriesOverThreshold(scoreArrayFiltered, multiLabelPredictionThreshold) :
Utility.getIndexesOnMaxEntries(scoreArrayFiltered));
}
if (Utility.toPrintDetailedDebuggingLogToConsole) {
Utility.debuggingLog(`UtilityLabelResolver.scoreObjectLabels(), argMax.indexesMax=${Utility.jsonStringify(argMax.indexesMax)}`);
}
// -------------------------------------------------------------------
const labelsPredictedScoreMax: number =
argMax.max;
const labelsPredictedIndexesMax: number[] =
argMax.indexesMax;
/** ---- NOTE-FOR-REFERENCE-all-entity-prediction-is-included-no-need-for-ArgMax-and-UNKNOWN-threshold ----
* let labelsPredictedMax: Label[] =
* labelsPredictedIndexesMax.map((x: number) => scoreResultArray[x].label);
*/
// -------------------------------------------------------------------
const labelsPredicted: Label[] =
scoreResultArrayFiltered.map((x: Result) => x.label);
const labelsPredictedIndexes: number[] =
labelsPredicted.map((x: Label) => Utility.carefullyAccessStringMap(
labelArrayAndMap.stringMap,
x.name));
const labelsPredictedStringArray: string[] =
labelsPredicted.map((label: Label) => Utility.outputLabelString(
label,
UtilityLabelResolver.toObfuscateLabelTextInReportUtilityLabelResolver));
// -------------------------------------------------------------------
const labelsPredictedClosestText: string[] =
labelsPredictedIndexesMax.map((x: number) => scoreResultArrayFiltered[x].closesttext);
/** ---- NOTE-FOR-REFERENCE-all-entity-prediction-is-included-no-need-for-ArgMax-and-UNKNOWN-threshold ----
* const unknownPrediction: boolean = labelsPredictedScoreMax < unknownLabelPredictionThreshold;
* if (unknownPrediction) {
* labelsPredictedIndexesMax = [Utility.carefullyAccessStringMap(labelArrayAndMap.stringMap, Utility.UnknownLabel)];
* labelsPredictedMax = [Label.newLabel(LabelType.Entity, Utility.UnknownLabel, 0, 0)];
* labelsPredictedClosestText = [];
* }
*/
// -------------------------------------------------------------------
if (Utility.toPrintDetailedDebuggingLogToConsole) {
Utility.debuggingLog(`UtilityLabelResolver.scoreObjectLabels(), labelsPredictedIndexes=${Utility.jsonStringify(labelsPredictedIndexes)}`);
Utility.debuggingLog(`UtilityLabelResolver.scoreObjectLabels(), labelsPredicted=${Utility.jsonStringify(labelsPredicted)}`);
}
// -------------------------------------------------------------------
const labelsPredictedConcatenated: string = Utility.concatenateDataArrayToDelimitedString(
labelsPredictedStringArray);
if (Utility.toPrintDetailedDebuggingLogToConsole) {
Utility.debuggingLog(`UtilityLabelResolver.scoreObjectLabels(), labelsPredictedConcatenated=${labelsPredictedConcatenated}`);
}
const labelsPredictedConcatenatedToHtmlTable: string = Utility.concatenateDataArrayToHtmlTable(
'', // ---- 'Label',
labelsPredictedStringArray);
if (Utility.toPrintDetailedDebuggingLogToConsole) {
Utility.debuggingLog(`UtilityLabelResolver.scoreObjectLabels(), labelsPredictedConcatenatedToHtmlTable=${labelsPredictedConcatenatedToHtmlTable}`);
}
if (Utility.toPrintDetailedDebuggingLogToConsole) {
Utility.debuggingLog(`UtilityLabelResolver.scoreObjectLabels(), labelsPredictedConcatenated="${Utility.jsonStringify(labelsPredictedConcatenated)}"`);
}
let labelsPredictedEvaluation: number = Utility.evaluateMultiLabelObjectExactPrediction(labels, labelsPredicted);
if (labelsPredictedEvaluation === PredictionType.TrueNegative) {
labelsPredictedEvaluation = PredictionType.FalseNegative; // ---- NOTE ----override the default logic, for entity, there is no true negative.
}
if (Utility.toPrintDetailedDebuggingLogToConsole) {
Utility.debuggingLog(`UtilityLabelResolver.scoreObjectLabels(), labelsPredictedEvaluation="${labelsPredictedEvaluation}"`);
}
/** ---- NOTE-MAY-NOT-HAVE-SCORE-FOR-ALL-LABELS ----
* const predictedScoreStructureHtmlTable: string =
* labelsPredictedConcatenatedToHtmlTable;
*/
const predictedScoreStructureHtmlTable: string = Utility.selectedScoreResultsToHtmlTable(
scoreResultArrayFiltered,
[...(new Array(scoreResultArrayFiltered.length)).keys()], // ---- NOTE-MAY-NOT-HAVE-SCORE-FOR-ALL-LABELS ---- labelsPredictedIndexes,
unknownLabelPredictionThreshold,
UtilityLabelResolver.toObfuscateLabelTextInReportUtilityLabelResolver,
'',
['Label', 'Score', 'Closest Example'],
['30%', '10%', '60%']);
if (Utility.toPrintDetailedDebuggingLogToConsole) {
Utility.debuggingLog(`UtilityLabelResolver.scoreObjectLabels(), predictedScoreStructureHtmlTable="${predictedScoreStructureHtmlTable}"`);
}
// ---- NOTE-MAY-NOT-HAVE-SCORE-FOR-ALL-LABELS ----
const labelsScoreStructureHtmlTable: string =
labelsConcatenatedToHtmlTable;
/** ---- NOTE-MAY-NOT-HAVE-SCORE-FOR-ALL-LABELS ----
* const labelsScoreStructureHtmlTable: string = Utility.selectedScoreResultsToHtmlTable(
* scoreResultArrayFiltered,
* [...(new Array(scoreResultArrayFiltered.length)).keys()], // ---- NOTE-MAY-NOT-HAVE-SCORE-FOR-ALL-LABELS ---- labelsIndexes,
* unknownLabelPredictionThreshold,
* UtilityLabelResolver.toObfuscateLabelTextInReportUtilityLabelResolver,
* '',
* ['Label', 'Score', 'Closest Example'],
* ['30%', '10%', '60%']);
* if (Utility.toPrintDetailedDebuggingLogToConsole) {
* Utility.debuggingLog(`UtilityLabelResolver.scoreObjectLabels(), labelsScoreStructureHtmlTable="${labelsScoreStructureHtmlTable}"`);
* }
*/
predictionStructureWithScoreLabelObjectArray.push(new PredictionStructureWithScoreLabelObject(
utterance,
labelsPredictedEvaluation,
labels,
labelsConcatenated,
labelsConcatenatedToHtmlTable,
labelsIndexes,
labelsPredicted,
labelsPredictedConcatenated,
labelsPredictedConcatenatedToHtmlTable,
labelsPredictedIndexes,
labelsPredictedScoreMax,
labelsPredictedClosestText,
scoreResultArrayFiltered,
scoreArrayFiltered,
predictedScoreStructureHtmlTable,
labelsScoreStructureHtmlTable));
if (Utility.toPrintDetailedDebuggingLogToConsole) {
Utility.debuggingLog(`UtilityLabelResolver.scoreObjectLabels(), finished scoring for utterance "${utterance}"`);
}
// ---- NOTE ---- debugging ouput.
if (Utility.toPrintDetailedDebuggingLogToConsole) {
for (const result of scoreResults) {
// eslint-disable-next-line max-depth
if (result) {
Utility.debuggingLog(`UtilityLabelResolver.scoreObjectLabels(), result=${Utility.jsonStringify(result)}`);
const closesttext: string = result.closesttext;
const score: number = result.score;
const label: any = result.label;
const labelname: string = label.name;
const labeltype: LabelType = label.labeltype;
const span: any = label.span;
const offset: number = span.offset;
const length: number = span.length;
Utility.debuggingLog(`UtilityLabelResolver.scoreObjectLabels(), closesttext=${closesttext}`);
Utility.debuggingLog(`UtilityLabelResolver.scoreObjectLabels(), score=${score}`);
Utility.debuggingLog(`UtilityLabelResolver.scoreObjectLabels(), label=${Utility.jsonStringify(label)}`);
Utility.debuggingLog(`UtilityLabelResolver.scoreObjectLabels(), Object.keys(label)=${Object.keys(label)}`);
Utility.debuggingLog(`UtilityLabelResolver.scoreObjectLabels(), label.name=${labelname}`);
Utility.debuggingLog(`UtilityLabelResolver.scoreObjectLabels(), label.labeltype=${labeltype}`);
Utility.debuggingLog(`UtilityLabelResolver.scoreObjectLabels(), span=${Utility.jsonStringify(span)}`);
Utility.debuggingLog(`UtilityLabelResolver.scoreObjectLabels(), Object.keys(span)=${Object.keys(span)}`);
Utility.debuggingLog(`UtilityLabelResolver.scoreObjectLabels(), label.span.offset=${offset}`);
Utility.debuggingLog(`UtilityLabelResolver.scoreObjectLabels(), label.span.length=${length}`);
}
}
}
if ((predictionStructureWithScoreLabelObjectArray.length % Utility.NumberOfInstancesPerProgressDisplayBatchForEntity) === 0) {
Utility.debuggingLog(`UtilityLabelResolver.scoreObjectLabels(): Added predictionStructureWithScoreLabelObjectArray.length=${predictionStructureWithScoreLabelObjectArray.length}`);
}
}
// ---------------------------------------------------------------------
}
Utility.debuggingLog(`UtilityLabelResolver.scoreObjectLabels(): Total added predictionStructureWithScoreLabelObjectArray.length=${predictionStructureWithScoreLabelObjectArray.length}`);
// Utility.debuggingLog('UtilityLabelResolver.scoreObjectLabels(), leaving');
// -----------------------------------------------------------------------
return predictionStructureWithScoreLabelObjectArray;
}