in packages/orchestratorlib/src/utilitylabelresolver.ts [312:585]
public static scoreBatchObjectLabels(
utteranceLabelsPairArray: StructTextLabelObjects[],
labelArrayAndMap: ILabelArrayAndMap,
multiLabelPredictionThreshold: number,
unknownLabelPredictionThreshold: number): PredictionStructureWithScoreLabelObject[] {
// -----------------------------------------------------------------------
if (Utility.toPrintDetailedDebuggingLogToConsole) {
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchObjectLabels(), multiLabelPredictionThreshold="${multiLabelPredictionThreshold}"`);
}
if (Utility.toPrintDetailedDebuggingLogToConsole) {
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchObjectLabels(), unknownLabelPredictionThreshold="${unknownLabelPredictionThreshold}"`);
}
Utility.debuggingLog('UtilityLabelResolver.scoreBatchObjectLabels(), entering');
// -----------------------------------------------------------------------
// ---- NOTE-FOR-DEBUGGING-ONLY ---- Utility.resetToPrintDetailedDebuggingLogToConsole(true);
// ---- NOTE-FOR-FUTURE ---- const hasUnknownLabelInMapAlready: boolean = Utility.UnknownLabel in labelArrayAndMap.stringMap;
// -----------------------------------------------------------------------
const utterances: string[] = utteranceLabelsPairArray.map((x: StructTextLabelObjects) => x.text);
const predictionStructureWithScoreLabelObjectArray: PredictionStructureWithScoreLabelObject[] = [];
if (UtilityDispatcher.isEmptyStringArray(utterances)) {
return predictionStructureWithScoreLabelObjectArray;
}
const scoreResultsBatch: any = LabelResolver.scoreBatch(utterances, LabelType.Entity);
if (utterances.length !== scoreResultsBatch.length) {
UtilityDispatcher.debuggingNamedThrow2(
'utterances.length !== scoreResultsBatch.length',
utterances.length,
scoreResultsBatch.length,
'utterances.length',
'scoreResultsBatch.length');
}
UtilityDispatcher.debuggingNamedLog1('UtilityLabelResolver.scoreBatchObjectLabels()', utterances.length, 'utterances.length');
UtilityDispatcher.debuggingNamedLog1('UtilityLabelResolver.scoreBatchObjectLabels()', scoreResultsBatch.length, 'scoreResultsBatch.length');
if (UtilityDispatcher.toPrintDetailedDebuggingLogToConsole) {
UtilityDispatcher.debuggingNamedLog1('UtilityLabelResolver.scoreBatchObjectLabels()', utterances, 'utterances');
}
if (UtilityDispatcher.toPrintDetailedDebuggingLogToConsole) {
UtilityDispatcher.debuggingNamedLog1('UtilityLabelResolver.scoreBatchObjectLabels()', scoreResultsBatch, 'scoreResultsBatch');
}
// -----------------------------------------------------------------------
for (let index: number = 0; index < scoreResultsBatch.length; index++) {
// ---------------------------------------------------------------------
const utteranceLabels: StructTextLabelObjects = utteranceLabelsPairArray[index];
const scoreResults: any = scoreResultsBatch[index];
if (UtilityDispatcher.toPrintDetailedDebuggingLogToConsole) {
UtilityDispatcher.debuggingNamedLog1('UtilityLabelResolver.scoreBatchObjectLabels()', index, 'index');
UtilityDispatcher.debuggingNamedLog1('UtilityLabelResolver.scoreBatchObjectLabels()', utteranceLabels, 'utteranceLabels');
UtilityDispatcher.debuggingNamedLog1('UtilityLabelResolver.scoreBatchObjectLabels()', scoreResults.length, 'scoreResults.length');
UtilityDispatcher.debuggingNamedLog1('UtilityLabelResolver.scoreBatchObjectLabels()', scoreResults, 'scoreResults');
}
// ---------------------------------------------------------------------
if (utteranceLabels) {
// -------------------------------------------------------------------
const utterance: string = utteranceLabels.text;
if (Utility.isEmptyString(utterance)) {
Utility.debuggingThrow('UtilityLabelResolver.scoreBatchObjectLabels() failed to produce a prediction for an empty utterance');
}
if (Utility.toPrintDetailedDebuggingLogToConsole) {
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchObjectLabels(), utterance=${utterance}`);
}
// -------------------------------------------------------------------
const labels: Label[] =
utteranceLabels.labels;
const labelsIndexes: number[] =
labels.map((x: Label) => Utility.carefullyAccessStringMap(
labelArrayAndMap.stringMap,
x.name));
const labelsStringArray: string[] =
labels.map((label: Label) => Utility.outputLabelString(
label,
UtilityLabelResolver.toObfuscateLabelTextInReportUtilityLabelResolver));
// -------------------------------------------------------------------
const labelsConcatenated: string = Utility.concatenateDataArrayToDelimitedString(
labelsStringArray);
if (Utility.toPrintDetailedDebuggingLogToConsole) {
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchObjectLabels(), labelsConcatenated=${labelsConcatenated}`);
}
const labelsConcatenatedToHtmlTable: string = Utility.concatenateDataArrayToHtmlTable(
'', // ---- 'Label',
labelsStringArray);
if (Utility.toPrintDetailedDebuggingLogToConsole) {
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchObjectLabels(), labelsConcatenatedToHtmlTable=${labelsConcatenatedToHtmlTable}`);
}
/** ---- NOTE-FOR-REFERENCE-ALTERNATIVE-LOGIC ----
* if (Utility.toPrintDetailedDebuggingLogToConsole) {
* Utility.debuggingLog(`UtilityLabelResolver.scoreBatchObjectLabels(), before calling LabelResolver.score(), utterance=${utterance}`);
* }
* // Utility.debuggingLog(`UtilityLabelResolver.scoreBatchObjectLabels(), before calling LabelResolver.score(), utterance=${utterance}`);
* const scoreResults: any = LabelResolver.score(utterance, LabelType.Entity);
* // Utility.debuggingLog(`UtilityLabelResolver.scoreBatchObjectLabels(), after calling LabelResolver.LabelResolver.score(), utterance=${utterance}`);
*/
if (!scoreResults) {
Utility.debuggingThrow(`UtilityLabelResolver.scoreBatchObjectLabels() failed to produce a prediction for utterance "${utterance}"`);
}
if (Utility.toPrintDetailedDebuggingLogToConsole) {
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchObjectLabels(), scoreResults=${Utility.jsonStringify(scoreResults)}`);
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchObjectLabels(), scoreResults.length=${scoreResults.length}`);
}
// Utility.debuggingLog(`UtilityLabelResolver.scoreBatchObjectLabels(), before calling Utility.scoreResultsToArray(), utterance=${utterance}`);
const scoreResultArray: Result[] = Utility.scoreResultsToArray(scoreResults, labelArrayAndMap.stringMap);
// Utility.debuggingLog(`UtilityLabelResolver.scoreBatchObjectLabels(), after calling Utility.scoreResultsToArray(), utterance=${utterance}`);
if (Utility.toPrintDetailedDebuggingLogToConsole) {
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchObjectLabels(), scoreResultArray.length=${scoreResultArray.length}`);
}
if (Utility.toPrintDetailedDebuggingLogToConsole) {
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchObjectLabels(), scoreResultArray)=${Utility.jsonStringify(scoreResultArray)}`);
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchObjectLabels(), scoreResultArray.length=${scoreResultArray.length}`);
}
// -------------------------------------------------------------------
const scoreResultArrayFiltered: Result[] =
scoreResultArray.filter((x: Result) => (x !== undefined));
// -------------------------------------------------------------------
const scoreArrayFiltered: number[] = scoreResultArrayFiltered.map(
(x: Result) => ((x === undefined) ? 0 : x.score));
let argMax: { 'indexesMax': number[]; 'max': number } =
{indexesMax: [], max: 0};
if (!Utility.isEmptyNumberArray(scoreArrayFiltered)) {
argMax =
((multiLabelPredictionThreshold > 0) ?
Utility.getIndexesOnMaxOrEntriesOverThreshold(scoreArrayFiltered, multiLabelPredictionThreshold) :
Utility.getIndexesOnMaxEntries(scoreArrayFiltered));
}
if (Utility.toPrintDetailedDebuggingLogToConsole) {
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchObjectLabels(), argMax.indexesMax=${Utility.jsonStringify(argMax.indexesMax)}`);
}
// -------------------------------------------------------------------
const labelsPredictedScoreMax: number =
argMax.max;
const labelsPredictedIndexesMax: number[] =
argMax.indexesMax;
/** ---- NOTE-FOR-REFERENCE-all-entity-prediction-is-included-no-need-for-ArgMax-and-UNKNOWN-threshold ----
* let labelsPredictedMax: Label[] =
* labelsPredictedIndexesMax.map((x: number) => scoreResultArray[x].label);
*/
// -------------------------------------------------------------------
const labelsPredicted: Label[] =
scoreResultArrayFiltered.map((x: Result) => x.label);
const labelsPredictedIndexes: number[] =
labelsPredicted.map((x: Label) => Utility.carefullyAccessStringMap(
labelArrayAndMap.stringMap,
x.name));
const labelsPredictedStringArray: string[] =
labelsPredicted.map((label: Label) => Utility.outputLabelString(
label,
UtilityLabelResolver.toObfuscateLabelTextInReportUtilityLabelResolver));
// -------------------------------------------------------------------
const labelsPredictedClosestText: string[] =
labelsPredictedIndexesMax.map((x: number) => scoreResultArrayFiltered[x].closesttext);
/** ---- NOTE-FOR-REFERENCE-all-entity-prediction-is-included-no-need-for-ArgMax-and-UNKNOWN-threshold ----
* const unknownPrediction: boolean = labelsPredictedScoreMax < unknownLabelPredictionThreshold;
* if (unknownPrediction) {
* labelsPredictedIndexesMax = [Utility.carefullyAccessStringMap(labelArrayAndMap.stringMap, Utility.UnknownLabel)];
* labelsPredictedMax = [Label.newLabel(LabelType.Entity, Utility.UnknownLabel, 0, 0)];
* labelsPredictedClosestText = [];
* }
*/
// -------------------------------------------------------------------
if (Utility.toPrintDetailedDebuggingLogToConsole) {
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchObjectLabels(), labelsPredictedIndexes=${Utility.jsonStringify(labelsPredictedIndexes)}`);
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchObjectLabels(), labelsPredicted=${Utility.jsonStringify(labelsPredicted)}`);
}
// -------------------------------------------------------------------
const labelsPredictedConcatenated: string = Utility.concatenateDataArrayToDelimitedString(
labelsPredictedStringArray);
if (Utility.toPrintDetailedDebuggingLogToConsole) {
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchObjectLabels(), labelsPredictedConcatenated=${labelsPredictedConcatenated}`);
}
const labelsPredictedConcatenatedToHtmlTable: string = Utility.concatenateDataArrayToHtmlTable(
'', // ---- 'Label',
labelsPredictedStringArray);
if (Utility.toPrintDetailedDebuggingLogToConsole) {
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchObjectLabels(), labelsPredictedConcatenatedToHtmlTable=${labelsPredictedConcatenatedToHtmlTable}`);
}
if (Utility.toPrintDetailedDebuggingLogToConsole) {
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchObjectLabels(), labelsPredictedConcatenated="${Utility.jsonStringify(labelsPredictedConcatenated)}"`);
}
let labelsPredictedEvaluation: number = Utility.evaluateMultiLabelObjectExactPrediction(labels, labelsPredicted);
if (labelsPredictedEvaluation === PredictionType.TrueNegative) {
labelsPredictedEvaluation = PredictionType.FalseNegative; // ---- NOTE ----override the default logic, for entity, there is no true negative.
}
if (Utility.toPrintDetailedDebuggingLogToConsole) {
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchObjectLabels(), labelsPredictedEvaluation="${labelsPredictedEvaluation}"`);
}
/** ---- NOTE-MAY-NOT-HAVE-SCORE-FOR-ALL-LABELS ----
* const predictedScoreStructureHtmlTable: string =
* labelsPredictedConcatenatedToHtmlTable;
*/
const predictedScoreStructureHtmlTable: string = Utility.selectedScoreResultsToHtmlTable(
scoreResultArrayFiltered,
[...(new Array(scoreResultArrayFiltered.length)).keys()], // ---- NOTE-MAY-NOT-HAVE-SCORE-FOR-ALL-LABELS ---- labelsPredictedIndexes,
unknownLabelPredictionThreshold,
UtilityLabelResolver.toObfuscateLabelTextInReportUtilityLabelResolver,
'',
['Label', 'Score', 'Closest Example'],
['30%', '10%', '60%']);
if (Utility.toPrintDetailedDebuggingLogToConsole) {
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchObjectLabels(), predictedScoreStructureHtmlTable="${predictedScoreStructureHtmlTable}"`);
}
// ---- NOTE-MAY-NOT-HAVE-SCORE-FOR-ALL-LABELS ----
const labelsScoreStructureHtmlTable: string =
labelsConcatenatedToHtmlTable;
/** ---- NOTE-MAY-NOT-HAVE-SCORE-FOR-ALL-LABELS ----
* const labelsScoreStructureHtmlTable: string = Utility.selectedScoreResultsToHtmlTable(
* scoreResultArrayFiltered,
* [...(new Array(scoreResultArrayFiltered.length)).keys()], // ---- NOTE-MAY-NOT-HAVE-SCORE-FOR-ALL-LABELS ---- labelsIndexes,
* unknownLabelPredictionThreshold,
* UtilityLabelResolver.toObfuscateLabelTextInReportUtilityLabelResolver,
* '',
* ['Label', 'Score', 'Closest Example'],
* ['30%', '10%', '60%']);
* if (Utility.toPrintDetailedDebuggingLogToConsole) {
* Utility.debuggingLog(`UtilityLabelResolver.scoreBatchObjectLabels(), labelsScoreStructureHtmlTable="${labelsScoreStructureHtmlTable}"`);
* }
*/
predictionStructureWithScoreLabelObjectArray.push(new PredictionStructureWithScoreLabelObject(
utterance,
labelsPredictedEvaluation,
labels,
labelsConcatenated,
labelsConcatenatedToHtmlTable,
labelsIndexes,
labelsPredicted,
labelsPredictedConcatenated,
labelsPredictedConcatenatedToHtmlTable,
labelsPredictedIndexes,
labelsPredictedScoreMax,
labelsPredictedClosestText,
scoreResultArrayFiltered,
scoreArrayFiltered,
predictedScoreStructureHtmlTable,
labelsScoreStructureHtmlTable));
if (Utility.toPrintDetailedDebuggingLogToConsole) {
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchObjectLabels(), finished scoring for utterance "${utterance}"`);
}
// ---- NOTE ---- debugging ouput.
if (Utility.toPrintDetailedDebuggingLogToConsole) {
for (const result of scoreResults) {
// eslint-disable-next-line max-depth
if (result) {
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchObjectLabels(), result=${Utility.jsonStringify(result)}`);
const closesttext: string = result.closesttext;
const score: number = result.score;
const label: any = result.label;
const labelname: string = label.name;
const labeltype: LabelType = label.labeltype;
const span: any = label.span;
const offset: number = span.offset;
const length: number = span.length;
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchObjectLabels(), closesttext=${closesttext}`);
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchObjectLabels(), score=${score}`);
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchObjectLabels(), label=${Utility.jsonStringify(label)}`);
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchObjectLabels(), Object.keys(label)=${Object.keys(label)}`);
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchObjectLabels(), label.name=${labelname}`);
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchObjectLabels(), label.labeltype=${labeltype}`);
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchObjectLabels(), span=${Utility.jsonStringify(span)}`);
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchObjectLabels(), Object.keys(span)=${Object.keys(span)}`);
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchObjectLabels(), label.span.offset=${offset}`);
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchObjectLabels(), label.span.length=${length}`);
}
}
}
if ((predictionStructureWithScoreLabelObjectArray.length % Utility.NumberOfInstancesPerProgressDisplayBatchForEntity) === 0) {
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchObjectLabels(): Added predictionStructureWithScoreLabelObjectArray.length=${predictionStructureWithScoreLabelObjectArray.length}`);
}
// -------------------------------------------------------------------
}
// ---------------------------------------------------------------------
}
// -----------------------------------------------------------------------
Utility.debuggingLog(`UtilityLabelResolver.scoreBatchObjectLabels(): Total added predictionStructureWithScoreLabelObjectArray.length=${predictionStructureWithScoreLabelObjectArray.length}`);
// Utility.debuggingLog('UtilityLabelResolver.scoreBatchObjectLabels(), leaving');
// -----------------------------------------------------------------------
return predictionStructureWithScoreLabelObjectArray;
}