in packages/lu/src/parser/lufile/parseFileContents.js [925:1251]
const parseAndHandleSimpleIntentSection = function (parsedContent, luResource, config) {
// handle intent
let intents = luResource.Sections.filter(s => s.SectionType === SectionType.SIMPLEINTENTSECTION);
let utteranceStringToObjectMap = new Map()
if (intents && intents.length > 0) {
let references = luResource.Sections.filter(s => s.SectionType === SectionType.REFERENCESECTION);
for (const intent of intents) {
let intentName = intent.Name;
if (InvalidCharsInIntentOrEntityName.some(x => intentName.includes(x))) {
let errorMsg = `Invalid intent line, intent name ${intentName} cannot contain any of the following characters: [<, >, *, %, &, :, \\, $]`;
throwDiagnosticError({
message: errorMsg,
line: intent.Range.Start.Line
});
}
// insert only if the intent is not already present.
addItemIfNotPresent(parsedContent.LUISJsonStructure, LUISObjNameEnum.INTENT, intentName);
for (const utteranceAndEntities of intent.UtteranceAndEntitiesMap) {
// add utterance
let utterance = utteranceAndEntities.utterance.trim();
// Fix for BF-CLI #122.
// Ensure only links are detected and passed on to be parsed.
if (helpers.isUtteranceLinkRef(utterance || '')) {
if (utterance.endsWith(']')) {
let index = utterance.lastIndexOf('[');
let referenceId = utterance.slice(index + 1, utterance.length - 1);
let reference = references.find(refer => refer.ReferenceId === referenceId)
if (!reference) {
let errorMsg = `Cannot find reference ${reference} when resolving utternace "${utteranceAndEntities.contextText}".`;
throwDiagnosticError({
message: errorMsg,
range: utteranceAndEntities.range
});
}
utterance = `${utterance.slice(0, index)}(${reference.Path})`
}
let parsedLinkUriInUtterance = helpers.parseLinkURISync(utterance);
// examine and add these to filestoparse list.
parsedContent.additionalFilesToParse.push(new fileToParse(parsedLinkUriInUtterance.fileName, false, parsedLinkUriInUtterance.path));
}
(utteranceAndEntities.entities || []).forEach(entity => {
let errors = []
if (InvalidCharsInIntentOrEntityName.some(x => entity.entity.includes(x))) {
let errorMsg = `Invalid utterance line, entity name ${entity.entity} in this utterance cannot contain any of the following characters: [<, >, *, %, &, :, \\, $]`;
let error = BuildDiagnostic({
message: errorMsg,
range: utteranceAndEntities.range
});
errors.push(error);
}
if (errors.length > 0) {
throw (new exception(retCode.errorCode.INVALID_LINE, errors.map(error => error.toString()).join('\n'), errors));
}
})
if (utteranceAndEntities.entities.length > 0) {
let entitiesFound = utteranceAndEntities.entities;
let havePatternAnyEntity = entitiesFound.find(item => item.type == LUISObjNameEnum.PATTERNANYENTITY);
if (havePatternAnyEntity !== undefined) {
if (!config.enablePattern) {
throwDiagnosticError({
message: 'Do not support Pattern. Please make sure enablePattern is set to true.',
range: utteranceAndEntities.range
});
}
utterance = handleAtForPattern(utterance, entitiesFound, parsedContent.LUISJsonStructure.flatListOfEntityAndRoles);
let mixedEntity = entitiesFound.filter(item => item.type != LUISObjNameEnum.PATTERNANYENTITY);
if (mixedEntity.length !== 0) {
let errorMsg = `Utterance "${utteranceAndEntities.contextText}" has mix of entites with labelled values and ones without. Please update utterance to either include labelled values for all entities or remove labelled values from all entities.`;
throwDiagnosticError({
message: errorMsg,
range: utteranceAndEntities.range
});
}
let prebuiltEntities = entitiesFound.filter(item => builtInTypes.consolidatedList.map(prebuiltEntity => prebuiltEntity.toLowerCase()).includes(item.entity.toLowerCase()));
prebuiltEntities.forEach(prebuiltEntity => {
if (parsedContent.LUISJsonStructure.prebuiltEntities.findIndex(e => e.name === prebuiltEntity.entity) < 0) {
let errorMsg = `Pattern "${utteranceAndEntities.contextText}" has prebuilt entity ${prebuiltEntity.entity}. Please define it explicitly with @ prebuilt ${prebuiltEntity.entity}.`;
throwDiagnosticError({
message: errorMsg,
range: utteranceAndEntities.range
});
}
})
let newPattern = new helperClass.pattern(utterance, intentName);
if (!utteranceStringToObjectMap.has(utterance)) {
parsedContent.LUISJsonStructure.patterns.push(newPattern);
utteranceStringToObjectMap.set(utterance, newPattern);
}
// add all entities to pattern.Any only if they do not have another type.
entitiesFound.forEach(entity => {
let simpleEntityInMaster = parsedContent.LUISJsonStructure.entities.find(item => item.name == entity.entity);
if (simpleEntityInMaster && entity.role) {
addItemOrRoleIfNotPresent(parsedContent.LUISJsonStructure, LUISObjNameEnum.ENTITIES, entity.entity, [entity.role.trim()]);
}
let compositeInMaster = parsedContent.LUISJsonStructure.composites.find(item => item.name == entity.entity);
if (compositeInMaster && entity.role) {
addItemOrRoleIfNotPresent(parsedContent.LUISJsonStructure, LUISObjNameEnum.COMPOSITES, entity.entity, [entity.role.trim()]);
}
let listEntityInMaster = parsedContent.LUISJsonStructure.closedLists.find(item => item.name == entity.entity);
if (listEntityInMaster && entity.role) {
addItemOrRoleIfNotPresent(parsedContent.LUISJsonStructure, LUISObjNameEnum.CLOSEDLISTS, entity.entity, [entity.role.trim()]);
}
let regexEntityInMaster = parsedContent.LUISJsonStructure.regex_entities.find(item => item.name == entity.entity);
if (regexEntityInMaster && entity.role) {
addItemOrRoleIfNotPresent(parsedContent.LUISJsonStructure, LUISObjNameEnum.REGEX, entity.entity, [entity.role.trim()]);
}
let prebuiltInMaster = parsedContent.LUISJsonStructure.prebuiltEntities.find(item => item.name == entity.entity);
if (prebuiltInMaster && entity.role) {
addItemOrRoleIfNotPresent(parsedContent.LUISJsonStructure, LUISObjNameEnum.PREBUILT, entity.entity, [entity.role.trim()]);
}
if (!simpleEntityInMaster &&
!compositeInMaster &&
!listEntityInMaster &&
!regexEntityInMaster &&
!prebuiltInMaster) {
if (entity.role && entity.role !== '') {
addItemOrRoleIfNotPresent(parsedContent.LUISJsonStructure, LUISObjNameEnum.PATTERNANYENTITY, entity.entity, [entity.role.trim()])
} else {
addItemIfNotPresent(parsedContent.LUISJsonStructure, LUISObjNameEnum.PATTERNANYENTITY, entity.entity);
}
}
});
} else {
entitiesFound.forEach(entity => {
// handle at prefix
entity = handleAtPrefix(entity, parsedContent.LUISJsonStructure.flatListOfEntityAndRoles);
// throw an error if phraselist entity is explicitly labelled in an utterance
let nonAllowedPhrseListEntityInUtterance = (parsedContent.LUISJsonStructure.model_features || []).find(item => item.name == entity.entity);
if (nonAllowedPhrseListEntityInUtterance !== undefined) {
// Fix for #1137
// Phrase list entity can have the same name as other entity types. Only throw if the phrase list has no other type definition and is labelled in an utterance.
let otherEntities = (parsedContent.LUISJsonStructure.entities || []).concat(
(parsedContent.LUISJsonStructure.prebuiltEntities || []),
(parsedContent.LUISJsonStructure.closedLists || []),
(parsedContent.LUISJsonStructure.regex_entities || []),
(parsedContent.LUISJsonStructure.model_features || []),
(parsedContent.LUISJsonStructure.composites || [])
);
if ((otherEntities || []).find(item => item.name == entity.entity) === undefined) {
let errorMsg = `Utterance "${utterance}" has invalid reference to Phrase List entity "${nonAllowedPhrseListEntityInUtterance.name}". Phrase list entities cannot be given an explicit labelled value.`;
throwDiagnosticError({
message: errorMsg,
range: utteranceAndEntities.range
});
}
}
// only add this entity if it has not already been defined as composite, list, prebuilt, regex
let compositeExists = (parsedContent.LUISJsonStructure.composites || []).find(item => item.name == entity.entity);
let listExists = (parsedContent.LUISJsonStructure.closedLists || []).find(item => item.name == entity.entity);
let prebuiltExists = (parsedContent.LUISJsonStructure.prebuiltEntities || []).find(item => item.name == entity.entity);
let regexExists = (parsedContent.LUISJsonStructure.regex_entities || []).find(item => item.name == entity.entity);
let patternAnyExists = (parsedContent.LUISJsonStructure.patternAnyEntities || []).find(item => item.name == entity.entity);
if (compositeExists === undefined && listExists === undefined && prebuiltExists === undefined && regexExists === undefined && patternAnyExists === undefined) {
if (!config.enableMLEntities) {
throwDiagnosticError({
message: 'Do not support ML entity. Please make sure enableMLEntities is set to true.',
range: utteranceAndEntities.range
});
}
if (entity.role && entity.role !== '') {
addItemOrRoleIfNotPresent(parsedContent.LUISJsonStructure, LUISObjNameEnum.ENTITIES, entity.entity, [entity.role.trim()]);
} else {
addItemIfNotPresent(parsedContent.LUISJsonStructure, LUISObjNameEnum.ENTITIES, entity.entity)
}
} else {
if (compositeExists !== undefined) {
if (entity.role) {
addItemOrRoleIfNotPresent(parsedContent.LUISJsonStructure, LUISObjNameEnum.COMPOSITES, entity.entity, [entity.role.trim()]);
}
} else if (listExists !== undefined) {
if (entity.role) {
addItemOrRoleIfNotPresent(parsedContent.LUISJsonStructure, LUISObjNameEnum.CLOSEDLISTS, entity.entity, [entity.role.trim()]);
} else {
if (!isChildEntity(entity, entitiesFound)) {
let errorMsg = `${entity.entity} has been defined as a LIST entity type. It cannot be explicitly included in a labelled utterance unless the label includes a role.`;
throwDiagnosticError({
message: errorMsg,
range: utteranceAndEntities.range
});
}
}
} else if (prebuiltExists !== undefined) {
if (entity.role) {
addItemOrRoleIfNotPresent(parsedContent.LUISJsonStructure, LUISObjNameEnum.PREBUILT, entity.entity, [entity.role.trim()]);
} else {
if (!isChildEntity(entity, entitiesFound)) {
let errorMsg = `${entity.entity} has been defined as a PREBUILT entity type. It cannot be explicitly included in a labelled utterance unless the label includes a role.`;
throwDiagnosticError({
message: errorMsg,
range: utteranceAndEntities.range
});
}
}
} else if (regexExists !== undefined) {
if (entity.role) {
addItemOrRoleIfNotPresent(parsedContent.LUISJsonStructure, LUISObjNameEnum.REGEX, entity.entity, [entity.role.trim()]);
} else {
if (!isChildEntity(entity, entitiesFound)) {
let errorMsg = `${entity.entity} has been defined as a Regex entity type. It cannot be explicitly included in a labelled utterance unless the label includes a role.`;
throwDiagnosticError({
message: errorMsg,
range: utteranceAndEntities.range
});
}
}
} else if (patternAnyExists !== undefined) {
if (entity.value != '') {
// Verify and add this as simple entity.
let roles = (entity.role && entity.role.trim() !== "") ? [entity.role.trim()] : [];
patternAnyExists.roles.forEach(role => roles.push(role));
addItemOrRoleIfNotPresent(parsedContent.LUISJsonStructure, LUISObjNameEnum.ENTITIES, entity.entity, roles);
let patternAnyIdx = -1;
(parsedContent.LUISJsonStructure.patternAnyEntities || []).find((item, idx) => {
if (item.name === entity.entity) {
patternAnyIdx = idx;
return true;
}
return false;
});
// delete pattern any entity
if (patternAnyIdx > -1) parsedContent.LUISJsonStructure.patternAnyEntities.splice(patternAnyIdx, 1);
} else if (entity.role) {
addItemOrRoleIfNotPresent(parsedContent.LUISJsonStructure, LUISObjNameEnum.PATTERNANYENTITY, entity.entity, [entity.role.trim()]);
}
}
}
});
// add utterance
let utteranceObject;
if (utteranceStringToObjectMap.has(utterance)) {
utteranceObject = utteranceStringToObjectMap.get(utterance);
} else {
utteranceObject = new helperClass.utterances(utterance, intentName, []);
parsedContent.LUISJsonStructure.utterances.push(utteranceObject);
utteranceStringToObjectMap.set(utterance, utteranceObject);
}
entitiesFound.forEach(item => {
if (item.startPos > item.endPos) {
let errorMsg = `No labelled value found for entity: "${item.entity}" in utterance: "${utteranceAndEntities.contextText}"`;
throwDiagnosticError({
message: errorMsg,
range: utteranceAndEntities.range,
errorCode: retCode.errorCode.MISSING_LABELLED_VALUE
});
}
let utteranceEntity = new helperClass.utteranceEntity(item.entity, item.startPos, item.endPos);
if (item.role && item.role !== '') {
utteranceEntity.role = item.role.trim();
}
// detect overlap and respect OnAmbiguousLabels
let priorLabelFound = utteranceObject.entities.find(item => {
if (item.entity === utteranceEntity.entity) {
if (utteranceEntity.role === undefined || utteranceEntity.role === '') {
if (item.role === undefined || item.role === '') {
// do the labels overlap?
if ((item.startPos >= utteranceEntity.startPos && item.endPos <= utteranceEntity.endPos) ||
(utteranceEntity.startPos >= item.startPos && utteranceEntity.endPos <= item.endPos)) {
return true;
}
}
}
}
return false;
});
if (priorLabelFound === undefined) {
utteranceObject.entities.push(utteranceEntity)
} else {
if (!utteranceObject.entities.find(item => deepEqual(item, utteranceEntity)))
{
let overlapHandling = parsedContent.LUISJsonStructure.onAmbiguousLabels || 'takeLongestLabel';
switch (overlapHandling.toLowerCase()) {
case 'takefirst':
break;
case 'takelast':
priorLabelFound.startPos = utteranceEntity.startPos;
priorLabelFound.endPos = utteranceEntity.endPos;
break;
case 'throwanerror':
let oldUtterance = expandUtterance(utterance, priorLabelFound);
let newUtterance = expandUtterance(utterance, utteranceEntity);
let errorMsg = `[Error] Duplicate overlapping labels found for entity '${priorLabelFound.name}' for Intent '${priorLabelFound.intent}'.\n 1. ${oldUtterance}\n 2. ${newUtterance}`;
throwDiagnosticError({
message: errorMsg,
range: utteranceAndEntities.range
});
default:
// take the longest label
if ((utteranceEntity.startPos >= priorLabelFound.startPos) && (utteranceEntity.endPos >= priorLabelFound.endPos)) {
priorLabelFound.startPos = utteranceEntity.startPos;
priorLabelFound.endPos = utteranceEntity.endPos;
}
}
}
}
});
}
} else {
// detect if utterance is a pattern and if so add it as a pattern
if (!utteranceStringToObjectMap.has(utterance)) {
if (helpers.isUtterancePattern(utterance)) {
let patternObject = new helperClass.pattern(utterance, intentName);
parsedContent.LUISJsonStructure.patterns.push(patternObject);
utteranceStringToObjectMap.set(utterance, patternObject);
} else {
let utteranceObject = new helperClass.utterances(utterance.replace(/\\[\[\]\(\)]/gi, match => match.slice(1)), intentName, []);
parsedContent.LUISJsonStructure.utterances.push(utteranceObject);
utteranceStringToObjectMap.set(utterance, utteranceObject);
}
}
}
}
}
}
}