in packages/Ludown/lib/parseFileContents.js [766:1054]
const parseAndHandleEntity = function (parsedContent, luResource, log, locale) {
// handle entity
let entities = luResource.Entities;
if (entities && entities.length > 0) {
for (const entity of entities) {
let entityName = entity.Name;
let entityType = entity.Type;
let parsedRoleAndType = helpers.getRolesAndType(entityType);
let entityRoles = parsedRoleAndType.roles;
entityType = parsedRoleAndType.entityType;
let pEntityName = (entityName.toLowerCase() === 'prebuilt') ? entityType : entityName;
// see if we already have this as Pattern.Any entity
// see if we already have this in patternAny entity collection; if so, remove it but remember the roles (if any)
for (let i in parsedContent.LUISJsonStructure.patternAnyEntities) {
if (parsedContent.LUISJsonStructure.patternAnyEntities[i].name === pEntityName) {
if (entityType.toLowerCase().trim().includes('phraselist')) {
let errorMsg = `Phrase lists cannot be used as an entity in a pattern "${pEntityName}"`;
let error = BuildDiagnostic({
message: errorMsg,
context: entity.ParseTree.entityLine()
})
throw (new exception(retCode.errorCode.INVALID_INPUT, error.toString()));
}
if (parsedContent.LUISJsonStructure.patternAnyEntities[i].roles.length !== 0) entityRoles = parsedContent.LUISJsonStructure.patternAnyEntities[i].roles;
parsedContent.LUISJsonStructure.patternAnyEntities.splice(i, 1);
break;
}
}
// add this entity to appropriate place
// is this a builtin type?
if (builtInTypes.consolidatedList.includes(entityType)) {
locale = locale ? locale.toLowerCase() : 'en-us';
// check if this pre-built entity is already labelled in an utterance and or added as a simple entity. if so, throw an error.
try {
let rolesImport = VerifyAndUpdateSimpleEntityCollection(parsedContent, entityType, entityName);
if (rolesImport.length !== 0) {
rolesImport.forEach(role => !entityRoles.includes(role) ? entityRoles.push(role) : undefined);
}
} catch (err) {
throw (err);
}
// verify if the requested entityType is available in the requested locale
let prebuiltCheck = builtInTypes.perLocaleAvailability[locale][entityType];
if (prebuiltCheck === null) {
if (log) {
process.stdout.write(chalk.default.yellowBright('[WARN]: Requested PREBUILT entity "' + entityType + ' is not available for the requested locale: ' + locale + '\n'));
process.stdout.write(chalk.default.yellowBright(' Skipping this prebuilt entity..\n'));
}
} else if (prebuiltCheck && prebuiltCheck.includes('datetime')) {
if (log) {
process.stdout.write(chalk.default.yellowBright('[WARN]: PREBUILT entity "' + entityType + ' is not available for the requested locale: ' + locale + '\n'));
process.stdout.write(chalk.default.yellowBright(' Switching to ' + builtInTypes.perLocaleAvailability[locale][entityType] + ' instead.\n'));
}
entityType = builtInTypes.perLocaleAvailability[locale][entityType];
addItemOrRoleIfNotPresent(parsedContent.LUISJsonStructure, LUISObjNameEnum.PREBUILT, entityType, entityRoles);
} else {
// add to prebuiltEntities if it does not exist there.
addItemOrRoleIfNotPresent(parsedContent.LUISJsonStructure, LUISObjNameEnum.PREBUILT, entityType, entityRoles);
}
} else if (entityType.toLowerCase() === 'simple') {
// add this to entities if it doesnt exist
addItemOrRoleIfNotPresent(parsedContent.LUISJsonStructure, LUISObjNameEnum.ENTITIES, entityName, entityRoles);
} else if (entityType.endsWith('=')) {
// is this qna maker alterations list?
if (entityType.includes(PARSERCONSTS.QNAALTERATIONS)) {
let alterationlist = [entity.Name];
if (entity.SynonymsOrPhraseList && entity.SynonymsOrPhraseList.length > 0) {
alterationlist = alterationlist.concat(entity.SynonymsOrPhraseList);
parsedContent.qnaAlterations.wordAlterations.push(new qnaAlterations.alterations(alterationlist));
} else {
let errorMsg = `QnA alteration section: "${alterationlist}" does not have list decoration. Prefix line with "-" or "+" or "*"`;
let error = BuildDiagnostic({
message: errorMsg,
context: entity.ParseTree.entityLine()
})
throw (new exception(retCode.errorCode.SYNONYMS_NOT_A_LIST, error.toString()));
}
} else {
// treat this as a LUIS list entity type
let parsedEntityTypeAndRole = helpers.getRolesAndType(entityType);
entityType = parsedEntityTypeAndRole.entityType;
(parsedEntityTypeAndRole.roles || []).forEach(role => !entityRoles.includes(role) ? entityRoles.push(role) : undefined);
// check if this list entity is already labelled in an utterance and or added as a simple entity. if so, throw an error.
try {
let rolesImport = VerifyAndUpdateSimpleEntityCollection(parsedContent, entityName, 'List');
if (rolesImport.length !== 0) {
rolesImport.forEach(role => {
if (!entityRoles.includes(role)) {
entityRoles.push(role)
}
})
}
} catch (err) {
throw (err);
}
// get normalized value
let normalizedValue = entityType.substring(0, entityType.length - 1).trim();
let synonymsList = entity.SynonymsOrPhraseList;
let closedListExists = helpers.filterMatch(parsedContent.LUISJsonStructure.closedLists, 'name', entityName);
if (closedListExists.length === 0) {
parsedContent.LUISJsonStructure.closedLists.push(new helperClass.closedLists(entityName, [new helperClass.subList(normalizedValue, synonymsList)], entityRoles));
} else {
// closed list with this name already exists
let subListExists = helpers.filterMatch(closedListExists[0].subLists, 'canonicalForm', normalizedValue);
if (subListExists.length === 0) {
closedListExists[0].subLists.push(new helperClass.subList(normalizedValue, synonymsList));
} else {
synonymsList.forEach(function (listItem) {
if (!subListExists[0].list.includes(listItem)) subListExists[0].list.push(listItem);
})
}
// see if the roles all exist and if not, add them
mergeRoles(closedListExists[0].roles, entityRoles);
}
}
} else if (entityType.toLowerCase().trim().indexOf('phraselist') === 0) {
if (entityRoles.length !== 0) {
let errorMsg = `Phrase list entity ${entityName} has invalid role definition with roles = ${entityRoles.join(', ')}. Roles are not supported for Phrase Lists`;
let error = BuildDiagnostic({
message: errorMsg,
context: entity.ParseTree.entityLine()
})
throw (new exception(retCode.errorCode.INVALID_INPUT, error.toString()));
}
// check if this phraselist entity is already labelled in an utterance and or added as a simple entity. if so, throw an error.
try {
let rolesImport = VerifyAndUpdateSimpleEntityCollection(parsedContent, entityName, 'Phrase List');
if (rolesImport.length !== 0) {
rolesImport.forEach(role => !entityRoles.includes(role) ? entityRoles.push(role) : undefined);
}
} catch (err) {
throw (err);
}
// is this interchangeable?
let intc = false;
if (entityType.toLowerCase().includes('interchangeable')) intc = true;
// add this to phraseList if it doesnt exist
let pLValues = new Array();
let plValuesList = "";
for (const phraseListValues of entity.SynonymsOrPhraseList) {
pLValues.push(phraseListValues.split(','));
plValuesList = plValuesList + phraseListValues + ',';
}
// remove the last ','
plValuesList = plValuesList.substring(0, plValuesList.lastIndexOf(','));
let modelExists = false;
if (parsedContent.LUISJsonStructure.model_features.length > 0) {
let modelIdx = 0;
for (modelIdx in parsedContent.LUISJsonStructure.model_features) {
if (parsedContent.LUISJsonStructure.model_features[modelIdx].name === entityName) {
modelExists = true;
break;
}
}
if (modelExists) {
if (parsedContent.LUISJsonStructure.model_features[modelIdx].mode === intc) {
// for each item in plValues, see if it already exists
pLValues.forEach(function (plValueItem) {
if (!parsedContent.LUISJsonStructure.model_features[modelIdx].words[0].includes(plValueItem)) parsedContent.LUISJsonStructure.model_features[modelIdx].words += ',' + pLValues;
})
} else {
let errorMsg = `Phrase list: "${entityName}" has conflicting definitions. One marked interchangeable and another not interchangeable`;
let error = BuildDiagnostic({
message: errorMsg,
context: entity.ParseTree.entityLine()
})
throw (new exception(retCode.errorCode.INVALID_INPUT, error.toString()));
}
} else {
parsedContent.LUISJsonStructure.model_features.push(new helperClass.modelObj(entityName, intc, plValuesList, true));
}
} else {
parsedContent.LUISJsonStructure.model_features.push(new helperClass.modelObj(entityName, intc, plValuesList, true));
}
} else if (entityType.startsWith('[')) {
// remove simple entity definitions for composites but carry forward roles.
// Find this entity if it exists in the simple entity collection
let simpleEntityExists = (parsedContent.LUISJsonStructure.entities || []).find(item => item.name == entityName);
if (simpleEntityExists !== undefined) {
// take and add any roles into the roles list
(simpleEntityExists.roles || []).forEach(role => !entityRoles.includes(role) ? entityRoles.push(role) : undefined);
// remove this simple entity definition
for (var idx = 0; idx < parsedContent.LUISJsonStructure.entities.length; idx++) {
if (parsedContent.LUISJsonStructure.entities[idx].name === simpleEntityExists.name) {
parsedContent.LUISJsonStructure.entities.splice(idx, 1);
}
}
}
// handle composite entity definition
// drop [] and trim
let childDefinition = entityType.trim().replace('[', '').replace(']', '').trim();
if (childDefinition.length === 0) {
let errorMsg = `Composite entity: ${entityName} is missing child entity definitions. Child entities are denoted via [entity1, entity2] notation.`;
let error = BuildDiagnostic({
message: errorMsg,
context: entity.ParseTree.entityLine()
})
throw (new exception(retCode.errorCode.INVALID_COMPOSITE_ENTITY, error.toString()));
}
// split the children based on ',' or ';' delimiter. Trim each child to remove white spaces.
let compositeChildren = childDefinition.split(new RegExp(/[,;]/g)).map(item => item.trim());
// add this composite entity if it does not exist
let compositeEntity = (parsedContent.LUISJsonStructure.composites || []).find(item => item.name == entityName);
if (compositeEntity === undefined) {
// add new composite entity
parsedContent.LUISJsonStructure.composites.push(new helperClass.compositeEntity(entityName, compositeChildren, entityRoles));
// remove composite that might have been tagged as a simple entity due to inline entity definition in an utterance
parsedContent.LUISJsonStructure.entities = (parsedContent.LUISJsonStructure.entities || []).filter(entity => entity.name != entityName);
} else {
if (JSON.stringify(compositeChildren.sort()) !== JSON.stringify(compositeEntity.children.sort())) {
let errorMsg = `Composite entity: ${entityName} has multiple definition with different children. \n 1. ${compositeChildren.join(', ')}\n 2. ${compositeEntity.children.join(', ')}`;
let error = BuildDiagnostic({
message: errorMsg,
context: entity.ParseTree.entityLine()
})
throw (new exception(retCode.errorCode.INVALID_COMPOSITE_ENTITY, error.toString()));
} else {
// update roles
addItemOrRoleIfNotPresent(parsedContent.LUISJsonStructure, LUISObjNameEnum.COMPOSITES, compositeEntity.name, entityRoles);
}
}
} else if (entityType.startsWith('/')) {
if (entityType.endsWith('/')) {
// check if this regex entity is already labelled in an utterance and or added as a simple entity. if so, throw an error.
try {
let rolesImport = VerifyAndUpdateSimpleEntityCollection(parsedContent, entityName, 'RegEx');
if (rolesImport.length !== 0) {
rolesImport.forEach(role => !entityRoles.includes(role) ? entityRoles.push(role) : undefined);
}
} catch (err) {
throw (err);
}
// handle regex entity
let regex = entityType.slice(1).slice(0, entityType.length - 2);
if (regex === '') {
let errorMsg = `RegEx entity: ${regExEntity.name} has empty regex pattern defined.`;
let error = BuildDiagnostic({
message: errorMsg,
context: entity.ParseTree.entityLine()
})
throw (new exception(retCode.errorCode.INVALID_REGEX_ENTITY, error.toString()));
}
// add this as a regex entity if it does not exist
let regExEntity = (parsedContent.LUISJsonStructure.regex_entities || []).find(item => item.name == entityName);
if (regExEntity === undefined) {
parsedContent.LUISJsonStructure.regex_entities.push(new helperClass.regExEntity(entityName, regex, entityRoles))
} else {
// throw an error if the pattern is different for the same entity
if (regExEntity.regexPattern !== regex) {
let errorMsg = `RegEx entity: ${regExEntity.name} has multiple regex patterns defined. \n 1. /${regex}/\n 2. /${regExEntity.regexPattern}/`;
let error = BuildDiagnostic({
message: errorMsg,
context: entity.ParseTree.entityLine()
})
throw (new exception(retCode.errorCode.INVALID_REGEX_ENTITY, error.toString()));
} else {
// update roles
addItemOrRoleIfNotPresent(parsedContent.LUISJsonStructure, LUISObjNameEnum.REGEX, regExEntity.name, entityRoles);
}
}
} else {
let errorMsg = `RegEx entity: ${regExEntity.name} is missing trailing '/'. Regex patterns need to be enclosed in forward slashes. e.g. /[0-9]/`;
let error = BuildDiagnostic({
message: errorMsg,
context: entity.ParseTree.entityLine()
})
throw (new exception(retCode.errorCode.INVALID_REGEX_ENTITY, error.toString()));
}
} else {
// TODO: handle other entity types
}
}
}
}