in packages/lu/src/parser/lufile/translate-helpers.js [36:249]
parseAndTranslate : async function(fileContent, ts) {
const initializeTS = new this.translationSettings();
const translationSettings = {...initializeTS, ...ts};
let batch_translate_size = translationSettings.batch_translate;
fileContent = helpers.sanitizeNewLines(fileContent);
let linesInFile = fileContent.split(NEWLINE);
let linesToTranslate = [];
let localizedContent = '';
let currentSectionType = '';
let inAnswer = false;
let lineCtr = 0;
for(let lineIndex in linesInFile) {
lineCtr++;
let currentLine = linesInFile[lineIndex].trim();
// is current line a comment?
if(currentLine.indexOf(PARSERCONSTS.COMMENT) === 0) {
if (inAnswer) {
addSegment(linesToTranslate, currentLine, true);
addSegment(linesToTranslate, NEWLINE, false);
continue;
}
if(translationSettings.translate_comments) {
addSegment(linesToTranslate, currentLine.charAt(0), false);
addSegment(linesToTranslate, currentLine.substring(1), true);
} else {
addSegment(linesToTranslate, currentLine, false);
}
} else if (currentLine.indexOf(PARSERCONSTS.FILTER) === 0) {
addSegment(linesToTranslate, currentLine, false);
currentSectionType = PARSERCONSTS.FILTER;
} else if (currentLine.indexOf(PARSERCONSTS.INTENT) === 0) {
if (inAnswer) {
addSegment(linesToTranslate, currentLine, true);
addSegment(linesToTranslate, NEWLINE, false);
continue;
}
let intentName = currentLine.substring(currentLine.indexOf(' ') + 1).trim();
//is this a QnA?
if(intentName.indexOf(PARSERCONSTS.QNA) === 0) {
let beforeQuestion = currentLine.substring(0, currentLine.indexOf(' ') + 1);
let question = intentName.slice(1).trim();
addSegment(linesToTranslate, beforeQuestion + '? ', false);
addSegment(linesToTranslate, question, true);
currentSectionType = PARSERCONSTS.QNA;
} else {
// we would not localize intent name but remember we are under intent section
currentSectionType = PARSERCONSTS.INTENT;
addSegment(linesToTranslate, currentLine, false);
}
} else if(currentLine.indexOf('-') === 0 ||
currentLine.indexOf('*') === 0 ||
currentLine.indexOf('+') === 0 ) {
if (inAnswer) {
addSegment(linesToTranslate, currentLine, true);
addSegment(linesToTranslate, NEWLINE, false);
continue;
}
// Fix for #1191. Do not localize meta-data filters for QnA.
if (currentSectionType === PARSERCONSTS.FILTER) {
addSegment(linesToTranslate, currentLine, false);
addSegment(linesToTranslate, NEWLINE, false);
continue;
}
let listSeparator = '';
let content = '';
switch (currentSectionType) {
case PARSERCONSTS.INTENT:
listSeparator = currentLine.charAt(0);
addSegment(linesToTranslate, listSeparator + ' ', false);
content = currentLine.slice(1).trim();
let skipChars = ['{', '}', '(', ')', '[', ']', '|', '=']
for (let i = 0; i < content.length; i++) {
let processedText = ''
let tslt = false
if (!skipChars.includes(content.charAt(i))) {
for (let j = i; j < content.length && !skipChars.includes(content.charAt(j)); j++) {
processedText += content.charAt(j)
}
tslt = true
} else if (content.charAt(i) == '{') {
for (let j = i; j < content.length && (content.charAt(j) !== '=' && content.charAt(j) !== '}'); j++) {
processedText += content.charAt(j)
}
} else {
processedText += content.charAt(i)
}
if (processedText.charAt(0) === ' ') {
addSegment(linesToTranslate, ' ', false)
}
addSegment(linesToTranslate, processedText, tslt)
content = content.slice(processedText.length)
i--
}
break;
case PARSERCONSTS.NEWENTITY:
// if current line is a normalized value, add it to the list to localize// strip line of the list separator
listSeparator = currentLine.charAt(0);
content = currentLine.slice(1).trim();
if (content.trim().endsWith(':')) {
let normalizedValueAsSynonym = content.replace(/:$/g, '').trim();
addSegment(linesToTranslate, `\t- ${normalizedValueAsSynonym}:`, false);
addSegment(linesToTranslate, NEWLINE, false);
addSegment(linesToTranslate, '\t\t- ', false);
addSegment(linesToTranslate, normalizedValueAsSynonym, true);
} else {
addSegment(linesToTranslate, '\t\t- ', false);
addSegment(linesToTranslate, content, true);
}
break;
case PARSERCONSTS.ENTITY:
case PARSERCONSTS.QNA:
default:
// strip line of the list separator
listSeparator = currentLine.charAt(0);
content = currentLine.slice(1).trim();
addSegment(linesToTranslate, listSeparator + ' ', false);
addSegment(linesToTranslate, content, true);
break;
}
} else if(currentLine.indexOf(PARSERCONSTS.ENTITY) === 0) {
if (inAnswer) {
addSegment(linesToTranslate, currentLine, true);
addSegment(linesToTranslate, NEWLINE, false);
continue;
}
// we need to localize qna alterations if specified.
let entityDef = currentLine.replace(PARSERCONSTS.ENTITY, '').split(':');
let entityName = entityDef[0];
let entityType = entityDef[1];
if(entityType.includes(PARSERCONSTS.QNAALTERATIONS)) {
addSegment(linesToTranslate, '$', false);
addSegment(linesToTranslate, entityName.trim(), true);
addSegment(linesToTranslate, ' : ' + PARSERCONSTS.QNAALTERATIONS + ' = ', false);
} else {
// we would not localize entity line but remember we are under entity section for list entities
// FIX for BF CLI # 121
// If list entity, add normalized value to list of synonyms to translate.
addSegment(linesToTranslate, currentLine, false);
if (entityType.trim().endsWith('=')) {
addSegment(linesToTranslate, NEWLINE, false);
let normalizedValueAsSynonym = entityType.replace('=', '').trim();
addSegment(linesToTranslate, '- ', false);
addSegment(linesToTranslate, normalizedValueAsSynonym, true);
}
}
} else if(currentLine.indexOf(PARSERCONSTS.ANSWER) === 0) {
if (inAnswer) {
let answerData = '';
}
addSegment(linesToTranslate, currentLine, false);
inAnswer = !inAnswer;
currentSectionType = PARSERCONSTS.ANSWER;
} else if (currentLine.indexOf(PARSERCONSTS.URLORFILEREF) ===0) {
if (inAnswer) {
addSegment(linesToTranslate, currentLine, true);
addSegment(linesToTranslate, NEWLINE, false);
continue;
}
currentSectionType = PARSERCONSTS.URLORFILEREF;
if(translationSettings.translate_link_text) {
const linkValueRegEx = new RegExp(/\(.*?\)/g);
let linkValueList = currentLine.trim().match(linkValueRegEx);
let linkValue = linkValueList[0].replace('(','').replace(')','');
const linkTextRegEx = new RegExp(/\[.*\]/g);
let linkTextList = currentLine.trim().match(linkTextRegEx);
let linkTextValue = linkTextList[0].replace('[','').replace(']','');
addSegment(linesToTranslate, '[', false);
addSegment(linesToTranslate, linkTextValue, true);
addSegment(linesToTranslate, ']', false);
addSegment(linesToTranslate, '(' + linkValue + ')', false);
} else {
addSegment(linesToTranslate, currentLine, false);
}
} else if(currentLine === '') {
if (inAnswer) {
addSegment(linesToTranslate, NEWLINE, false);
continue;
}
} else if(currentLine.indexOf(PARSERCONSTS.NEWENTITY) === 0) {
// Nothing in the entity line should be localized.
addSegment(linesToTranslate, currentLine, false);
currentSectionType = PARSERCONSTS.NEWENTITY;
} else {
if (inAnswer) {
addSegment(linesToTranslate, currentLine, true);
addSegment(linesToTranslate, NEWLINE, false);
continue;
} else {
throw(new exception(retCode.errorCode.INVALID_INPUT_FILE, 'Error: Unexpected line encountered when parsing \n' + '[' + lineIndex + ']:' + currentLine));
}
}
addSegment(linesToTranslate, NEWLINE, false);
// do we have any payload to localize? and have we hit the batch size limit?
if ((linesToTranslate.length !== 0) && (lineCtr % batch_translate_size === 0)) {
try {
localizedContent += await batchTranslateText(linesToTranslate, translationSettings);
linesToTranslate = [];
} catch (err) {
throw (err)
}
}
}
if (linesToTranslate.length !== 0) {
try {
localizedContent += await batchTranslateText(linesToTranslate, translationSettings);
linesToTranslate = [];
} catch (err) {
throw (err)
}
}
return localizedContent;
},