in lambda/es-proxy-layer/lib/kendra.js [275:584]
async function routeKendraRequest(event, context) {
// remove any prior session attributes for kendra
_.unset(event,"res.session.qnabotcontext.kendra.kendraQueryId") ;
_.unset(event,"res.session.qnabotcontext.kendra.kendraIndexId") ;
_.unset(event,"res.session.qnabotcontext.kendra.kendraResultId") ;
_.unset(event,"res.session.qnabotcontext.kendra.kendraResponsibleQid") ;
let promises = [];
let resArray = [];
let kendraClient = undefined;
// if test environment, then use mock-up of kendraClient
if (event.test) {
var mockup = './test/mockClient' + event.test + '.js';
kendraClient = require(mockup);
} else {
AWS.config.update({
maxRetries: _.get(event.req["_settings"], "KENDRAFAQ_CONFIG_MAX_RETRIES"),
retryDelayOptions: {
base: _.get(event.req["_settings"], "KENDRAFAQ_CONFIG_RETRY_DELAY")
},
});
kendraClient = (process.env.REGION ?
new AWS.Kendra({apiVersion: '2019-02-03', region: process.env.REGION}) :
new AWS.Kendra({apiVersion: '2019-02-03'})
);
}
// process query against Kendra for QnABot
let indexes = event.req["_settings"]["ALT_SEARCH_KENDRA_INDEXES"] ? event.req["_settings"]["ALT_SEARCH_KENDRA_INDEXES"] : process.env.KENDRA_INDEXES
var kendraResultsCached = _.get(event.res, "kendraResultsCached");
if (indexes && indexes.length) {
try {
// parse JSON array of kendra indexes
kendraIndexes = JSON.parse(indexes);
} catch (err) {
// assume setting is a string containing single index
kendraIndexes = [ indexes ];
}
}
if (kendraIndexes === undefined) {
throw new Error('Undefined Kendra Indexes');
}
// This function can handle configuration with an array of kendraIndexes.
// Iterate through this area and perform queries against Kendra.
kendraIndexes.forEach(function (index, i) {
// if results cached from KendraFAQ, skip index by pushing Promise to resolve cached results
if (kendraResultsCached && index===kendraResultsCached.originalKendraIndexId) {
qnabot.log(`retrieving cached kendra results`)
promises.push(new Promise(function(resolve, reject) {
var data = kendraResultsCached
_.set(event.req, "kendraResultsCached", "cached and retrieved"); // cleans the logs
data.originalKendraIndexId = index;
qnabot.log("Data from Kendra request:" + JSON.stringify(data,null,2));
resArray.push(data);
resolve(data);
}));
return;
}
const params = {
IndexId: index, /* required */
QueryText: event.req["question"], /* required */
};
let p = kendraRequester(kendraClient,params,resArray);
promises.push(p);
});
// wait for all kendra queries to complete
await Promise.all(promises);
// process kendra query responses and update answer content
/* default message text - can be overridden using QnABot SSM Parameter Store Custom Property */
let topAnswerMessage = event.req["_settings"]["ALT_SEARCH_KENDRA_TOP_ANSWER_MESSAGE"] + "\n\n"; //"Amazon Kendra suggested answer. \n\n ";
let topAnswerMessageMd = event.req["_settings"]["ALT_SEARCH_KENDRA_TOP_ANSWER_MESSAGE"] == "" ? "" : `*${event.req["_settings"]["ALT_SEARCH_KENDRA_TOP_ANSWER_MESSAGE"]}* \n `;
let answerMessage = event.req["_settings"]["ALT_SEARCH_KENDRA_ANSWER_MESSAGE"];
let answerMessageMd = event.req["_settings"]["ALT_SEARCH_KENDRA_ANSWER_MESSAGE"] == "" ? "" : `*${answerMessage}* \n `;
let faqanswerMessage = event.req["_settings"]["ALT_SEARCH_KENDRA_FAQ_MESSAGE"] + "\n\n"; //'Answer from Amazon Kendra FAQ.'
let faqanswerMessageMd = event.req["_settings"]["ALT_SEARCH_KENDRA_FAQ_MESSAGE"] == "" ? "" : `*${event.req["_settings"]["ALT_SEARCH_KENDRA_FAQ_MESSAGE"]}* \n`
let minimum_score = event.req["_settings"]["ALT_SEARCH_KENDRA_FALLBACK_CONFIDENCE_SCORE"];
let useFullMessageForSpeech = _.get(event.req,"_settings.ALT_SEARCH_KENDRA_ABBREVIATE_MESSAGE_FOR_SSML","true").toString().toUpperCase() === "FALSE"
let speechMessage = "";
let helpfulLinksMsg = 'Source Link';
let maxDocumentCount = _.get(event.req,'_settings.ALT_SEARCH_KENDRA_MAX_DOCUMENT_COUNT',2);
var seenTop = false;
let searchTypes = _.get(event.req,"_settings.ALT_SEARCH_KENDRA_RESPONSE_TYPES","ANSWER,DOCUMENT,QUESTION_ANSWER").toUpperCase().split(",")
let foundAnswerCount = 0;
let foundDocumentCount = 0;
let kendraQueryId;
let kendraIndexId;
let kendraResultId;
let answerDocumentUris = new Set();
let helpfulDocumentsUris = new Set();
let signS3Urls = _.get(event.req,"_settings.ALT_SEARCH_KENDRA_S3_SIGNED_URLS",true);
let expireSeconds = _.get(event.req,"_settings.ALT_SEARCH_KENDRA_S3_SIGNED_URL_EXPIRE_SECS",300);
var answerTextMd
var debug_results = [];
let allFilteredMessages = [];
resArray.forEach(function (res) {
if (res && res.ResultItems.length > 0) {
res.ResultItems.forEach(function (element, i) {
if(!confidence_filter(minimum_score,element)){
return;
}
if(!response_filter(searchTypes,element)){
return;
}
if(seenTop){
return;
}
/* Note - only the first answer will be provided back to the requester */
if (element.Type === 'ANSWER' && foundAnswerCount === 0 && element.AdditionalAttributes &&
element.AdditionalAttributes.length > 0 &&
element.AdditionalAttributes[0].Value.TextWithHighlightsValue.Text) {
answerMessage += '\n\n ' + element.AdditionalAttributes[0].Value.TextWithHighlightsValue.Text.replace(/\r?\n|\r/g, " ");
allFilteredMessages.push(answerMessage)
// Emboldens the highlighted phrases returned by the Kendra response API in markdown format
answerTextMd = element.AdditionalAttributes[0].Value.TextWithHighlightsValue.Text.replace(/\r?\n|\r/g, " ");
// iterates over the answer highlights in sorted order of BeginOffset, merges the overlapping intervals
let sorted_highlights = mergeIntervals(element.AdditionalAttributes[0].Value.TextWithHighlightsValue.Highlights);
let j, elem;
for (j=0; j<sorted_highlights.length; j++) {
elem = sorted_highlights[j];
let offset = 4*j;
if (elem.TopAnswer == true) { // if top answer is found, then answer is abbreviated to this phrase
seenTop = true;
answerMessageMd = topAnswerMessageMd;
answerTextMd = addMarkdownHighlights(answerTextMd, elem.BeginOffset+offset, elem.EndOffset+offset, true) ;
answerMessage = topAnswerMessage + answerTextMd + '.';
speechMessage = answerTextMd ;
break;
} else {
answerTextMd = addMarkdownHighlights(answerTextMd, elem.BeginOffset+offset, elem.EndOffset+offset, false) ;
}
}
answerMessageMd = answerMessageMd + '\n\n' + answerTextMd;
// Shortens the speech response to contain say the longest highlighted phrase ONLY IF top answer not found
if (seenTop == false) {
var longest_highlight = longestInterval(sorted_highlights);
let answerText = element.AdditionalAttributes[0].Value.TextWithHighlightsValue.Text.replace(/\r?\n|\r/g, " ");
// speechMessage = answerText.substring(longest_highlight.BeginOffset, longest_highlight.EndOffset) + '.';
var pattern = new RegExp('[^.]* '+longest_highlight+'[^.]*\.[^.]*\.')
pattern.lastIndex = 0; // must reset this property of regex object for searches
speechMessage = pattern.exec(answerText)[0]
}
// Convert S3 Object URLs to signed URLs
answerDocumentUris.add(element);
kendraQueryId = res.QueryId; // store off the QueryId to use as a session attribute for feedback
kendraIndexId = res.originalKendraIndexId; // store off the Kendra IndexId to use as a session attribute for feedback
kendraResultId = element.Id; // store off resultId to use as a session attribute for feedback
foundAnswerCount++;
debug_results.push(create_debug_object(element))
} else if (element.Type === 'QUESTION_ANSWER' && element.AdditionalAttributes && element.AdditionalAttributes.length > 1) {
// There will be 2 elements - [0] - QuestionText, [1] - AnswerText
if(isSyncedFromQnABot(element)){
return;
}
let message = element.AdditionalAttributes[1].Value.TextWithHighlightsValue.Text.replace(/\r?\n|\r/g, " ")
answerMessage = faqanswerMessage + '\n\n ' + message;
allFilteredMessages.push(message)
seenTop = true; // if the answer is in the FAQ, don't show document extracts
answerDocumentUris=[];
let answerTextMd = element.AdditionalAttributes[1].Value.TextWithHighlightsValue.Text.replace(/\r?\n|\r/g, " ");
// iterates over the FAQ answer highlights in sorted order of BeginOffset, merges the overlapping intervals
let sorted_highlights = mergeIntervals(element.AdditionalAttributes[1].Value.TextWithHighlightsValue.Highlights);
let j, elem;
for (j=0; j<sorted_highlights.length; j++) {
elem = sorted_highlights[j];
let offset = 4*j;
answerTextMd = addMarkdownHighlights(answerTextMd, elem.BeginOffset+offset, elem.EndOffset+offset, false) ;
}
answerMessageMd = faqanswerMessageMd + '\n\n' + answerTextMd;
kendraQueryId = res.QueryId; // store off the QueryId to use as a session attribute for feedback
kendraIndexId = res.originalKendraIndexId; // store off the Kendra IndexId to use as a session attribute for feedback
kendraResultId = element.Id; // store off resultId to use as a session attribute for feedback
foundAnswerCount++;
debug_results.push(create_debug_object(element))
} else if (element.Type === 'DOCUMENT' && element.DocumentExcerpt.Text && element.DocumentURI) {
const docInfo = {}
// if topAnswer found, then do not show document excerpts
if (seenTop == false) {
docInfo.text = element.DocumentExcerpt.Text.replace(/\r?\n|\r/g, " ");
allFilteredMessages.push(docInfo.text)
// iterates over the document excerpt highlights in sorted order of BeginOffset, merges overlapping intervals
var sorted_highlights = mergeIntervals(element.DocumentExcerpt.Highlights);
var j, elem;
for (j=0; j<sorted_highlights.length; j++) {
elem = sorted_highlights[j];
let offset = 4*j;
let beginning = docInfo.text.substring(0, elem.BeginOffset+offset);
let highlight = docInfo.text.substring(elem.BeginOffset+offset, elem.EndOffset+offset);
let rest = docInfo.text.substr(elem.EndOffset+offset);
docInfo.text = beginning + '**' + highlight + '**' + rest;
};
if (foundAnswerCount == 0 && foundDocumentCount == 0) {
speechMessage = element.DocumentExcerpt.Text.replace(/\r?\n|\r/g, " ");;
if (sorted_highlights.length > 0) {
var highlight = speechMessage.substring(sorted_highlights[0].BeginOffset, sorted_highlights[0].EndOffset)
var pattern = new RegExp('[^.]* '+highlight+'[^.]*\.[^.]*\.')
pattern.lastIndex = 0; // must reset this property of regex object for searches
var regexMatch = pattern.exec(speechMessage)
//TODO: Investigate this. Should this be a nohits scenerio?
if(regexMatch){
speechMessage = regexMatch[0]
}
}
}
}
// but even if topAnswer is found, show URL in markdown
docInfo.uri = `${element.DocumentURI}`;
let title;
if (element.DocumentTitle && element.DocumentTitle.Text) {
docInfo.Title = element.DocumentTitle.Text;
}
helpfulDocumentsUris.add(docInfo);
// foundAnswerCount++;
foundDocumentCount++;
debug_results.push(create_debug_object(element))
}
});
}
});
// update QnABot answer content for ssml, markdown, and text
let ssmlMessage = ""
let hit;
let markdown = answerMessageMd;
let message = answerMessage;
if (foundAnswerCount > 0 || foundDocumentCount > 0) {
ssmlMessage = `${answerMessage.substring(0,600).replace(/\r?\n|\r/g, " ")}`;
if (speechMessage != "") {
ssmlMessage = `${speechMessage.substring(0,600).replace(/\r?\n|\r/g, " ")}`;
}
let lastIndex = ssmlMessage.lastIndexOf('.');
if (lastIndex > 0) {
ssmlMessage = ssmlMessage.substring(0,lastIndex);
}
ssmlMessage = `<speak> ${ssmlMessage} </speak>`;
}
if (answerDocumentUris.size > 0) {
markdown += `\n\n ${helpfulLinksMsg}: `;
answerDocumentUris.forEach(function(element) {
// Convert S3 Object URLs to signed URLs
if (signS3Urls) {
element.DocumentURI = signS3URL(element.DocumentURI, expireSeconds);
}
markdown += `<span translate=no>[${element.DocumentTitle.Text}](${element.DocumentURI})</span>`;
});
}
let idx=foundAnswerCount;
if (seenTop == false){
helpfulDocumentsUris.forEach(function (element) {
if (idx++ < maxDocumentCount) {
markdown += `\n\n`;
markdown += `***`;
markdown += `\n\n <br>`;
if (element.text && element.text.length > 0 && event.req._preferredResponseType != "SSML") { //don't append doc search to SSML answers
markdown += `\n\n ${element.text}`;
message += `\n\n ${element.text}`;
}
let label = element.Title ;
// Convert S3 Object URLs to signed URLs
if (signS3Urls) {
element.uri = signS3URL(element.uri, expireSeconds)
}
markdown += `\n\n ${helpfulLinksMsg}: <span translate=no>[${label}](${element.uri})</span>`;
}
});
}
var req = event.req;
if(useFullMessageForSpeech){
ssmlMessage = allFilteredMessages.length > 0 ? allFilteredMessages[0] : ssmlMessage
}
hit = create_hit(message,markdown,ssmlMessage, foundAnswerCount + foundDocumentCount, debug_results,{
kendraQueryId: kendraQueryId,
kendraIndexId: kendraIndexId,
kendraResultId: kendraResultId,
kendraFoundAnswerCount: foundAnswerCount,
kendraFoundDocumentCount: foundDocumentCount,
maxDocuments: maxDocumentCount
})
qnabot.log("Returning event: ", JSON.stringify(hit, null, 2));
return hit;
}