in libraries/Parsers/Microsoft.Bot.Builder.Parsers.LU/Visitor.cs [121:211]
private static List<object> TokenizeUtterance(string expression)
{
var splitString = new List<object>();
var currentList = splitString;
AuxEntity currentEntity = null;
var entityNameCapture = false;
var entityValueCapture = false;
var entityRoleCapture = false;
TextElementEnumerator charEnum = StringInfo.GetTextElementEnumerator(expression);
while (charEnum.MoveNext())
{
var unicodeCharacter = charEnum.GetTextElement();
switch (unicodeCharacter)
{
case "{":
var newEntity = new AuxEntity
{
EntityName = string.Empty,
Role = string.Empty,
EntityValue = null,
Parent = currentEntity
};
currentList.Add(newEntity);
currentEntity = newEntity;
entityNameCapture = true;
entityValueCapture = false;
entityRoleCapture = false;
break;
case "}":
currentEntity = currentEntity.Parent;
currentList = currentEntity != null ? currentEntity.EntityValue : splitString;
entityNameCapture = false;
entityValueCapture = false;
entityRoleCapture = false;
break;
case "=":
currentEntity.EntityValue = new List<object>();
currentList = currentEntity.EntityValue;
entityNameCapture = false;
entityValueCapture = true;
entityRoleCapture = false;
break;
case ":":
if (currentEntity != null && !string.IsNullOrEmpty(currentEntity.EntityName) && entityNameCapture)
{
entityNameCapture = false;
entityValueCapture = false;
entityRoleCapture = true;
}
else
{
currentList.Add(unicodeCharacter);
}
break;
default:
if (entityNameCapture)
{
currentEntity.EntityName += unicodeCharacter;
}
else if (entityValueCapture)
{
if (unicodeCharacter == " ")
{
// we do not want leading spaces
if (currentList.Count != 0)
{
currentList.Add(unicodeCharacter);
}
}
else
{
currentList.Add(unicodeCharacter);
}
}
else if (entityRoleCapture)
{
currentEntity.Role += unicodeCharacter;
}
else
{
currentList.Add(unicodeCharacter);
}
break;
}
}
return splitString;
}