private static List TokenizeUtterance()

in libraries/Parsers/Microsoft.Bot.Builder.Parsers.LU/Visitor.cs [121:211]


        private static List<object> TokenizeUtterance(string expression)
        {
            var splitString = new List<object>();
            var currentList = splitString;
            AuxEntity currentEntity = null;
            var entityNameCapture = false;
            var entityValueCapture = false;
            var entityRoleCapture = false;

            TextElementEnumerator charEnum = StringInfo.GetTextElementEnumerator(expression);
            while (charEnum.MoveNext())
            {
                var unicodeCharacter = charEnum.GetTextElement();
                switch (unicodeCharacter)
                {
                    case "{":
                        var newEntity = new AuxEntity
                        {
                            EntityName = string.Empty,
                            Role = string.Empty,
                            EntityValue = null,
                            Parent = currentEntity
                        };
                        currentList.Add(newEntity);
                        currentEntity = newEntity;
                        entityNameCapture = true;
                        entityValueCapture = false;
                        entityRoleCapture = false;
                        break;
                    case "}":
                        currentEntity = currentEntity.Parent;
                        currentList = currentEntity != null ? currentEntity.EntityValue : splitString;
                        entityNameCapture = false;
                        entityValueCapture = false;
                        entityRoleCapture = false;
                        break;
                    case "=":
                        currentEntity.EntityValue = new List<object>();
                        currentList = currentEntity.EntityValue;
                        entityNameCapture = false;
                        entityValueCapture = true;
                        entityRoleCapture = false;
                        break;
                    case ":":
                        if (currentEntity != null && !string.IsNullOrEmpty(currentEntity.EntityName) && entityNameCapture)
                        {
                            entityNameCapture = false;
                            entityValueCapture = false;
                            entityRoleCapture = true;
                        }
                        else
                        {
                            currentList.Add(unicodeCharacter);
                        }

                        break;
                    default:
                        if (entityNameCapture)
                        {
                            currentEntity.EntityName += unicodeCharacter;
                        }
                        else if (entityValueCapture)
                        {
                            if (unicodeCharacter == " ")
                            {
                                // we do not want leading spaces
                                if (currentList.Count != 0)
                                {
                                    currentList.Add(unicodeCharacter);
                                }
                            }
                            else
                            {
                                currentList.Add(unicodeCharacter);
                            }
                        }
                        else if (entityRoleCapture)
                        {
                            currentEntity.Role += unicodeCharacter;
                        }
                        else
                        {
                            currentList.Add(unicodeCharacter);
                        }

                        break;
                }
            }

            return splitString;
        }