in interactive_engine/executor/ir/common/src/expr_parse/token.rs [332:537]
fn partial_tokens_to_tokens(mut tokens: &[PartialToken]) -> ExprResult<Vec<Token>> {
let mut result = Vec::new();
let mut recent_token: Option<Token> = None;
while !tokens.is_empty() {
let first = tokens[0].clone();
let second = tokens.get(1).cloned();
let third = tokens.get(2).cloned();
let mut cutoff = 2;
let curr_token = match first {
PartialToken::Token(token) => {
cutoff = 1;
Some(token)
}
PartialToken::Literal(literal) => {
cutoff = 1;
if let Ok(number) = literal.parse::<i64>() {
Some(Token::Int(number))
} else if let Ok(number) = literal.parse::<f64>() {
Some(Token::Float(number))
} else if let Ok(boolean) = literal.parse::<bool>() {
Some(Token::Boolean(boolean))
} else if literal.to_lowercase().as_str() == "within" {
Some(Token::Within)
} else if literal.to_lowercase().as_str() == "without" {
Some(Token::Without)
} else if literal.to_lowercase().as_str() == "startswith" {
Some(Token::StartsWith)
} else if literal.to_lowercase().as_str() == "endswith" {
Some(Token::EndsWith)
} else if literal.to_lowercase().as_str() == "isnull" {
Some(Token::IsNull)
} else {
// To parse the float of the form `<coefficient>e{+,-}<exponent>`,
// for example [Literal("10e"), Minus, Literal("3")] => "1e-3".parse().
match (second, third) {
(Some(second), Some(third))
if second == PartialToken::Minus
|| second == PartialToken::Token(Token::Plus) =>
{
let second_sign = match second {
PartialToken::Minus => "-",
_ => "+",
};
let third_num = match third {
PartialToken::Literal(s) => s,
_ => "".to_string(),
};
if let Ok(number) =
format!("{}{}{}", literal, second_sign, third_num).parse::<f64>()
{
cutoff = 3;
Some(Token::Float(number))
} else {
Some(Token::Identifier(literal.to_string()))
}
}
_ => Some(Token::Identifier(literal.to_string())),
}
}
}
PartialToken::Whitespace => {
cutoff = 1;
None
}
PartialToken::Minus => {
// Should we consider minus as a negative sign
let is_negative_sign =
{ recent_token.is_none() || !recent_token.as_ref().unwrap().is_operand() };
if is_negative_sign {
match &second {
// Be aware that minus can represent both subtraction or negative sign
// if it is a negative sign, it must be directly trailed by a number.
// However, we cannot actually tell whether the case "x -y", is actually
// subtracting x by y, or -y must be treated as a number.
Some(PartialToken::Literal(literal)) => {
// Must check whether previous is what
if let Ok(number) = literal.parse::<i64>() {
Some(Token::Int(-number))
} else if let Ok(number) = literal.parse::<f64>() {
Some(Token::Float(-number))
} else {
return Err(ExprError::unmatched_partial_token(first, second));
}
}
_ => {
cutoff = 1;
Some(Token::Minus)
}
}
} else {
cutoff = 1;
Some(Token::Minus)
}
}
PartialToken::Eq => match second {
Some(PartialToken::Eq) => Some(Token::Eq),
_ => {
return Err(ExprError::unmatched_partial_token(first, second));
}
},
PartialToken::ExclamationMark => match second {
Some(PartialToken::Eq) => Some(Token::Ne),
_ => {
cutoff = 1;
Some(Token::Not)
}
},
PartialToken::Gt => match second {
Some(PartialToken::Eq) => Some(Token::Ge),
Some(PartialToken::Gt) => Some(Token::BitRShift), // >>
_ => {
cutoff = 1;
Some(Token::Gt)
}
},
PartialToken::Lt => match second {
Some(PartialToken::Eq) => Some(Token::Le),
Some(PartialToken::Lt) => Some(Token::BitLShift), // <<
_ => {
cutoff = 1;
Some(Token::Lt)
}
},
PartialToken::Ampersand => match second {
Some(PartialToken::Ampersand) => Some(Token::And),
// _ => return Err(ExprError::unmatched_partial_token(first, second)),
_ => {
cutoff = 1;
Some(Token::BitAnd)
}
},
PartialToken::VerticalBar => match second {
Some(PartialToken::VerticalBar) => Some(Token::Or),
// _ => return Err(ExprError::unmatched_partial_token(first, second)),
_ => {
cutoff = 1;
Some(Token::BitOr)
}
},
PartialToken::LBracket | PartialToken::LCBracket => {
let is_bracket = first == PartialToken::LBracket;
cutoff = 3;
if (is_bracket && third != Some(PartialToken::RBracket))
|| (!is_bracket && third != Some(PartialToken::RCBracket))
{
return Err(ExprError::UnmatchedLRBrackets);
} else {
let mut token_array: Vec<Token> = Vec::new();
match second {
Some(PartialToken::Token(Token::String(ref s))) => {
let elements = s.split(",");
for e in elements {
let t = partial_tokens_to_tokens(&str_to_partial_tokens(e)?)?;
if t.is_empty() {
// do nothing
} else if t.len() == 1 {
token_array.push(t[0].clone())
} else {
return Err(format!("invalid token: {:?}", second)
.as_str()
.into());
}
}
}
_ => {
return Err(format!("invalid token: {:?}", second)
.as_str()
.into())
}
}
let result = token_array_to_token(token_array)?;
if is_bracket {
Some(result)
} else {
if let Token::IdentArray(vec) = result {
Some(Token::IdentMap(vec))
} else {
unreachable!()
}
}
}
}
PartialToken::Hat => match second {
Some(PartialToken::Hat) => Some(Token::Power),
_ => {
cutoff = 1;
Some(Token::BitXor)
}
},
_ => {
return Err(format!("invalid token: {:?}", first)
.as_str()
.into());
}
};
if let Some(token) = curr_token.clone() {
result.push(token);
recent_token = curr_token.clone();
}
tokens = &tokens[cutoff..];
}
Ok(result)
}