in smt2parser/src/lexer.rs [208:379]
fn next(&mut self) -> Option<Token> {
while self.skip_whitespace() || self.skip_comment() {}
match self.peek_byte() {
// Parentheses
Some(b'(') => {
self.consume_byte();
Some(Token::LeftParen)
}
Some(b')') => {
self.consume_byte();
Some(Token::RightParen)
}
// Quoted symbols
Some(b'|') => {
self.consume_byte();
let mut content = Vec::new();
while let Some(c) = self.read_byte() {
if c == b'|' {
return match String::from_utf8(content) {
Ok(s) => Some(Token::Symbol(s)),
Err(_) => None,
};
}
content.push(c);
}
// Do not accept EOI as a terminator.
None
}
// String literals
Some(b'"') => {
self.consume_byte();
let mut content = Vec::new();
while let Some(c) = self.read_byte() {
if c == b'"' {
if let Some(d) = self.peek_byte() {
if *d == b'"' {
self.consume_byte();
content.push(b'"');
continue;
}
}
return match String::from_utf8(content) {
Ok(s) => Some(Token::String(s)),
Err(_) => None,
};
}
content.push(c);
}
// Do not accept EOI as a terminator.
None
}
// Binary and Hexadecimal literals
Some(b'#') => {
self.consume_byte();
match self.peek_byte() {
Some(b'b') => {
self.consume_byte();
let mut content = Vec::new();
while let Some(c) = self.peek_byte() {
match c {
b'0' => content.push(false),
b'1' => content.push(true),
_ => break,
}
self.consume_byte();
}
if content.is_empty() {
None
} else {
Some(Token::Binary(content))
}
}
Some(b'x') => {
self.consume_byte();
let mut content = Vec::new();
while let Some(c) = self.peek_byte() {
match c {
b'0'..=b'9' => content.push(c - b'0'),
b'a'..=b'f' => content.push(c - b'a' + 10),
b'A'..=b'F' => content.push(c - b'A' + 10),
_ => break,
}
self.consume_byte();
}
if content.is_empty() {
None
} else {
Some(Token::Hexadecimal(content))
}
}
_ => None,
}
}
// Number literals
Some(digit @ b'0'..=b'9') => {
let mut numerator = vec![*digit];
self.consume_byte();
while let Some(c) = self.peek_byte() {
if !is_digit_byte(*c) {
break;
}
numerator.push(*c);
self.consume_byte();
}
if numerator.len() > 1 && numerator.starts_with(b"0") {
return None;
}
let numerator = String::from_utf8(numerator).unwrap();
match self.peek_byte() {
Some(b'.') => {
self.consume_byte();
let mut denumerator = Vec::new();
while let Some(c) = self.peek_byte() {
if !is_digit_byte(*c) {
break;
}
denumerator.push(*c);
self.consume_byte();
}
if denumerator.is_empty() {
return None;
}
let denumerator = String::from_utf8(denumerator).unwrap();
let num =
num::BigInt::from_str_radix(&(numerator + &denumerator), 10).ok()?;
let denom = num::BigInt::from(10u32).pow(denumerator.len() as u32);
Some(Token::Decimal(Decimal::new(num, denom)))
}
_ => Some(Token::Numeral(
Numeral::from_str_radix(&numerator, 10).ok()?,
)),
}
}
// Keywords
Some(b':') => {
self.consume_byte();
let mut content = Vec::new();
while let Some(c) = self.peek_byte() {
if !is_symbol_byte(*c) {
break;
}
content.push(*c);
self.consume_byte();
}
match String::from_utf8(content) {
Ok(s) => Some(Token::Keyword(s)),
Err(_) => None,
}
}
// Symbols (including `_` and `!`)
Some(c) if is_non_digit_symbol_byte(*c) => {
let mut content = vec![*c];
self.consume_byte();
while let Some(c) = self.peek_byte() {
if !is_symbol_byte(*c) {
break;
}
content.push(*c);
self.consume_byte();
}
match self.lookup_symbol(&content) {
Some(token) => Some(token.clone()),
None => match String::from_utf8(content) {
Ok(s) => Some(Token::Symbol(s)),
Err(_) => None,
},
}
}
// EOI or Error
_ => None,
}
}