in rhai/src/tokenizer.rs [2462:2612]
fn next(&mut self) -> Option<Self::Item> {
let (within_interpolated, compress_script) = {
let control = &mut *self.state.tokenizer_control.borrow_mut();
if control.is_within_text {
// Switch to text mode terminated by back-tick
self.state.is_within_text_terminated_by = Some('`');
// Reset it
control.is_within_text = false;
}
(
self.state.is_within_text_terminated_by.is_some(),
control.compressed.is_some(),
)
};
let (token, pos) = match get_next_token(&mut self.stream, &mut self.state, &mut self.pos) {
// {EOF}
None => return None,
// {EOF} after unterminated string.
// The only case where `TokenizeState.is_within_text_terminated_by` is set is when
// a verbatim string or a string with continuation encounters {EOF}.
// This is necessary to handle such cases for line-by-line parsing, but for an entire
// script it is a syntax error.
Some((Token::StringConstant(..), pos)) if self.state.is_within_text_terminated_by.is_some() => {
self.state.is_within_text_terminated_by = None;
return Some((Token::LexError(LERR::UnterminatedString.into()), pos));
}
// Reserved keyword/symbol
Some((Token::Reserved(s), pos)) => (match
(s.as_str(),
#[cfg(not(feature = "no_custom_syntax"))]
self.engine.is_custom_keyword(&*s),
#[cfg(feature = "no_custom_syntax")]
false
)
{
("===", false) => Token::LexError(LERR::ImproperSymbol(s.to_string(),
"'===' is not a valid operator. This is not JavaScript! Should it be '=='?".to_string(),
).into()),
("!==", false) => Token::LexError(LERR::ImproperSymbol(s.to_string(),
"'!==' is not a valid operator. This is not JavaScript! Should it be '!='?".to_string(),
).into()),
("->", false) => Token::LexError(LERR::ImproperSymbol(s.to_string(),
"'->' is not a valid symbol. This is not C or C++!".to_string()).into()),
("<-", false) => Token::LexError(LERR::ImproperSymbol(s.to_string(),
"'<-' is not a valid symbol. This is not Go! Should it be '<='?".to_string(),
).into()),
(":=", false) => Token::LexError(LERR::ImproperSymbol(s.to_string(),
"':=' is not a valid assignment operator. This is not Go or Pascal! Should it be simply '='?".to_string(),
).into()),
(":;", false) => Token::LexError(LERR::ImproperSymbol(s.to_string(),
"':;' is not a valid symbol. Should it be '::'?".to_string(),
).into()),
("::<", false) => Token::LexError(LERR::ImproperSymbol(s.to_string(),
"'::<>' is not a valid symbol. This is not Rust! Should it be '::'?".to_string(),
).into()),
("(*" | "*)", false) => Token::LexError(LERR::ImproperSymbol(s.to_string(),
"'(* .. *)' is not a valid comment format. This is not Pascal! Should it be '/* .. */'?".to_string(),
).into()),
("# {", false) => Token::LexError(LERR::ImproperSymbol(s.to_string(),
"'#' is not a valid symbol. Should it be '#{'?".to_string(),
).into()),
// Reserved keyword/operator that is custom.
#[cfg(not(feature = "no_custom_syntax"))]
(.., true) => Token::Custom(s),
#[cfg(feature = "no_custom_syntax")]
(.., true) => unreachable!("no custom operators"),
// Reserved keyword that is not custom and disabled.
(token, false) if self.engine.is_symbol_disabled(token) => {
let msg = format!("reserved {} '{token}' is disabled", if is_valid_identifier(token) { "keyword"} else {"symbol"});
Token::LexError(LERR::ImproperSymbol(s.to_string(), msg).into())
},
// Reserved keyword/operator that is not custom.
(.., false) => Token::Reserved(s),
}, pos),
// Custom keyword
#[cfg(not(feature = "no_custom_syntax"))]
Some((Token::Identifier(s), pos)) if self.engine.is_custom_keyword(&*s) => {
(Token::Custom(s), pos)
}
// Custom keyword/symbol - must be disabled
#[cfg(not(feature = "no_custom_syntax"))]
Some((token, pos)) if token.is_literal() && self.engine.is_custom_keyword(token.literal_syntax()) => {
if self.engine.is_symbol_disabled(token.literal_syntax()) {
// Disabled standard keyword/symbol
(Token::Custom(Box::new(token.literal_syntax().into())), pos)
} else {
// Active standard keyword - should never be a custom keyword!
unreachable!("{:?} is an active keyword", token)
}
}
// Disabled symbol
Some((token, pos)) if token.is_literal() && self.engine.is_symbol_disabled(token.literal_syntax()) => {
(Token::Reserved(Box::new(token.literal_syntax().into())), pos)
}
// Normal symbol
Some(r) => r,
};
// Run the mapper, if any
let token = if let Some(func) = self.token_mapper {
func(token, pos, &self.state)
} else {
token
};
// Collect the compressed script, if needed
if compress_script {
let control = &mut *self.state.tokenizer_control.borrow_mut();
if let Some(ref mut compressed) = control.compressed {
if !matches!(token, Token::EOF) {
use std::fmt::Write;
let last_token = self.state.last_token.as_ref().unwrap();
let mut buf = SmartString::new_const();
if last_token.is_empty() {
write!(buf, "{token}").unwrap();
} else if within_interpolated
&& matches!(
token,
Token::StringConstant(..) | Token::InterpolatedString(..)
)
{
compressed.push_str(&last_token[1..]);
} else {
buf = last_token.clone();
}
if !buf.is_empty() && !compressed.is_empty() {
let cur = buf.chars().next().unwrap();
if cur == '_' || is_id_first_alphabetic(cur) || is_id_continue(cur) {
let prev = compressed.chars().last().unwrap();
if prev == '_' || is_id_first_alphabetic(prev) || is_id_continue(prev) {
compressed.push(' ');
}
}
}
compressed.push_str(&buf);
}
}
}
Some((token, pos))
}