in tokenizers/src/tokenizer/normalizer.rs [242:268]
fn validate_range<T: RangeBounds<usize> + Clone>(
&self,
range: Range<T>,
) -> Option<Range<std::ops::Range<usize>>> {
match range {
Range::Original(_) => {
let r = range.into_full_range(self.original.len());
if !(self.original.is_char_boundary(r.start)
&& self.original.is_char_boundary(r.end))
{
None
} else {
Some(Range::Original(r))
}
}
Range::Normalized(_) => {
let r = range.into_full_range(self.normalized.len());
if !(self.normalized.is_char_boundary(r.start)
&& self.normalized.is_char_boundary(r.end))
{
None
} else {
Some(Range::Normalized(r))
}
}
}
}