fn decode_chain()

in tokenizers/src/decoders/wordpiece.rs [47:61]

• 15 lines of code
• 5 McCabe index (conditional complexity)

    fn decode_chain(&self, mut tokens: Vec<String>) -> Result<Vec<String>> {
        for (i, token) in tokens.iter_mut().enumerate() {
            if i != 0 {
                if let Some(tk) = token.strip_prefix(&self.prefix) {
                    *token = tk.to_string();
                } else {
                    *token = format!(" {token}");
                }
            }
            if self.cleanup {
                *token = cleanup(token);
            }
        }
        Ok(tokens)
    }