func cleanUp()

in Sources/Tokenizers/Tokenizer.swift [356:371]

• 15 lines of code
• 1 McCabe index (conditional complexity)

    func cleanUp(text: String) -> String {
        guard cleanUpTokenizationSpaces else { return text }

        return
            text
                .replacingOccurrences(of: " .", with: ".")
                .replacingOccurrences(of: " ?", with: "?")
                .replacingOccurrences(of: " !", with: "!")
                .replacingOccurrences(of: " ,", with: ",")
                .replacingOccurrences(of: " ' ", with: "'")
                .replacingOccurrences(of: " n't", with: "n't")
                .replacingOccurrences(of: " 'm", with: "'m")
                .replacingOccurrences(of: " 's", with: "'s")
                .replacingOccurrences(of: " 've", with: "'ve")
                .replacingOccurrences(of: " 're", with: "'re")
    }