def tokenize_line()

in vizseq/_data/tokenizers.py [0:0]

• 13 lines of code
• 6 McCabe index (conditional complexity)

    def tokenize_line(cls, line: str, tokenization: VizSeqTokenization) -> str:
        if tokenization == VizSeqTokenization.none:
            return line
        elif tokenization == VizSeqTokenization.mteval_13a:
            return Tokenizer13a()(line)
        elif tokenization == VizSeqTokenization.mteval_v14_international:
            return TokenizerV14International()(line)
        elif tokenization == VizSeqTokenization.zh:
            return TokenizerZh()(line)
        elif tokenization == VizSeqTokenization.char:
            return _tokenize_by_char(line)
        else:
            raise ValueError(f'Unknown tokenization {tokenization.name}')