def __call__()

in src/datatrove/utils/japanese_tokenizer.py [0:0]

• 29 lines of code
• 11 McCabe index (conditional complexity)

    def __call__(self, text: str) -> Doc:
        # convert sudachipy.morpheme.Morpheme to DetailedToken and merge continuous spaces
        sudachipy_tokens = self.tokenizer.tokenize(text)
        dtokens = self._get_dtokens(sudachipy_tokens)
        dtokens, spaces = get_dtokens_and_spaces(dtokens, text)

        # create Doc with tag bi-gram based part-of-speech identification rules
        words, tags, inflections, lemmas, norms, readings, sub_tokens_list = zip(*dtokens) if dtokens else [[]] * 7
        sub_tokens_list = list(sub_tokens_list)
        doc = Doc(self.vocab, words=words, spaces=spaces)
        next_pos = None  # for bi-gram rules
        for idx, (token, dtoken) in enumerate(zip(doc, dtokens)):
            token.tag_ = dtoken.tag
            if next_pos:  # already identified in previous iteration
                token.pos = next_pos
                next_pos = None
            else:
                token.pos, next_pos = resolve_pos(
                    token.orth_,
                    dtoken.tag,
                    tags[idx + 1] if idx + 1 < len(tags) else None,
                )
            # if there's no lemma info (it's an unk) just use the surface
            token.lemma_ = dtoken.lemma if dtoken.lemma else dtoken.surface
            morph = {}
            if dtoken.inf:
                # it's normal for this to be empty for non-inflecting types
                morph["Inflection"] = dtoken.inf
            token.norm_ = dtoken.norm
            if dtoken.reading:
                # punctuation is its own reading, but we don't want values like
                # "=" here
                morph["Reading"] = re.sub("[=|]", "_", dtoken.reading)
            # ! Important to prevent the memory leak !
            # token.morph = MorphAnalysis(self.vocab, morph)
        if self.need_subtokens:
            doc.user_data["sub_tokens"] = sub_tokens_list
        return doc