notebooks/src/code/data/mlm.py [69:82]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        batch = self.tokenizer(
            [example.word_texts for example in examples],
            is_split_into_words=True,
            return_attention_mask=True,
            padding=bool(self.pad_to_multiple_of),
            pad_to_multiple_of=self.pad_to_multiple_of,
            return_tensors=self.return_tensors,
        )
        # Map through the bounding boxes to the generated tokens:
        # We do this by augmenting the list of word bboxes to include the special token bboxes,
        # editing the word_ids mapping from tokens->words to match special tokens to their special
        # boxes (instead of None), and then applying this set of indexes to produce the token-wise
        # boxes including special tokens.
        bbox_tensors_by_example = []
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



notebooks/src/code/data/ner.py [80:93]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        batch = self.tokenizer(
            [example.word_texts for example in examples],
            is_split_into_words=True,
            return_attention_mask=True,
            padding=bool(self.pad_to_multiple_of),
            pad_to_multiple_of=self.pad_to_multiple_of,
            return_tensors=self.return_tensors,
        )
        # Map through the bounding boxes and word labels to the generated tokens:
        # We do this by augmenting the list of word bboxes/labels to include values for the special
        # tokens, editing the word_ids mapping from tokens->words to match special tokens to their
        # special values (instead of None), and then applying this set of indexes to produce the
        # token-wise boxes/labels including special tokens.
        bbox_tensors_by_example = []
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -