parsers/MovieReview/MovieReview_Finetune_Preprocess.py [217:257]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        annotate_as_neg = False  # Needed to associate an annotation to each token
        annotate_as_pos = False  # Needed to associate an annotation to each token

        input_type_ids = []
        annotations = []
        tokens = []

        tokens.append("[CLS]")
        input_type_ids.append(0)
        annotations.append(0)

        for token in parsed_example:
            if token == '<neg>':
                # print(f'found {token}!')
                assert not annotate_as_pos
                annotate_as_neg = True
            elif token == '<pos>':
                # print(f'found {token}!')
                assert not annotate_as_neg
                annotate_as_pos = True
            elif token == '</neg>':
                # print(f'found {token}!')
                assert annotate_as_neg
                assert not annotate_as_pos
                annotate_as_neg = False
            elif token == '</pos>':
                # print(f'found {token}!')
                assert annotate_as_pos, sentence
                assert not annotate_as_neg
                annotate_as_pos = False
            else:
                if annotate_as_neg or annotate_as_pos:
                    annotations.append(1)
                else:
                    annotations.append(0)
                tokens.append(token)
                input_type_ids.append(0)

        tokens.append("[SEP]")
        input_type_ids.append(0)
        annotations.append(0)
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



parsers/Spouse/Spouse_Finetune_Preprocess.py [188:228]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        annotate_as_neg = False  # Needed to associate an annotation to each token
        annotate_as_pos = False  # Needed to associate an annotation to each token

        input_type_ids = []
        annotations = []
        tokens = []

        tokens.append("[CLS]")
        input_type_ids.append(0)
        annotations.append(0)

        for token in parsed_example:
            if token == '<neg>':
                #print(f'found {token}!')
                assert not annotate_as_pos
                annotate_as_neg = True
            elif token == '<pos>':
                #print(f'found {token}!')
                assert not annotate_as_neg
                annotate_as_pos = True
            elif token == '</neg>':
                #print(f'found {token}!')
                assert annotate_as_neg
                assert not annotate_as_pos
                annotate_as_neg = False
            elif token == '</pos>':
                #print(f'found {token}!')
                assert annotate_as_pos, sentence
                assert not annotate_as_neg
                annotate_as_pos = False
            else:
                if annotate_as_neg or annotate_as_pos:
                    annotations.append(1)
                else:
                    annotations.append(0)
                tokens.append(token)
                input_type_ids.append(0)

        tokens.append("[SEP]")
        input_type_ids.append(0)
        annotations.append(0)
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -