parsers/MovieReview/MovieReview_Finetune_Preprocess.py [283:318]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
        example_features.append(
            MovieReviewInputFeatures(
                unique_example_id=example_unique_id,
                unique_sentence_id=sentence_unique_id,
                tokens=tokens,
                annotations=annotations,
                input_ids=input_ids,
                input_mask=input_mask,
                input_type_ids=input_type_ids)
        )
        sentence_unique_id += 1

        dataset_features.append((example_target, example_features))

        example_no +=1
        print(f'Parsed example {example_no}')
    return dataset_features


def _process_examples(dataset):
    """Read a list of `InputExample`s from list."""
    examples = []
    unique_id = 0

    for doc in dataset:
        sample = doc['sample']
        target = doc['target']
        text_a = sample  # this is a list of sentences
        text_b = None  # we do not have pairs of sentences, we just need words embeddings for each document

        examples.append(
            MovieReviewInputExample(unique_id=unique_id, target=target, text_a=text_a, text_b=text_b))
        unique_id += 1
        # See convert_examples_to_features

    return examples
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



parsers/MovieReview/MovieReview_Preprocess.py [291:326]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
            example_features.append(
                MovieReviewInputFeatures(
                    unique_example_id=example_unique_id,
                    unique_sentence_id=sentence_unique_id,
                    tokens=tokens,
                    annotations=annotations,
                    input_ids=input_ids,
                    input_mask=input_mask,
                    input_type_ids=input_type_ids)
            )
            sentence_unique_id += 1

        dataset_features.append((example_target, example_features))

        example_no +=1
        print(f'Parsed example {example_no}')
    return dataset_features


def _process_examples(dataset):
    """Read a list of `InputExample`s from list."""
    examples = []
    unique_id = 0

    for doc in dataset:
        sample = doc['sample']
        target = doc['target']
        text_a = sample  # this is a list of sentences
        text_b = None  # we do not have pairs of sentences, we just need words embeddings for each document

        examples.append(
            MovieReviewInputExample(unique_id=unique_id, target=target, text_a=text_a, text_b=text_b))
        unique_id += 1
        # See convert_examples_to_features

    return examples
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -