dataset_utils/data_iterator.py [86:145]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
):
    TEXT = data.Field(
        fix_length=max_length,
        eos_token="</s>",
        pad_token="<pad>",
        init_token="<s>",
        unk_token="<unk>",
        include_lengths=True,
    )
    label = data.LabelField()
    train_dialog = data.TabularDataset(
        path=os.path.join(data_folder, dataset, prefix + dataset + "_train.csv"),
        format="csv",
        fields=[
            ("context_id", None),
            ("Sentence1", TEXT),
            ("Sentence2", TEXT),
            ("Label", label),
        ],
    )
    valid_dialog = data.TabularDataset(
        path=os.path.join(data_folder, dataset, prefix + dataset + "_dev.csv"),
        format="csv",
        fields=[
            ("context_id", None),
            ("Sentence1", TEXT),
            ("Sentence2", TEXT),
            ("Label", label),
        ],
    )
    test_dialog = data.TabularDataset(
        path=os.path.join(data_folder, dataset, prefix + dataset + "_test.csv"),
        format="csv",
        fields=[
            ("context_id", None),
            ("Sentence1", TEXT),
            ("Sentence2", TEXT),
            ("Label", label),
        ],
    )

    TEXT.build_vocab(train_dialog, min_freq=3)
    label.build_vocab(train_dialog)

    train_dialog_iter, valid_dialog_iter, test_dialog_iter = BucketIterator.splits(
        (train_dialog, valid_dialog, test_dialog),
        batch_size=batch_size,
        sort_key=lambda x: len(x.Label),
        sort_within_batch=False,
        device=device,
    )

    # return train_dialog_iter, valid_dialog_iter, test_dialog_iter, TEXT.vocab.stoi['<pad>'], len(TEXT.vocab), TEXT.vocab.itos#TEXT.vocab
    return (
        train_dialog_iter,
        valid_dialog_iter,
        test_dialog_iter,
        TEXT.vocab,
        label.vocab,
    )
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



dataset_utils/data_iterator.py [222:281]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
):
    TEXT = data.Field(
        fix_length=max_length,
        eos_token="</s>",
        pad_token="<pad>",
        init_token="<s>",
        unk_token="<unk>",
        include_lengths=True,
    )
    label = data.LabelField()
    train_dialog = data.TabularDataset(
        path=os.path.join(data_folder, dataset, prefix + dataset + "_train.csv"),
        format="csv",
        fields=[
            ("context_id", None),
            ("Sentence1", TEXT),
            ("Sentence2", TEXT),
            ("Label", label),
        ],
    )
    valid_dialog = data.TabularDataset(
        path=os.path.join(data_folder, dataset, prefix + dataset + "_dev.csv"),
        format="csv",
        fields=[
            ("context_id", None),
            ("Sentence1", TEXT),
            ("Sentence2", TEXT),
            ("Label", label),
        ],
    )
    test_dialog = data.TabularDataset(
        path=os.path.join(data_folder, dataset, prefix + dataset + "_test.csv"),
        format="csv",
        fields=[
            ("context_id", None),
            ("Sentence1", TEXT),
            ("Sentence2", TEXT),
            ("Label", label),
        ],
    )

    TEXT.build_vocab(train_dialog, min_freq=3)
    label.build_vocab(train_dialog)

    train_dialog_iter, valid_dialog_iter, test_dialog_iter = BucketIterator.splits(
        (train_dialog, valid_dialog, test_dialog),
        batch_size=batch_size,
        sort_key=lambda x: len(x.Label),
        sort_within_batch=False,
        device=device,
    )

    # return train_dialog_iter, valid_dialog_iter, test_dialog_iter, TEXT.vocab.stoi['<pad>'], len(TEXT.vocab), TEXT.vocab.itos#TEXT.vocab
    return (
        train_dialog_iter,
        valid_dialog_iter,
        test_dialog_iter,
        TEXT.vocab,
        label.vocab,
    )
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -