optimum_benchmark/preprocessors/dataset_preprocessor.py [72:85]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
            truncation=scenario_config.truncation,
            max_length=max_length,
            padding=padding,
        )

    dataset = dataset.map(
        function=tokenize_function,
        desc="Running tokenizer on dataset",
        remove_columns=dataset.features,
        writer_batch_size=50,
        batched=True,
    ).with_format("torch")

    return dataset
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



optimum_benchmark/preprocessors/dataset_preprocessor.py [433:446]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
            truncation=scenario_config.truncation,
            max_length=max_length,
            padding=padding,
        )

    dataset = dataset.map(
        function=tokenize_function,
        desc="Running tokenizer on dataset",
        remove_columns=dataset.features,
        writer_batch_size=50,
        batched=True,
    ).with_format("torch")

    return dataset
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -