def build_data_pipeline()

in src/chug/wds/pipeline.py [0:0]

• 15 lines of code
• 1 McCabe index (conditional complexity)

def build_data_pipeline(
        shards: ShardSpec,
        task_pipeline: List[Callable],
        is_training: bool = False,
        batch_size: int = 0,
        resampled: bool = False,
        multi_interval: bool = False,
        seed: int = 0,
        shared_interval_count: int = -1,
        num_batches_per_worker: int = 0,
        sample_shuffle_initial: int = 1,
        sample_shuffle_size: int = 1,
        collate_fn: Optional[Callable] = None,
        batched_task_pipeline: Optional[List[Callable]] = None,
        handler=wds.reraise_exception,