in src/transformers/data/datasets/kinetics_datasets.py [0:0]
def build_dataloader(cfg):
# build dataset
train_dataset = VideoDataset(mode='train', clip_len=cfg.CONFIG.DATA.CLIP_LEN,
frame_sample_rate=cfg.CONFIG.DATA.FRAME_RATE,
video_dir=cfg.CONFIG.DATA.TRAIN_DATA_PATH,
label_dir=cfg.CONFIG.DATA.TRAIN_ANNO_PATH)
if not cfg.CONFIG.TEST.MULTI_VIEW_TEST:
# for validation during training
val_dataset = VideoDataset(mode='validation', clip_len=cfg.CONFIG.DATA.CLIP_LEN,
frame_sample_rate=cfg.CONFIG.DATA.FRAME_RATE,
video_dir=cfg.CONFIG.DATA.VAL_DATA_PATH,
label_dir=cfg.CONFIG.DATA.VAL_ANNO_PATH)
else:
# for multiview test
val_dataset = VideoDataset(mode='test', clip_len=cfg.CONFIG.DATA.CLIP_LEN,
frame_sample_rate=cfg.CONFIG.DATA.FRAME_RATE,
video_dir=cfg.CONFIG.DATA.VAL_DATA_PATH,
label_dir=cfg.CONFIG.DATA.VAL_ANNO_PATH)
# build data sampler
if cfg.DDP_CONFIG.DISTRIBUTED:
train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset)
val_sampler = torch.utils.data.distributed.DistributedSampler(val_dataset)
else:
train_sampler = None
val_sampler = None
# build data loader
train_loader = torch.utils.data.DataLoader(
train_dataset, batch_size=cfg.CONFIG.TRAIN.BATCH_SIZE, shuffle=train_sampler is None,
num_workers=9, sampler=train_sampler, pin_memory=True)
if not cfg.CONFIG.TEST.MULTI_VIEW_TEST:
# for validation during training
val_loader = torch.utils.data.DataLoader(
val_dataset, batch_size=cfg.CONFIG.VAL.BATCH_SIZE, shuffle=(val_sampler is None),
num_workers=9, sampler=val_sampler, pin_memory=True)
else:
# for multiview test
val_loader = torch.utils.data.DataLoader(
val_dataset, batch_size=cfg.CONFIG.TEST.BATCH_SIZE, shuffle=(val_sampler is None),
num_workers=9, sampler=val_sampler, pin_memory=True)
return train_loader, val_loader, train_sampler, val_sampler, None