tools/epic/handobj/train_net.py [247:411]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - verb_labels, noun_labels = labels[:, 0], labels[:, 1] # Compute the errors. ks = (1, 5) verb_num_topks_correct = metrics.topks_correct(verb_preds, verb_labels, ks) noun_num_topks_correct = metrics.topks_correct(noun_preds, noun_labels, ks) action_num_topks_correct = metrics.multitask_topks_correct( (verb_preds, noun_preds), (verb_labels, noun_labels), ks, ) verb_top1_err, verb_top5_err = [ (1.0 - x / verb_preds.size(0)) * 100.0 for x in verb_num_topks_correct ] noun_top1_err, noun_top5_err = [ (1.0 - x / noun_preds.size(0)) * 100.0 for x in noun_num_topks_correct ] top1_err, top5_err = [ (1.0 - x / verb_preds.size(0)) * 100.0 for x in action_num_topks_correct ] if cfg.NUM_GPUS > 1: top1_err, top5_err, verb_top1_err, verb_top5_err, noun_top1_err, noun_top5_err = du.all_reduce( [top1_err, top5_err, verb_top1_err, verb_top5_err, noun_top1_err, noun_top5_err] ) # Copy the errors from GPU to CPU (sync point). top1_err, top5_err, verb_top1_err, verb_top5_err, noun_top1_err, noun_top5_err= ( top1_err.item(), top5_err.item(), verb_top1_err.item(), verb_top5_err.item(), noun_top1_err.item(), noun_top5_err.item(), ) stats = { "verb_top1_err": verb_top1_err, "verb_top5_err": verb_top5_err, "noun_top1_err": noun_top1_err, "noun_top5_err": noun_top5_err, } val_meter.iter_toc() # Update and log stats. val_meter.update_stats( top1_err, top5_err, inputs[0].size(0) * cfg.NUM_GPUS, stats=stats ) # write to tensorboard format if available. if writer is not None: writer.add_scalars( {"Val/Top1_err": top1_err, "Val/Top5_err": top5_err}, global_step=len(val_loader) * cur_epoch + cur_iter, ) val_meter.update_predictions(preds, labels) val_meter.log_iter_stats(cur_epoch, cur_iter) val_meter.iter_tic() # Log epoch stats. is_best_epoch = val_meter.log_epoch_stats(cur_epoch) # write to tensorboard format if available. val_meter.reset() return is_best_epoch def calculate_and_update_precise_bn(loader, model, num_iters=200): """ Update the stats in bn layers by calculate the precise stats. Args: loader (loader): data loader to provide training data. model (model): model to update the bn stats. num_iters (int): number of iterations to compute and update the bn stats. """ def _gen_loader(): for inputs, _, _, _ in loader: if isinstance(inputs, (list,)): for i in range(len(inputs)): inputs[i] = inputs[i].cuda(non_blocking=True) else: inputs = inputs.cuda(non_blocking=True) yield inputs # Update the bn stats. update_bn_stats(model, _gen_loader(), num_iters) def build_trainer(cfg): """ Build training model and its associated tools, including optimizer, dataloaders and meters. Args: cfg (CfgNode): configs. Details can be found in slowfast/config/defaults.py Returns: model (nn.Module): training model. optimizer (Optimizer): optimizer. train_loader (DataLoader): training data loader. val_loader (DataLoader): validatoin data loader. precise_bn_loader (DataLoader): training data loader for computing precise BN. train_meter (TrainMeter): tool for measuring training stats. val_meter (ValMeter): tool for measuring validation stats. """ # Build the video model and print model statistics. model = build_model(cfg) if du.is_master_proc() and cfg.LOG_MODEL_INFO: misc.log_model_info(model, cfg, use_train_input=True) # Construct the optimizer. optimizer = optim.construct_optimizer(model, cfg) # Create the video train and val loaders. train_loader = loader.construct_loader(cfg, "train") val_loader = loader.construct_loader(cfg, "val") precise_bn_loader = loader.construct_loader( cfg, "train", is_precise_bn=True ) # Create meters. train_meter = TrainMeter(len(train_loader), cfg) val_meter = ValMeter(len(val_loader), cfg) return ( model, optimizer, train_loader, val_loader, precise_bn_loader, train_meter, val_meter, ) def train(cfg): """ Train a video model for many epochs on train set and evaluate it on val set. Args: cfg (CfgNode): configs. Details can be found in slowfast/config/defaults.py """ # Set up environment. du.init_distributed_training(cfg) # Set random seed from configs. np.random.seed(cfg.RNG_SEED) torch.manual_seed(cfg.RNG_SEED) # Setup logging format. logging.setup_logging(cfg.OUTPUT_DIR) # Init multigrid. multigrid = None if cfg.MULTIGRID.LONG_CYCLE or cfg.MULTIGRID.SHORT_CYCLE: multigrid = MultigridSchedule() cfg = multigrid.init_multigrid(cfg) if cfg.MULTIGRID.LONG_CYCLE: cfg, _ = multigrid.update_long_cycle(cfg, cur_epoch=0) # Print config. logger.info("Train with config:") logger.info(pprint.pformat(cfg)) # Build the video model and print model statistics. model = build_model(cfg) if du.is_master_proc() and cfg.LOG_MODEL_INFO: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - tools/epic/train_net.py [208:372]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - verb_labels, noun_labels = labels[:, 0], labels[:, 1] # Compute the errors. ks = (1, 5) verb_num_topks_correct = metrics.topks_correct(verb_preds, verb_labels, ks) noun_num_topks_correct = metrics.topks_correct(noun_preds, noun_labels, ks) action_num_topks_correct = metrics.multitask_topks_correct( (verb_preds, noun_preds), (verb_labels, noun_labels), ks, ) verb_top1_err, verb_top5_err = [ (1.0 - x / verb_preds.size(0)) * 100.0 for x in verb_num_topks_correct ] noun_top1_err, noun_top5_err = [ (1.0 - x / noun_preds.size(0)) * 100.0 for x in noun_num_topks_correct ] top1_err, top5_err = [ (1.0 - x / verb_preds.size(0)) * 100.0 for x in action_num_topks_correct ] if cfg.NUM_GPUS > 1: top1_err, top5_err, verb_top1_err, verb_top5_err, noun_top1_err, noun_top5_err = du.all_reduce( [top1_err, top5_err, verb_top1_err, verb_top5_err, noun_top1_err, noun_top5_err] ) # Copy the errors from GPU to CPU (sync point). top1_err, top5_err, verb_top1_err, verb_top5_err, noun_top1_err, noun_top5_err= ( top1_err.item(), top5_err.item(), verb_top1_err.item(), verb_top5_err.item(), noun_top1_err.item(), noun_top5_err.item(), ) stats = { "verb_top1_err": verb_top1_err, "verb_top5_err": verb_top5_err, "noun_top1_err": noun_top1_err, "noun_top5_err": noun_top5_err, } val_meter.iter_toc() # Update and log stats. val_meter.update_stats( top1_err, top5_err, inputs[0].size(0) * cfg.NUM_GPUS, stats=stats ) # write to tensorboard format if available. if writer is not None: writer.add_scalars( {"Val/Top1_err": top1_err, "Val/Top5_err": top5_err}, global_step=len(val_loader) * cur_epoch + cur_iter, ) val_meter.update_predictions(preds, labels) val_meter.log_iter_stats(cur_epoch, cur_iter) val_meter.iter_tic() # Log epoch stats. is_best_epoch = val_meter.log_epoch_stats(cur_epoch) # write to tensorboard format if available. val_meter.reset() return is_best_epoch def calculate_and_update_precise_bn(loader, model, num_iters=200): """ Update the stats in bn layers by calculate the precise stats. Args: loader (loader): data loader to provide training data. model (model): model to update the bn stats. num_iters (int): number of iterations to compute and update the bn stats. """ def _gen_loader(): for inputs, _, _, _ in loader: if isinstance(inputs, (list,)): for i in range(len(inputs)): inputs[i] = inputs[i].cuda(non_blocking=True) else: inputs = inputs.cuda(non_blocking=True) yield inputs # Update the bn stats. update_bn_stats(model, _gen_loader(), num_iters) def build_trainer(cfg): """ Build training model and its associated tools, including optimizer, dataloaders and meters. Args: cfg (CfgNode): configs. Details can be found in slowfast/config/defaults.py Returns: model (nn.Module): training model. optimizer (Optimizer): optimizer. train_loader (DataLoader): training data loader. val_loader (DataLoader): validatoin data loader. precise_bn_loader (DataLoader): training data loader for computing precise BN. train_meter (TrainMeter): tool for measuring training stats. val_meter (ValMeter): tool for measuring validation stats. """ # Build the video model and print model statistics. model = build_model(cfg) if du.is_master_proc() and cfg.LOG_MODEL_INFO: misc.log_model_info(model, cfg, use_train_input=True) # Construct the optimizer. optimizer = optim.construct_optimizer(model, cfg) # Create the video train and val loaders. train_loader = loader.construct_loader(cfg, "train") val_loader = loader.construct_loader(cfg, "val") precise_bn_loader = loader.construct_loader( cfg, "train", is_precise_bn=True ) # Create meters. train_meter = TrainMeter(len(train_loader), cfg) val_meter = ValMeter(len(val_loader), cfg) return ( model, optimizer, train_loader, val_loader, precise_bn_loader, train_meter, val_meter, ) def train(cfg): """ Train a video model for many epochs on train set and evaluate it on val set. Args: cfg (CfgNode): configs. Details can be found in slowfast/config/defaults.py """ # Set up environment. du.init_distributed_training(cfg) # Set random seed from configs. np.random.seed(cfg.RNG_SEED) torch.manual_seed(cfg.RNG_SEED) # Setup logging format. logging.setup_logging(cfg.OUTPUT_DIR) # Init multigrid. multigrid = None if cfg.MULTIGRID.LONG_CYCLE or cfg.MULTIGRID.SHORT_CYCLE: multigrid = MultigridSchedule() cfg = multigrid.init_multigrid(cfg) if cfg.MULTIGRID.LONG_CYCLE: cfg, _ = multigrid.update_long_cycle(cfg, cur_epoch=0) # Print config. logger.info("Train with config:") logger.info(pprint.pformat(cfg)) # Build the video model and print model statistics. model = build_model(cfg) if du.is_master_proc() and cfg.LOG_MODEL_INFO: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -