in tools/train.py [0:0]
def main():
args = parse_args()
if args.model_type is not None:
assert args.model_type in CONFIG_TEMPLATE_ZOO, 'model_type must be in [%s]' % (
', '.join(CONFIG_TEMPLATE_ZOO.keys()))
print('model_type=%s, config file will be replaced by %s' %
(args.model_type, CONFIG_TEMPLATE_ZOO[args.model_type]))
args.config = CONFIG_TEMPLATE_ZOO[args.model_type]
if args.config.startswith('http'):
r = requests.get(args.config)
# download config in current dir
tpath = args.config.split('/')[-1]
while not osp.exists(tpath):
try:
with open(tpath, 'wb') as code:
code.write(r.content)
except:
pass
args.config = tpath
# build cfg
if args.user_config_params is None:
cfg = mmcv_config_fromfile(args.config)
else:
cfg = pai_config_fromfile(args.config, args.user_config_params,
args.model_type)
# set multi-process settings
setup_multi_processes(cfg)
# set cudnn_benchmark
if cfg.get('cudnn_benchmark', False):
torch.backends.cudnn.benchmark = True
# update configs according to CLI args
if args.work_dir is not None:
cfg.work_dir = args.work_dir
if cfg.get('work_dir', None) is None:
cfg.work_dir = './work_dir'
# if `work_dir` is oss path, redirect `work_dir` to local path, add `oss_work_dir` point to oss path,
# and use osssync hook to upload log and ckpt in work_dir to oss_work_dir
if cfg.work_dir.startswith('oss://'):
cfg.oss_work_dir = cfg.work_dir
cfg.work_dir = osp.join('work_dirs',
cfg.work_dir.replace('oss://', ''))
else:
cfg.oss_work_dir = None
if args.resume_from is not None and len(args.resume_from) > 0:
cfg.resume_from = args.resume_from
if args.load_from is not None and len(args.load_from) > 0:
cfg.load_from = args.load_from
# dynamic adapt mmdet models
mmlab_utils.dynamic_adapt_for_mmlab(cfg)
cfg.gpus = args.gpus
# check memcached package exists
if importlib.util.find_spec('mc') is None:
traverse_replace(cfg, 'memcached', False)
# check oss_config and init oss io
if cfg.get('oss_io_config', None) is not None:
io.access_oss(**cfg.oss_io_config)
# init distributed env first, since logger depends on the dist info.
if not is_torchacc_enabled():
if args.launcher == 'none':
assert cfg.model.type not in \
['DeepCluster', 'MOCO', 'SimCLR', 'ODC', 'NPID'], \
'{} does not support non-dist training.'.format(cfg.model.type)
else:
if args.launcher == 'slurm':
cfg.dist_params['port'] = args.port
init_dist(args.launcher, **cfg.dist_params)
distributed = torch.cuda.is_available(
) and torch.distributed.is_initialized()
# create work_dir
if not io.exists(cfg.work_dir):
io.makedirs(cfg.work_dir)
# init the logger before other steps
timestamp = time.strftime('%Y%m%d_%H%M%S', time.localtime())
log_file = osp.join(cfg.work_dir, '{}.log'.format(timestamp))
logger = get_root_logger(log_file=log_file, log_level=cfg.log_level)
# init the meta dict to record some important information such as
# environment info and seed, which will be logged
meta = dict()
# log env info
env_info_dict = collect_env()
env_info = '\n'.join([('{}: {}'.format(k, v))
for k, v in env_info_dict.items()])
dash_line = '-' * 60 + '\n'
logger.info('Environment info:\n' + dash_line + env_info + '\n' +
dash_line)
meta['env_info'] = env_info
# log some basic info
logger.info('Distributed training: {}'.format(distributed))
logger.info('Config:\n{}'.format(cfg.text))
logger.info('Config Dict:\n{}'.format(json.dumps(cfg._cfg_dict)))
logger.info('GPU INFO : {}'.format(torch.cuda.get_device_name(0)))
# set random seeds
# Using different seeds for different ranks may reduce accuracy
seed = init_random_seed(args.seed, device=get_device())
seed = seed + dist.get_rank() if args.diff_seed else seed
if is_torchacc_enabled():
assert seed is not None, 'Must provide `seed` to sync model initializer if use torchacc!'
if seed is not None:
logger.info('Set random seed to {}, deterministic: {}'.format(
seed, args.deterministic))
set_random_seed(seed, deterministic=args.deterministic)
cfg.seed = seed
meta['seed'] = seed
if args.pretrained is not None:
assert isinstance(args.pretrained, str)
cfg.model.pretrained = args.pretrained
model = build_model(cfg.model)
if is_master():
print(model)
if 'stage' in cfg.model and cfg.model['stage'] == 'EDGE':
from easycv.utils.flops_counter import get_model_info
get_model_info(model, cfg.img_scale, cfg.model, logger)
assert len(cfg.workflow) == 1, 'Validation is called by hook.'
if cfg.checkpoint_config is not None:
# save easycv version, config file content and class names in
# checkpoints as meta data
cfg.checkpoint_config.meta = dict(
easycv_version=__version__, config=cfg.text)
# build dataloader
if not is_dali_dataset_type(cfg.data.train['type']):
shuffle = cfg.data.train.pop('shuffle', True)
print(f'data shuffle: {shuffle}')
# for odps data_source
if cfg.data.train.data_source.type == 'OdpsReader' and cfg.data.train.data_source.get(
'odps_io_config', None) is None:
cfg.data.train.data_source['odps_io_config'] = cfg.get(
'odps_io_config', None)
assert (
cfg.data.train.data_source.get('odps_io_config',
None) is not None
), 'odps config must be set in cfg file / cfg.data.train.data_source !!'
shuffle = False
if getattr(cfg.data, 'pin_memory', False):
mmlab_utils.fix_dc_pin_memory()
datasets = [build_dataset(cfg.data.train)]
data_loaders = [
build_dataloader(
ds,
cfg.data.imgs_per_gpu,
cfg.data.workers_per_gpu,
cfg.gpus,
dist=distributed,
shuffle=shuffle,
pin_memory=getattr(cfg.data, 'pin_memory', False),
replace=getattr(cfg.data, 'sampling_replace', False),
seed=cfg.seed,
# The default should be set to True, because sometimes the last batch is not sampled enough, causing an error in batchnorm
drop_last=getattr(cfg.data, 'drop_last', True),
reuse_worker_cache=cfg.data.get('reuse_worker_cache', False),
persistent_workers=cfg.data.get('persistent_workers', False),
collate_hooks=cfg.data.get('train_collate_hooks', []),
use_repeated_augment_sampler=cfg.data.get(
'use_repeated_augment_sampler', False)) for ds in datasets
]
else:
default_args = dict(
batch_size=cfg.data.imgs_per_gpu,
workers_per_gpu=cfg.data.workers_per_gpu,
distributed=distributed)
dataset = build_dataset(cfg.data.train, default_args)
data_loaders = [dataset.get_dataloader()]
# # add an attribute for visualization convenience
train_model(
model,
data_loaders,
cfg,
distributed=distributed,
timestamp=timestamp,
meta=meta,
validate=(not args.no_validate),
use_fp16=args.fp16)