_base_ = ['configs/base.py'] # If point cloud range is changed, the models should also change their point # cloud range accordingly point_cloud_range = [-51.2, -51.2, -5.0, 51.2, 51.2, 3.0] voxel_size = [0.2, 0.2, 8] img_norm_cfg = dict( mean=[103.530, 116.280, 123.675], std=[1.0, 1.0, 1.0], to_rgb=False) # For nuScenes we usually do 10-class detection CLASSES = [ 'car', 'truck', 'construction_vehicle', 'bus', 'trailer', 'barrier', 'motorcycle', 'bicycle', 'pedestrian', 'traffic_cone' ] input_modality = dict( use_lidar=False, use_camera=True, use_radar=False, use_map=False, use_external=True) embed_dim = 256 pos_dim = embed_dim // 2 ffn_dim = embed_dim * 2 num_levels = 4 bev_h = 200 bev_w = 200 queue_length = 4 # each sequence contains `queue_length` frames. adapt_jit = False # set True when export jit trace model or blade model model = dict( type='BEVFormer', use_grid_mask=True, video_test_mode=True, img_backbone=dict( type='ResNet', depth=101, num_stages=4, out_indices=(2, 3, 4), frozen_stages=1, norm_cfg=dict(type='BN', requires_grad=False), norm_eval=True, style='caffe', dcn=dict(type='DCNv2', deform_groups=1, fallback_on_stride=False), stage_with_dcn=(False, False, True, True), zero_init_residual=True), img_neck=dict( type='FPN', in_channels=[512, 1024, 2048], out_channels=embed_dim, start_level=0, add_extra_convs='on_output', num_outs=num_levels, relu_before_extra_convs=True), pts_bbox_head=dict( type='BEVFormerHead', bev_h=bev_h, bev_w=bev_w, num_query=900, num_classes=10, in_channels=embed_dim, sync_cls_avg_factor=True, with_box_refine=True, as_two_stage=False, transformer=dict( type='PerceptionTransformer', rotate_prev_bev=True, use_shift=True, use_can_bus=True, embed_dims=embed_dim, encoder=dict( type='BEVFormerEncoder', num_layers=6, pc_range=point_cloud_range, num_points_in_pillar=4, return_intermediate=False, transformerlayers=dict( type='BEVFormerLayer', adapt_jit=adapt_jit, attn_cfgs=[ dict( type='TemporalSelfAttention', embed_dims=embed_dim, num_levels=1), dict( type='SpatialCrossAttention', pc_range=point_cloud_range, deformable_attention=dict( type='MSDeformableAttention3D', embed_dims=embed_dim, num_points=8, num_levels=num_levels, adapt_jit=adapt_jit), embed_dims=embed_dim, ) ], ffn_cfgs=dict( type='FFN', embed_dims=256, feedforward_channels=ffn_dim, num_fcs=2, ffn_drop=0.1, act_cfg=dict(type='ReLU', inplace=True), ), operation_order=('self_attn', 'norm', 'cross_attn', 'norm', 'ffn', 'norm'))), decoder=dict( type='Detr3DTransformerDecoder', num_layers=6, return_intermediate=True, transformerlayers=dict( type='DetrTransformerDecoderLayer', attn_cfgs=[ dict( type='MultiheadAttention', embed_dims=embed_dim, num_heads=8, dropout=0.1), dict( type='CustomMSDeformableAttention', embed_dims=embed_dim, num_levels=1, adapt_jit=adapt_jit), ], ffn_cfgs=dict( type='FFN', embed_dims=256, feedforward_channels=ffn_dim, num_fcs=2, ffn_drop=0.1, act_cfg=dict(type='ReLU', inplace=True), ), operation_order=('self_attn', 'norm', 'cross_attn', 'norm', 'ffn', 'norm')))), bbox_coder=dict( type='NMSFreeBBoxCoder', post_center_range=[-61.2, -61.2, -10.0, 61.2, 61.2, 10.0], pc_range=point_cloud_range, max_num=300, voxel_size=voxel_size, num_classes=10), positional_encoding=dict( type='LearnedPositionalEncoding', num_feats=pos_dim, row_num_embed=bev_h, col_num_embed=bev_w, ), loss_cls=dict( type='FocalLoss', use_sigmoid=True, gamma=2.0, alpha=0.25, loss_weight=2.0), loss_bbox=dict(type='L1Loss', loss_weight=0.25), loss_iou=dict(type='GIoULoss', loss_weight=0.0)), # model training and testing settings train_cfg=dict( pts=dict( grid_size=[512, 512, 1], voxel_size=voxel_size, point_cloud_range=point_cloud_range, out_size_factor=4, assigner=dict( type='HungarianBBoxAssigner3D', cls_cost=dict(type='FocalLossCost', weight=2.0), reg_cost=dict(type='BBox3DL1Cost', weight=0.25), iou_cost=dict( type='IoUCost', weight=0.0 ), # Fake cost. This is just to make it compatible with DETR head. pc_range=point_cloud_range)))) dataset_type = 'NuScenesDataset' data_root = 'data/nuScenes/nuscenes-v1.0/' train_pipeline = [ dict(type='PhotoMetricDistortionMultiViewImage'), dict(type='ObjectRangeFilter', point_cloud_range=point_cloud_range), dict(type='ObjectNameFilter', classes=CLASSES), dict(type='NormalizeMultiviewImage', **img_norm_cfg), dict(type='PadMultiViewImage', size_divisor=32), dict(type='DefaultFormatBundle3D', class_names=CLASSES), dict( type='Collect3D', keys=['gt_bboxes_3d', 'gt_labels_3d', 'img'], meta_keys=('filename', 'ori_shape', 'img_shape', 'lidar2img', 'depth2img', 'cam2img', 'pad_shape', 'scale_factor', 'flip', 'pcd_horizontal_flip', 'pcd_vertical_flip', 'box_mode_3d', 'box_type_3d', 'img_norm_cfg', 'pcd_trans', 'sample_idx', 'prev_idx', 'next_idx', 'pcd_scale_factor', 'pcd_rotation', 'pts_filename', 'transformation_3d_flow', 'scene_token', 'can_bus')) ] test_pipeline = [ dict(type='NormalizeMultiviewImage', **img_norm_cfg), dict(type='PadMultiViewImage', size_divisor=32), dict( type='MultiScaleFlipAug3D', img_scale=(1600, 900), pts_scale_ratio=1, flip=False, transforms=[ dict( type='DefaultFormatBundle3D', class_names=CLASSES, with_label=False), dict( type='Collect3D', keys=['img'], meta_keys=('filename', 'ori_shape', 'img_shape', 'lidar2img', 'depth2img', 'cam2img', 'pad_shape', 'scale_factor', 'flip', 'pcd_horizontal_flip', 'pcd_vertical_flip', 'box_mode_3d', 'box_type_3d', 'img_norm_cfg', 'pcd_trans', 'sample_idx', 'prev_idx', 'next_idx', 'pcd_scale_factor', 'pcd_rotation', 'pts_filename', 'transformation_3d_flow', 'scene_token', 'can_bus')) ]) ] data = dict( imgs_per_gpu=1, # 8gpus, total batch size=8 workers_per_gpu=4, pin_memory=True, # shuffler_sampler=dict(type='DistributedGroupSampler'), # nonshuffler_sampler=dict(type='DistributedSampler'), train=dict( type=dataset_type, data_source=dict( type='Det3dSourceNuScenes', data_root=data_root, ann_file=data_root + 'nuscenes_infos_temporal_train.pkl', pipeline=[ dict( type='LoadMultiViewImageFromFiles', to_float32=True, backend='turbojpeg'), dict( type='LoadAnnotations3D', with_bbox_3d=True, with_label_3d=True, with_attr_label=False) ], classes=CLASSES, modality=input_modality, test_mode=False, use_valid_flag=True, # we use box_type_3d='LiDAR' in kitti and nuscenes dataset # and box_type_3d='Depth' in sunrgbd and scannet dataset. box_type_3d='LiDAR'), pipeline=train_pipeline, queue_length=queue_length, ), val=dict( imgs_per_gpu=1, type=dataset_type, data_source=dict( type='Det3dSourceNuScenes', data_root=data_root, ann_file=data_root + 'nuscenes_infos_temporal_val.pkl', pipeline=[ dict( type='LoadMultiViewImageFromFiles', to_float32=True, backend='turbojpeg') ], classes=CLASSES, modality=input_modality, test_mode=True), pipeline=test_pipeline)) paramwise_cfg = {'img_backbone': dict(lr_mult=0.1)} optimizer = dict( type='AdamW', lr=2e-4, paramwise_options=paramwise_cfg, weight_decay=0.01) optimizer_config = dict(grad_clip=dict(max_norm=35, norm_type=2)) # learning policy lr_config = dict( policy='CosineAnnealing', warmup='linear', warmup_iters=500, warmup_ratio=1.0 / 3, min_lr_ratio=1e-3) total_epochs = 24 eval_config = dict(initial=False, interval=1, gpu_collect=False) eval_pipelines = [ dict( mode='test', data=data['val'], dist_eval=True, evaluators=[ dict( type='NuScenesEvaluator', classes=CLASSES, result_names=['pts_bbox']) ], ) ] load_from = 'https://github.com/zhiqi-li/storage/releases/download/v1.0/r101_dcn_fcos3d_pretrain.pth' log_config = dict( interval=50, hooks=[dict(type='TextLoggerHook'), dict(type='TensorboardLoggerHook')]) checkpoint_config = dict(interval=1) cudnn_benchmark = True export = dict( type='blade', blade_config=dict( enable_fp16=True, fp16_fallback_op_ratio=0.0, customize_op_black_list=[ 'aten::select', 'aten::index', 'aten::slice', 'aten::view', 'aten::upsample', 'aten::clamp' ]))