eft/apps/evalfrompkl.py [410:635]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - glViewer.show() # Absolute error (MPJPE) error = torch.sqrt(((pred_keypoints_3d - gt_keypoints_3d) ** 2).sum(dim=-1)).mean(dim=-1).cpu().numpy() error_upper = torch.sqrt(((pred_keypoints_3d - gt_keypoints_3d) ** 2).sum(dim=-1)).mean(dim=-1).cpu().numpy() # mpjpe[step * batch_size:step * batch_size + curr_batch_size] = error # Reconstuction_error r_error = reconstruction_error(pred_keypoints_3d.cpu().numpy(), gt_keypoints_3d.cpu().numpy(), reduction=None) r_error_upper = reconstruction_error(pred_keypoints_3d.cpu().numpy(), gt_keypoints_3d.cpu().numpy(), reduction=None) # recon_err[step * batch_size:step * batch_size + curr_batch_size] = r_error for ii, p in enumerate(batch['imgname'][:len(r_error)]): seqName = os.path.basename( os.path.dirname(p)) # quant_mpjpe[step * batch_size:step * batch_size + curr_batch_size] = error if seqName not in quant_mpjpe.keys(): quant_mpjpe[seqName] = [] quant_recon_err[seqName] = [] quant_mpjpe[seqName].append(error[ii]) quant_recon_err[seqName].append(r_error[ii]) # Reconstuction_error # quant_recon_err[step * batch_size:step * batch_size + curr_batch_size] = r_error list_mpjpe = np.hstack([ quant_mpjpe[k] for k in quant_mpjpe]) list_reconError = np.hstack([ quant_recon_err[k] for k in quant_recon_err]) if bVerbose: print(">>> {} : MPJPE {:.02f} mm, error: {:.02f} mm | Total MPJPE {:.02f} mm, error {:.02f} mm".format(seqName, np.mean(error)*1000, np.mean(r_error)*1000, np.hstack(list_mpjpe).mean()*1000, np.hstack(list_reconError).mean()*1000) ) # print("MPJPE {}, error: {}".format(np.mean(error)*100, np.mean(r_error)*100)) # If mask or part evaluation, render the mask and part images # if eval_masks or eval_parts: # mask, parts = renderer(pred_vertices, pred_camera) # Mask evaluation (for LSP) if eval_masks: center = batch['center'].cpu().numpy() scale = batch['scale'].cpu().numpy() # Dimensions of original image orig_shape = batch['orig_shape'].cpu().numpy() for i in range(curr_batch_size): # After rendering, convert imate back to original resolution pred_mask = uncrop(mask[i].cpu().numpy(), center[i], scale[i], orig_shape[i]) > 0 # Load gt mask gt_mask = cv2.imread(os.path.join(annot_path, batch['maskname'][i]), 0) > 0 # Evaluation consistent with the original UP-3D code accuracy += (gt_mask == pred_mask).sum() pixel_count += np.prod(np.array(gt_mask.shape)) for c in range(2): cgt = gt_mask == c cpred = pred_mask == c tp[c] += (cgt & cpred).sum() fp[c] += (~cgt & cpred).sum() fn[c] += (cgt & ~cpred).sum() f1 = 2 * tp / (2 * tp + fp + fn) # Part evaluation (for LSP) if eval_parts: center = batch['center'].cpu().numpy() scale = batch['scale'].cpu().numpy() orig_shape = batch['orig_shape'].cpu().numpy() for i in range(curr_batch_size): pred_parts = uncrop(parts[i].cpu().numpy().astype(np.uint8), center[i], scale[i], orig_shape[i]) # Load gt part segmentation gt_parts = cv2.imread(os.path.join(annot_path, batch['partname'][i]), 0) # Evaluation consistent with the original UP-3D code # 6 parts + background for c in range(7): cgt = gt_parts == c cpred = pred_parts == c cpred[gt_parts == 255] = 0 parts_tp[c] += (cgt & cpred).sum() parts_fp[c] += (~cgt & cpred).sum() parts_fn[c] += (cgt & ~cpred).sum() gt_parts[gt_parts == 255] = 0 pred_parts[pred_parts == 255] = 0 parts_f1 = 2 * parts_tp / (2 * parts_tp + parts_fp + parts_fn) parts_accuracy += (gt_parts == pred_parts).sum() parts_pixel_count += np.prod(np.array(gt_parts.shape)) # Print intermediate results during evaluation if bVerbose: if step % log_freq == log_freq - 1: if eval_pose: print('MPJPE: ' + str(1000 * mpjpe[:step * batch_size].mean())) print('Reconstruction Error: ' + str(1000 * recon_err[:step * batch_size].mean())) print() if eval_masks: print('Accuracy: ', accuracy / pixel_count) print('F1: ', f1.mean()) print() if eval_parts: print('Parts Accuracy: ', parts_accuracy / parts_pixel_count) print('Parts F1 (BG): ', parts_f1[[0,1,2,3,4,5,6]].mean()) print() # if step==3: #Debug # break # Save reconstructions to a file for further processing if save_results: np.savez(result_file, pred_joints=pred_joints, pose=smpl_pose, betas=smpl_betas, camera=smpl_camera) # Print final results during evaluation if bVerbose: print('*** Final Results ***') print() if eval_pose: # if bVerbose: # print('MPJPE: ' + str(1000 * mpjpe.mean())) # print('Reconstruction Error: ' + str(1000 * recon_err.mean())) # print() list_mpjpe = np.hstack([ quant_mpjpe[k] for k in quant_mpjpe]) list_reconError = np.hstack([ quant_recon_err[k] for k in quant_recon_err]) output_str ='SeqNames; ' for seq in quant_mpjpe: output_str += seq + ';' output_str +='\n MPJPE; ' quant_mpjpe_avg_mm = np.hstack(list_mpjpe).mean()*1000 output_str += "Avg {:.02f} mm; ".format( quant_mpjpe_avg_mm) for seq in quant_mpjpe: output_str += '{:.02f}; '.format(1000 * np.hstack(quant_mpjpe[seq]).mean()) output_str +='\n Recon Error; ' quant_recon_error_avg_mm = np.hstack(list_reconError).mean()*1000 output_str +="Avg {:.02f}mm; ".format( quant_recon_error_avg_mm ) for seq in quant_recon_err: output_str += '{:.02f}; '.format(1000 * np.hstack(quant_recon_err[seq]).mean()) if bVerbose: print(output_str) else: print(">>> Test on 3DPW: MPJPE: {} | quant_recon_error_avg_mm: {}".format(quant_mpjpe_avg_mm, quant_recon_error_avg_mm) ) #Export log to json if os.path.exists("/private/home/hjoo/codes/fairmocap/benchmarks_eval"): targetFile = "evalFromPkl_" + os.path.basename(pklDir) + ".txt" targetFile = os.path.join('/private/home/hjoo/codes/fairmocap/benchmarks_eval',targetFile) print(f"\n Writing output to:{targetFile}") text_file = open(targetFile, "w") text_file.write(output_str) text_file.write("\n Input Pkl Dir:{}".format(pklDir)) text_file.close() return quant_mpjpe_avg_mm, quant_recon_error_avg_mm if bVerbose: if eval_masks: print('Accuracy: ', accuracy / pixel_count) print('F1: ', f1.mean()) print() if eval_parts: print('Parts Accuracy: ', parts_accuracy / parts_pixel_count) print('Parts F1 (BG): ', parts_f1[[0,1,2,3,4,5,6]].mean()) print() return -1 #Should return something if __name__ == '__main__': if len(sys.argv)>1: args = parser.parse_args() else: params =['--checkpoint','/home/hjoo/Dropbox (Facebook)/spinouput/2019-10-29-00:48-out/test1/checkpoints/2019_10_29-01_11_29.pt','--dataset','3dpw', '--log_freq','20'] params =['--checkpoint','/home/hjoo/Dropbox (Facebook)/spinouput/10-30-40560-coco3d_first/checkpoints/2019_10_30-13_57_53.pt','--dataset','h36m-p1', '--log_freq','20'] # params =['--checkpoint','data/model_checkpoint.pt','--dataset','h36m-p1', '--log_freq','20','--num_workers',0] params =['--checkpoint','/home/hjoo/Dropbox (Facebook)/spinouput/10-31-5896-ours_coco3d_all/checkpoints/2019_10_31-11_37_20.pt'] #wCOCO3D only early first try params =['--checkpoint','/home/hjoo/Dropbox (Facebook)/spinouput/10-31-50173-spin_all/checkpoints/2019_11_01-22_18_03.pt'] #wCOCO3D only early first try params =['--checkpoint','logs/11-07-55557-w_upper0_2_spin_all-5372/checkpoints/2019_11_07-21_13_54-best-58.088939636945724.pt'] params =['--checkpoint','/home/hjoo/Dropbox (Facebook)/spinouput/11-14-84106-fromInit_coco3d_cocoplus3d-2878/checkpoints/2019_11_15-23_18_29-best-61.0622800886631.pt'] #Ours 3D (no Aug!) #Semi-final params =['--checkpoint','logs/11-13-78679-bab_spin_mlc3d_fter60_ag-9589/checkpoints/2019_11_14-02_14_28-best-55.79321086406708.pt'] #Ours 3D + augmentation params =['--checkpoint','logs/11-13-78681-bab_spin_mlc3d_fter65-1249/checkpoints/2019_11_14-05_36_41-best-56.23840540647507.pt'] #Ours 3D .... this is also with Aug!! params =['--checkpoint','spinmodel_shared/11-13-78679-bab_spin_mlc3d_fter60_ag-9589/checkpoints/2019_11_14-02_14_28-best-55.79321086406708.pt'] #Ours 3D + augmentation params =['--checkpoint','logs/11-13-78679-bab_spin_mlc3d_fter60-7183/checkpoints/2019_11_14-08_12_35-best-56.12510070204735.pt'] #Ours 3D (no Aug!) params =['--checkpoint','data/model_checkpoint.pt'] #Original params +=['--dataset','3dpw-vibe'] # params +=['--num_workers',0] args = parser.parse_args(params) args.batch_size =64 args.num_workers =4 model = hmr(config.SMPL_MEAN_PARAMS) checkpoint = torch.load(args.checkpoint) model.load_state_dict(checkpoint['model'], strict=False) model.cuda() model.eval() # # Setup evaluation dataset # # dataset = BaseDataset(None, '3dpw', is_train=False, bMiniTest=False) # dataset = BaseDataset(None, '3dpw', is_train=False, bMiniTest=False, bEnforceUpperOnly=False) # # dataset = BaseDataset(None, '3dpw-crop', is_train=False, bMiniTest=False, bEnforceUpperOnly=True) # # # Run evaluation # run_evaluation(model, '3dpw',dataset , args.result_file, # batch_size=args.batch_size, # shuffle=args.shuffle, # log_freq=args.log_freq, num_workers=args.num_workers) # # Setup evaluation dataset # dataset = BaseDataset(None, 'h36m-p1', is_train=False, bMiniTest=False, bEnforceUpperOnly=False) # # Run evaluation # run_evaluation(model, 'h36m-p1', dataset, args.result_file, # batch_size=args.batch_size, # shuffle=args.shuffle, # log_freq=args.log_freq, num_workers=args.num_workers) # Setup evaluation dataset dataset = BaseDataset(None, args.dataset, is_train=False, bMiniTest=False, bEnforceUpperOnly=False) # Run evaluation run_evaluation(model, args.dataset, dataset, args.result_file, batch_size=args.batch_size, shuffle=args.shuffle, log_freq=args.log_freq, num_workers=args.num_workers) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - eft/apps/evalfrompkl_frankmocap.py [458:683]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - glViewer.show() # Absolute error (MPJPE) error = torch.sqrt(((pred_keypoints_3d - gt_keypoints_3d) ** 2).sum(dim=-1)).mean(dim=-1).cpu().numpy() error_upper = torch.sqrt(((pred_keypoints_3d - gt_keypoints_3d) ** 2).sum(dim=-1)).mean(dim=-1).cpu().numpy() # mpjpe[step * batch_size:step * batch_size + curr_batch_size] = error # Reconstuction_error r_error = reconstruction_error(pred_keypoints_3d.cpu().numpy(), gt_keypoints_3d.cpu().numpy(), reduction=None) r_error_upper = reconstruction_error(pred_keypoints_3d.cpu().numpy(), gt_keypoints_3d.cpu().numpy(), reduction=None) # recon_err[step * batch_size:step * batch_size + curr_batch_size] = r_error for ii, p in enumerate(batch['imgname'][:len(r_error)]): seqName = os.path.basename( os.path.dirname(p)) # quant_mpjpe[step * batch_size:step * batch_size + curr_batch_size] = error if seqName not in quant_mpjpe.keys(): quant_mpjpe[seqName] = [] quant_recon_err[seqName] = [] quant_mpjpe[seqName].append(error[ii]) quant_recon_err[seqName].append(r_error[ii]) # Reconstuction_error # quant_recon_err[step * batch_size:step * batch_size + curr_batch_size] = r_error list_mpjpe = np.hstack([ quant_mpjpe[k] for k in quant_mpjpe]) list_reconError = np.hstack([ quant_recon_err[k] for k in quant_recon_err]) if bVerbose: print(">>> {} : MPJPE {:.02f} mm, error: {:.02f} mm | Total MPJPE {:.02f} mm, error {:.02f} mm".format(seqName, np.mean(error)*1000, np.mean(r_error)*1000, np.hstack(list_mpjpe).mean()*1000, np.hstack(list_reconError).mean()*1000) ) # print("MPJPE {}, error: {}".format(np.mean(error)*100, np.mean(r_error)*100)) # If mask or part evaluation, render the mask and part images # if eval_masks or eval_parts: # mask, parts = renderer(pred_vertices, pred_camera) # Mask evaluation (for LSP) if eval_masks: center = batch['center'].cpu().numpy() scale = batch['scale'].cpu().numpy() # Dimensions of original image orig_shape = batch['orig_shape'].cpu().numpy() for i in range(curr_batch_size): # After rendering, convert imate back to original resolution pred_mask = uncrop(mask[i].cpu().numpy(), center[i], scale[i], orig_shape[i]) > 0 # Load gt mask gt_mask = cv2.imread(os.path.join(annot_path, batch['maskname'][i]), 0) > 0 # Evaluation consistent with the original UP-3D code accuracy += (gt_mask == pred_mask).sum() pixel_count += np.prod(np.array(gt_mask.shape)) for c in range(2): cgt = gt_mask == c cpred = pred_mask == c tp[c] += (cgt & cpred).sum() fp[c] += (~cgt & cpred).sum() fn[c] += (cgt & ~cpred).sum() f1 = 2 * tp / (2 * tp + fp + fn) # Part evaluation (for LSP) if eval_parts: center = batch['center'].cpu().numpy() scale = batch['scale'].cpu().numpy() orig_shape = batch['orig_shape'].cpu().numpy() for i in range(curr_batch_size): pred_parts = uncrop(parts[i].cpu().numpy().astype(np.uint8), center[i], scale[i], orig_shape[i]) # Load gt part segmentation gt_parts = cv2.imread(os.path.join(annot_path, batch['partname'][i]), 0) # Evaluation consistent with the original UP-3D code # 6 parts + background for c in range(7): cgt = gt_parts == c cpred = pred_parts == c cpred[gt_parts == 255] = 0 parts_tp[c] += (cgt & cpred).sum() parts_fp[c] += (~cgt & cpred).sum() parts_fn[c] += (cgt & ~cpred).sum() gt_parts[gt_parts == 255] = 0 pred_parts[pred_parts == 255] = 0 parts_f1 = 2 * parts_tp / (2 * parts_tp + parts_fp + parts_fn) parts_accuracy += (gt_parts == pred_parts).sum() parts_pixel_count += np.prod(np.array(gt_parts.shape)) # Print intermediate results during evaluation if bVerbose: if step % log_freq == log_freq - 1: if eval_pose: print('MPJPE: ' + str(1000 * mpjpe[:step * batch_size].mean())) print('Reconstruction Error: ' + str(1000 * recon_err[:step * batch_size].mean())) print() if eval_masks: print('Accuracy: ', accuracy / pixel_count) print('F1: ', f1.mean()) print() if eval_parts: print('Parts Accuracy: ', parts_accuracy / parts_pixel_count) print('Parts F1 (BG): ', parts_f1[[0,1,2,3,4,5,6]].mean()) print() # if step==3: #Debug # break # Save reconstructions to a file for further processing if save_results: np.savez(result_file, pred_joints=pred_joints, pose=smpl_pose, betas=smpl_betas, camera=smpl_camera) # Print final results during evaluation if bVerbose: print('*** Final Results ***') print() if eval_pose: # if bVerbose: # print('MPJPE: ' + str(1000 * mpjpe.mean())) # print('Reconstruction Error: ' + str(1000 * recon_err.mean())) # print() list_mpjpe = np.hstack([ quant_mpjpe[k] for k in quant_mpjpe]) list_reconError = np.hstack([ quant_recon_err[k] for k in quant_recon_err]) output_str ='SeqNames; ' for seq in quant_mpjpe: output_str += seq + ';' output_str +='\n MPJPE; ' quant_mpjpe_avg_mm = np.hstack(list_mpjpe).mean()*1000 output_str += "Avg {:.02f} mm; ".format( quant_mpjpe_avg_mm) for seq in quant_mpjpe: output_str += '{:.02f}; '.format(1000 * np.hstack(quant_mpjpe[seq]).mean()) output_str +='\n Recon Error; ' quant_recon_error_avg_mm = np.hstack(list_reconError).mean()*1000 output_str +="Avg {:.02f}mm; ".format( quant_recon_error_avg_mm ) for seq in quant_recon_err: output_str += '{:.02f}; '.format(1000 * np.hstack(quant_recon_err[seq]).mean()) if bVerbose: print(output_str) else: print(">>> Test on 3DPW: MPJPE: {} | quant_recon_error_avg_mm: {}".format(quant_mpjpe_avg_mm, quant_recon_error_avg_mm) ) #Export log to json if os.path.exists("/private/home/hjoo/codes/fairmocap/benchmarks_eval"): targetFile = "evalFromPkl_" + os.path.basename(pklDir) + ".txt" targetFile = os.path.join('/private/home/hjoo/codes/fairmocap/benchmarks_eval',targetFile) print(f"\n Writing output to:{targetFile}") text_file = open(targetFile, "w") text_file.write(output_str) text_file.write("\n Input Pkl Dir:{}".format(pklDir)) text_file.close() return quant_mpjpe_avg_mm, quant_recon_error_avg_mm if bVerbose: if eval_masks: print('Accuracy: ', accuracy / pixel_count) print('F1: ', f1.mean()) print() if eval_parts: print('Parts Accuracy: ', parts_accuracy / parts_pixel_count) print('Parts F1 (BG): ', parts_f1[[0,1,2,3,4,5,6]].mean()) print() return -1 #Should return something if __name__ == '__main__': if len(sys.argv)>1: args = parser.parse_args() else: params =['--checkpoint','/home/hjoo/Dropbox (Facebook)/spinouput/2019-10-29-00:48-out/test1/checkpoints/2019_10_29-01_11_29.pt','--dataset','3dpw', '--log_freq','20'] params =['--checkpoint','/home/hjoo/Dropbox (Facebook)/spinouput/10-30-40560-coco3d_first/checkpoints/2019_10_30-13_57_53.pt','--dataset','h36m-p1', '--log_freq','20'] # params =['--checkpoint','data/model_checkpoint.pt','--dataset','h36m-p1', '--log_freq','20','--num_workers',0] params =['--checkpoint','/home/hjoo/Dropbox (Facebook)/spinouput/10-31-5896-ours_coco3d_all/checkpoints/2019_10_31-11_37_20.pt'] #wCOCO3D only early first try params =['--checkpoint','/home/hjoo/Dropbox (Facebook)/spinouput/10-31-50173-spin_all/checkpoints/2019_11_01-22_18_03.pt'] #wCOCO3D only early first try params =['--checkpoint','logs/11-07-55557-w_upper0_2_spin_all-5372/checkpoints/2019_11_07-21_13_54-best-58.088939636945724.pt'] params =['--checkpoint','/home/hjoo/Dropbox (Facebook)/spinouput/11-14-84106-fromInit_coco3d_cocoplus3d-2878/checkpoints/2019_11_15-23_18_29-best-61.0622800886631.pt'] #Ours 3D (no Aug!) #Semi-final params =['--checkpoint','logs/11-13-78679-bab_spin_mlc3d_fter60_ag-9589/checkpoints/2019_11_14-02_14_28-best-55.79321086406708.pt'] #Ours 3D + augmentation params =['--checkpoint','logs/11-13-78681-bab_spin_mlc3d_fter65-1249/checkpoints/2019_11_14-05_36_41-best-56.23840540647507.pt'] #Ours 3D .... this is also with Aug!! params =['--checkpoint','spinmodel_shared/11-13-78679-bab_spin_mlc3d_fter60_ag-9589/checkpoints/2019_11_14-02_14_28-best-55.79321086406708.pt'] #Ours 3D + augmentation params =['--checkpoint','logs/11-13-78679-bab_spin_mlc3d_fter60-7183/checkpoints/2019_11_14-08_12_35-best-56.12510070204735.pt'] #Ours 3D (no Aug!) params =['--checkpoint','data/model_checkpoint.pt'] #Original params +=['--dataset','3dpw-vibe'] # params +=['--num_workers',0] args = parser.parse_args(params) args.batch_size =64 args.num_workers =4 model = hmr(config.SMPL_MEAN_PARAMS) checkpoint = torch.load(args.checkpoint) model.load_state_dict(checkpoint['model'], strict=False) model.cuda() model.eval() # # Setup evaluation dataset # # dataset = BaseDataset(None, '3dpw', is_train=False, bMiniTest=False) # dataset = BaseDataset(None, '3dpw', is_train=False, bMiniTest=False, bEnforceUpperOnly=False) # # dataset = BaseDataset(None, '3dpw-crop', is_train=False, bMiniTest=False, bEnforceUpperOnly=True) # # # Run evaluation # run_evaluation(model, '3dpw',dataset , args.result_file, # batch_size=args.batch_size, # shuffle=args.shuffle, # log_freq=args.log_freq, num_workers=args.num_workers) # # Setup evaluation dataset # dataset = BaseDataset(None, 'h36m-p1', is_train=False, bMiniTest=False, bEnforceUpperOnly=False) # # Run evaluation # run_evaluation(model, 'h36m-p1', dataset, args.result_file, # batch_size=args.batch_size, # shuffle=args.shuffle, # log_freq=args.log_freq, num_workers=args.num_workers) # Setup evaluation dataset dataset = BaseDataset(None, args.dataset, is_train=False, bMiniTest=False, bEnforceUpperOnly=False) # Run evaluation run_evaluation(model, args.dataset, dataset, args.result_file, batch_size=args.batch_size, shuffle=args.shuffle, log_freq=args.log_freq, num_workers=args.num_workers) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -