eft/apps/eval.py [300:484]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - list_mpjpe = np.hstack([ quant_mpjpe[k] for k in quant_mpjpe]) list_reconError = np.hstack([ quant_recon_err[k] for k in quant_recon_err]) if bVerbose: print(">>> {} : MPJPE {:.02f} mm, error: {:.02f} mm | Total MPJPE {:.02f} mm, error {:.02f} mm".format(seqName, np.mean(error)*1000, np.mean(r_error)*1000, np.hstack(list_mpjpe).mean()*1000, np.hstack(list_reconError).mean()*1000) ) # print("MPJPE {}, error: {}".format(np.mean(error)*100, np.mean(r_error)*100)) # If mask or part evaluation, render the mask and part images # if eval_masks or eval_parts: # mask, parts = renderer(pred_vertices, pred_camera) # Mask evaluation (for LSP) if eval_masks: center = batch['center'].cpu().numpy() scale = batch['scale'].cpu().numpy() # Dimensions of original image orig_shape = batch['orig_shape'].cpu().numpy() for i in range(curr_batch_size): # After rendering, convert imate back to original resolution pred_mask = uncrop(mask[i].cpu().numpy(), center[i], scale[i], orig_shape[i]) > 0 # Load gt mask gt_mask = cv2.imread(os.path.join(annot_path, batch['maskname'][i]), 0) > 0 # Evaluation consistent with the original UP-3D code accuracy += (gt_mask == pred_mask).sum() pixel_count += np.prod(np.array(gt_mask.shape)) for c in range(2): cgt = gt_mask == c cpred = pred_mask == c tp[c] += (cgt & cpred).sum() fp[c] += (~cgt & cpred).sum() fn[c] += (cgt & ~cpred).sum() f1 = 2 * tp / (2 * tp + fp + fn) # Part evaluation (for LSP) if eval_parts: center = batch['center'].cpu().numpy() scale = batch['scale'].cpu().numpy() orig_shape = batch['orig_shape'].cpu().numpy() for i in range(curr_batch_size): pred_parts = uncrop(parts[i].cpu().numpy().astype(np.uint8), center[i], scale[i], orig_shape[i]) # Load gt part segmentation gt_parts = cv2.imread(os.path.join(annot_path, batch['partname'][i]), 0) # Evaluation consistent with the original UP-3D code # 6 parts + background for c in range(7): cgt = gt_parts == c cpred = pred_parts == c cpred[gt_parts == 255] = 0 parts_tp[c] += (cgt & cpred).sum() parts_fp[c] += (~cgt & cpred).sum() parts_fn[c] += (cgt & ~cpred).sum() gt_parts[gt_parts == 255] = 0 pred_parts[pred_parts == 255] = 0 parts_f1 = 2 * parts_tp / (2 * parts_tp + parts_fp + parts_fn) parts_accuracy += (gt_parts == pred_parts).sum() parts_pixel_count += np.prod(np.array(gt_parts.shape)) # Print intermediate results during evaluation if bVerbose: if step % log_freq == log_freq - 1: if eval_pose: print('MPJPE: ' + str(1000 * mpjpe[:step * batch_size].mean())) print('Reconstruction Error: ' + str(1000 * recon_err[:step * batch_size].mean())) print() if eval_masks: print('Accuracy: ', accuracy / pixel_count) print('F1: ', f1.mean()) print() if eval_parts: print('Parts Accuracy: ', parts_accuracy / parts_pixel_count) print('Parts F1 (BG): ', parts_f1[[0,1,2,3,4,5,6]].mean()) print() # if step==3: #Debug # break # Save reconstructions to a file for further processing if save_results: np.savez(result_file, pred_joints=pred_joints, pose=smpl_pose, betas=smpl_betas, camera=smpl_camera) # Print final results during evaluation if bVerbose: print('*** Final Results ***') print() evalLog ={} if eval_pose: # if bVerbose: # print('MPJPE: ' + str(1000 * mpjpe.mean())) # print('Reconstruction Error: ' + str(1000 * recon_err.mean())) # print() list_mpjpe = np.hstack([ quant_mpjpe[k] for k in quant_mpjpe]) list_reconError = np.hstack([ quant_recon_err[k] for k in quant_recon_err]) output_str ='SeqNames; ' for seq in quant_mpjpe: output_str += seq + ';' output_str +='\n MPJPE; ' quant_mpjpe_avg_mm = np.hstack(list_mpjpe).mean()*1000 output_str += "Avg {:.02f} mm; ".format( quant_mpjpe_avg_mm) for seq in quant_mpjpe: output_str += '{:.02f}; '.format(1000 * np.hstack(quant_mpjpe[seq]).mean()) output_str +='\n Recon Error; ' quant_recon_error_avg_mm = np.hstack(list_reconError).mean()*1000 output_str +="Avg {:.02f}mm; ".format( quant_recon_error_avg_mm ) for seq in quant_recon_err: output_str += '{:.02f}; '.format(1000 * np.hstack(quant_recon_err[seq]).mean()) if bVerbose: print(output_str) else: print(">>> Test on 3DPW: MPJPE: {} | quant_recon_error_avg_mm: {}".format(quant_mpjpe_avg_mm, quant_recon_error_avg_mm) ) #Save output to dict # evalLog['checkpoint']= args.checkpoint evalLog['testdb'] = dataset_name evalLog['datasize'] = len(data_loader.dataset) for seq in quant_mpjpe: quant_mpjpe[seq] = 1000 * np.hstack(quant_mpjpe[seq]).mean() for seq in quant_recon_err: quant_recon_err[seq] = 1000 * np.hstack(quant_recon_err[seq]).mean() evalLog['quant_mpjpe'] = quant_mpjpe #MPJPE evalLog['quant_recon_err']= quant_recon_err #PA-MPJPE evalLog['quant_output_logstr']= output_str #PA-MPJPE evalLog['quant_mpjpe_avg_mm'] = quant_mpjpe_avg_mm #MPJPE evalLog['quant_recon_error_avg_mm']= quant_recon_error_avg_mm #PA-MPJPE # return quant_mpjpe_avg_mm, quant_recon_error_avg_mm, evalLog return evalLog if bVerbose: if eval_masks: print('Accuracy: ', accuracy / pixel_count) print('F1: ', f1.mean()) print() if eval_parts: print('Parts Accuracy: ', parts_accuracy / parts_pixel_count) print('Parts F1 (BG): ', parts_f1[[0,1,2,3,4,5,6]].mean()) print() return -1 #Should return something def eval_main(params): args = parser.parse_args(params) model = hmr(config.SMPL_MEAN_PARAMS) if os.path.isdir(args.checkpoint): fileCands = os.listdir(args.checkpoint) fileCandsBest = [n for n in fileCands if "-best-" in n] if len(fileCandsBest)>0: bestCand = sorted(fileCandsBest)[-1] else: bestCand = sorted(fileCands)[-1] args.checkpoint = os.path.join(args.checkpoint, bestCand) assert os.path.isfile(args.checkpoint) checkpoint = torch.load(args.checkpoint) model.load_state_dict(checkpoint['model'], strict=False) model.cuda() model.eval() # Load if eval is alread done evalFolder = os.path.dirname(args.checkpoint)+"/../evallog" evalLogFileName = os.path.join(evalFolder, os.path.basename(args.checkpoint)[:-3] + '.json' ) if os.path.exists(evalLogFileName): #Bug recompute all print(f"Found already processed log: {evalLogFileName}") with open(evalLogFileName,'r') as f: evalLogAll = json.load(f) else: evalLogAll ={} evalLogAll['checkpoint'] = args.checkpoint - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - eft/apps/eval_multicrop.py [276:459]: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - list_mpjpe = np.hstack([ quant_mpjpe[k] for k in quant_mpjpe]) list_reconError = np.hstack([ quant_recon_err[k] for k in quant_recon_err]) if bVerbose: print(">>> {} : MPJPE {:.02f} mm, error: {:.02f} mm | Total MPJPE {:.02f} mm, error {:.02f} mm".format(seqName, np.mean(error)*1000, np.mean(r_error)*1000, np.hstack(list_mpjpe).mean()*1000, np.hstack(list_reconError).mean()*1000) ) # print("MPJPE {}, error: {}".format(np.mean(error)*100, np.mean(r_error)*100)) # If mask or part evaluation, render the mask and part images # if eval_masks or eval_parts: # mask, parts = renderer(pred_vertices, pred_camera) # Mask evaluation (for LSP) if eval_masks: center = batch['center'].cpu().numpy() scale = batch['scale'].cpu().numpy() # Dimensions of original image orig_shape = batch['orig_shape'].cpu().numpy() for i in range(curr_batch_size): # After rendering, convert imate back to original resolution pred_mask = uncrop(mask[i].cpu().numpy(), center[i], scale[i], orig_shape[i]) > 0 # Load gt mask gt_mask = cv2.imread(os.path.join(annot_path, batch['maskname'][i]), 0) > 0 # Evaluation consistent with the original UP-3D code accuracy += (gt_mask == pred_mask).sum() pixel_count += np.prod(np.array(gt_mask.shape)) for c in range(2): cgt = gt_mask == c cpred = pred_mask == c tp[c] += (cgt & cpred).sum() fp[c] += (~cgt & cpred).sum() fn[c] += (cgt & ~cpred).sum() f1 = 2 * tp / (2 * tp + fp + fn) # Part evaluation (for LSP) if eval_parts: center = batch['center'].cpu().numpy() scale = batch['scale'].cpu().numpy() orig_shape = batch['orig_shape'].cpu().numpy() for i in range(curr_batch_size): pred_parts = uncrop(parts[i].cpu().numpy().astype(np.uint8), center[i], scale[i], orig_shape[i]) # Load gt part segmentation gt_parts = cv2.imread(os.path.join(annot_path, batch['partname'][i]), 0) # Evaluation consistent with the original UP-3D code # 6 parts + background for c in range(7): cgt = gt_parts == c cpred = pred_parts == c cpred[gt_parts == 255] = 0 parts_tp[c] += (cgt & cpred).sum() parts_fp[c] += (~cgt & cpred).sum() parts_fn[c] += (cgt & ~cpred).sum() gt_parts[gt_parts == 255] = 0 pred_parts[pred_parts == 255] = 0 parts_f1 = 2 * parts_tp / (2 * parts_tp + parts_fp + parts_fn) parts_accuracy += (gt_parts == pred_parts).sum() parts_pixel_count += np.prod(np.array(gt_parts.shape)) # Print intermediate results during evaluation if bVerbose: if step % log_freq == log_freq - 1: if eval_pose: print('MPJPE: ' + str(1000 * mpjpe[:step * batch_size].mean())) print('Reconstruction Error: ' + str(1000 * recon_err[:step * batch_size].mean())) print() if eval_masks: print('Accuracy: ', accuracy / pixel_count) print('F1: ', f1.mean()) print() if eval_parts: print('Parts Accuracy: ', parts_accuracy / parts_pixel_count) print('Parts F1 (BG): ', parts_f1[[0,1,2,3,4,5,6]].mean()) print() # if step==3: #Debug # break # Save reconstructions to a file for further processing if save_results: np.savez(result_file, pred_joints=pred_joints, pose=smpl_pose, betas=smpl_betas, camera=smpl_camera) # Print final results during evaluation if bVerbose: print('*** Final Results ***') print() evalLog ={} if eval_pose: # if bVerbose: # print('MPJPE: ' + str(1000 * mpjpe.mean())) # print('Reconstruction Error: ' + str(1000 * recon_err.mean())) # print() list_mpjpe = np.hstack([ quant_mpjpe[k] for k in quant_mpjpe]) list_reconError = np.hstack([ quant_recon_err[k] for k in quant_recon_err]) output_str ='SeqNames; ' for seq in quant_mpjpe: output_str += seq + ';' output_str +='\n MPJPE; ' quant_mpjpe_avg_mm = np.hstack(list_mpjpe).mean()*1000 output_str += "Avg {:.02f} mm; ".format( quant_mpjpe_avg_mm) for seq in quant_mpjpe: output_str += '{:.02f}; '.format(1000 * np.hstack(quant_mpjpe[seq]).mean()) output_str +='\n Recon Error; ' quant_recon_error_avg_mm = np.hstack(list_reconError).mean()*1000 output_str +="Avg {:.02f}mm; ".format( quant_recon_error_avg_mm ) for seq in quant_recon_err: output_str += '{:.02f}; '.format(1000 * np.hstack(quant_recon_err[seq]).mean()) if bVerbose: print(output_str) else: print(">>> Test on 3DPW: MPJPE: {} | quant_recon_error_avg_mm: {}".format(quant_mpjpe_avg_mm, quant_recon_error_avg_mm) ) #Save output to dict # evalLog['checkpoint']= args.checkpoint evalLog['testdb'] = dataset_name evalLog['datasize'] = len(data_loader.dataset) for seq in quant_mpjpe: quant_mpjpe[seq] = 1000 * np.hstack(quant_mpjpe[seq]).mean() for seq in quant_recon_err: quant_recon_err[seq] = 1000 * np.hstack(quant_recon_err[seq]).mean() evalLog['quant_mpjpe'] = quant_mpjpe #MPJPE evalLog['quant_recon_err']= quant_recon_err #PA-MPJPE evalLog['quant_output_logstr']= output_str #PA-MPJPE evalLog['quant_mpjpe_avg_mm'] = quant_mpjpe_avg_mm #MPJPE evalLog['quant_recon_error_avg_mm']= quant_recon_error_avg_mm #PA-MPJPE # return quant_mpjpe_avg_mm, quant_recon_error_avg_mm, evalLog return evalLog if bVerbose: if eval_masks: print('Accuracy: ', accuracy / pixel_count) print('F1: ', f1.mean()) print() if eval_parts: print('Parts Accuracy: ', parts_accuracy / parts_pixel_count) print('Parts F1 (BG): ', parts_f1[[0,1,2,3,4,5,6]].mean()) print() return -1 #Should return something def eval_main(params): args = parser.parse_args(params) model = hmr(config.SMPL_MEAN_PARAMS) if os.path.isdir(args.checkpoint): fileCands = os.listdir(args.checkpoint) fileCandsBest = [n for n in fileCands if "-best-" in n] if len(fileCandsBest)>0: bestCand = sorted(fileCandsBest)[-1] else: bestCand = sorted(fileCands)[-1] args.checkpoint = os.path.join(args.checkpoint, bestCand) assert os.path.isfile(args.checkpoint) checkpoint = torch.load(args.checkpoint) model.load_state_dict(checkpoint['model'], strict=False) model.cuda() model.eval() # Load if eval is alread done evalFolder = os.path.dirname(args.checkpoint)+"/../evallog" evalLogFileName = os.path.join(evalFolder, os.path.basename(args.checkpoint)[:-3] + '.json' ) if os.path.exists(evalLogFileName): #Bug recompute all print(f"Found already processed log: {evalLogFileName}") with open(evalLogFileName,'r') as f: evalLogAll = json.load(f) else: evalLogAll ={} evalLogAll['checkpoint'] = args.checkpoint - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -