in visualization/FitAdam/src/FitToBody.cpp [857:1121]
void Adam_Fit_PAF(TotalModel &adam, smpl::SMPLParams &frame_param, Eigen::MatrixXd &BodyJoints, Eigen::MatrixXd &rFoot, Eigen::MatrixXd &lFoot, Eigen::MatrixXd &rHandJoints,
Eigen::MatrixXd &lHandJoints, Eigen::MatrixXd &faceJoints, Eigen::MatrixXd &PAF, Eigen::MatrixXd &surface_constraint, double* calibK,
uint regressor_type, bool quan, bool fitPAFfirst, bool fit_face_exp, bool euler)
{
using namespace Eigen;
const auto start = std::chrono::high_resolution_clock::now();
if (fitPAFfirst) // if true, fit onto only PAF first
{
std::cout << "Fitting to 3D skeleton as the first step" << std::endl;
ceres::Problem init_problem;
AdamFitData data(adam, BodyJoints, rFoot, lFoot, faceJoints, lHandJoints, rHandJoints, PAF, surface_constraint,
false, false, nullptr, true, false);
AdamFullCost* adam_cost;
adam_cost = new AdamFullCost(data, regressor_type, false, euler);
init_problem.AddResidualBlock(adam_cost,
NULL,
frame_param.m_adam_t.data(),
frame_param.m_adam_pose.data(),
frame_param.m_adam_coeffs.data());
// for (int i = 0; i < TotalModel::NUM_POSE_PARAMETERS; i++)
// {
// init_problem.SetParameterLowerBound(frame_param.m_adam_pose.data(), i, -180);
// init_problem.SetParameterUpperBound(frame_param.m_adam_pose.data(), i, 180);
// }
ceres::Solver::Options init_options;
ceres::Solver::Summary init_summary;
SetSolverOptions(&init_options);
init_options.function_tolerance = 1e-4;
init_options.max_num_iterations = 20;
init_options.use_nonmonotonic_steps = true;
init_options.num_linear_solver_threads = 1;
init_options.minimizer_progress_to_stdout = true;
// if (quan) init_problem.SetParameterBlockConstant(frame_param.m_adam_coeffs.data());
adam_cost->toggle_activate(false, false, false);
adam_cost->toggle_rigid_body(true);
const auto start_solve = std::chrono::high_resolution_clock::now();
ceres::Solve(init_options, &init_problem, &init_summary);
std::cout << init_summary.FullReport() << std::endl;
//Body Prior (coef) //////////////////////////////////////////////////////////////////////////
CoeffsParameterNormDiff* cost_prior_body_coeffs = new CoeffsParameterNormDiff(TotalModel::NUM_SHAPE_COEFFICIENTS);
ceres::LossFunction* loss_weight_prior_body_coeffs = new ceres::ScaledLoss(NULL,
quan? 1e-2 : 1e-2,
ceres::TAKE_OWNERSHIP);
init_problem.AddResidualBlock(cost_prior_body_coeffs,
loss_weight_prior_body_coeffs,
frame_param.m_adam_coeffs.data());
//Body Prior (pose) //////////////////////////////////////////////////////////////////////////
AdamBodyPoseParamPriorDiff* cost_prior_body_pose = new AdamBodyPoseParamPriorDiff(TotalModel::NUM_POSE_PARAMETERS);
ceres::LossFunction* loss_weight_prior_body_pose = new ceres::ScaledLoss(NULL,
quan? 1e-2 : 1e-1,
// 1e-2,
ceres::TAKE_OWNERSHIP);
std::fill(cost_prior_body_pose->weight.data() + 3, cost_prior_body_pose->weight.data() + 9, 10.);
std::fill(cost_prior_body_pose->weight.data() + 12, cost_prior_body_pose->weight.data() + 18, 10.);
init_problem.AddResidualBlock(cost_prior_body_pose,
loss_weight_prior_body_pose,
frame_param.m_adam_pose.data());
if(!euler)
{
Eigen::Matrix<double, 72, TotalModel::NUM_POSE_PARAMETERS> prior_A; prior_A.setZero();
prior_A.block<72, 66>(0, 0) = adam.smpl_pose_prior_A.block<72, 66>(0, 0); // for body, use the prior from SMPL
Eigen::Matrix<double, TotalModel::NUM_POSE_PARAMETERS, 1> prior_mu; prior_mu.setZero();
prior_mu.block<66, 1>(0, 0) = -adam.smpl_pose_prior_mu.block<66, 1>(0, 0); // use the prior from SMPL (negative)
ceres::CostFunction *pose_reg = new ceres::NormalPrior(prior_A, prior_mu);
ceres::LossFunction *pose_reg_loss = new ceres::ScaledLoss(NULL,
10,
ceres::TAKE_OWNERSHIP);
init_problem.AddResidualBlock(pose_reg,
pose_reg_loss,
frame_param.m_adam_pose.data());
for (int i = 0; i < TotalModel::NUM_POSE_PARAMETERS; i++) cost_prior_body_pose->weight[i] = 0.0; // only use regularization for fingers
Eigen::MatrixXd hand_prior_A(120, TotalModel::NUM_POSE_PARAMETERS); hand_prior_A.setZero();
Eigen::Matrix<double, TotalModel::NUM_POSE_PARAMETERS, 1> hand_prior_mu; hand_prior_mu.setZero();
hand_prior_mu.block<60, 1>(66, 0) = -adam.hand_pose_prior_mu; hand_prior_mu.block<60, 1>(126, 0) = adam.hand_pose_prior_mu;
for (int i = 66; i < 126; i += 3) hand_prior_mu(i, 0) = -hand_prior_mu(i, 0);
hand_prior_A.block<60, 60>(0, 66) = -adam.hand_pose_prior_A; hand_prior_A.block<60, 60>(60, 126) = adam.hand_pose_prior_A;
for (int i = 66; i < 126; i += 3) hand_prior_A.col(i) = -hand_prior_A.col(i);
ceres::CostFunction *hand_pose_reg = new ceres::NormalPrior(hand_prior_A, hand_prior_mu);
ceres::LossFunction *hand_pose_reg_loss = new ceres::ScaledLoss(NULL,
10,
ceres::TAKE_OWNERSHIP);
init_problem.AddResidualBlock(hand_pose_reg,
hand_pose_reg_loss,
frame_param.m_adam_pose.data());
}
init_options.function_tolerance = 1e-4;
adam_cost->toggle_activate(true, true, true);
adam_cost->toggle_rigid_body(false);
ceres::Solve(init_options, &init_problem, &init_summary);
std::cout << init_summary.FullReport() << std::endl;
const auto duration_solve = std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::high_resolution_clock::now() - start_solve).count();
std::cout << "3D solve time: " << duration_solve * 1e-6 << "\n";
frame_param.m_adam_t[2] = 200.0; // for fitting onto projection
}
std::cout << "Fitting to 2D skeleton Projection" << std::endl;
ceres::Problem problem;
AdamFitData data(adam, BodyJoints, rFoot, lFoot, faceJoints, lHandJoints, rHandJoints, PAF, surface_constraint, false, true, calibK, true, surface_constraint.cols() > 0);
AdamFullCost* adam_cost;
adam_cost = new AdamFullCost(data, regressor_type, fit_face_exp, euler);
if (fit_face_exp)
problem.AddResidualBlock(adam_cost,
NULL,
frame_param.m_adam_t.data(),
frame_param.m_adam_pose.data(),
frame_param.m_adam_coeffs.data(),
frame_param.m_adam_facecoeffs_exp.data());
else
problem.AddResidualBlock(adam_cost,
NULL,
frame_param.m_adam_t.data(),
frame_param.m_adam_pose.data(),
frame_param.m_adam_coeffs.data());
ceres::Solver::Options options;
ceres::Solver::Summary summary;
SetSolverOptions(&options);
options.function_tolerance = 1e-4;
options.max_num_iterations = 30;
options.use_nonmonotonic_steps = true;
options.num_linear_solver_threads = 1;
options.minimizer_progress_to_stdout = true;
adam_cost->toggle_activate(false, false, false);
adam_cost->toggle_rigid_body(true);
// if(!fitPAFfirst && !quan) // if fitPAFfirst, should be the first frame in video, allow shape change
// {
// problem.SetParameterBlockConstant(frame_param.m_adam_coeffs.data());
// }
const auto start_solve = std::chrono::high_resolution_clock::now();
if(!quan) // for quantitative, don't solve this time, especially for failure cases
{
ceres::Solve(options, &problem, &summary);
std::cout << summary.FullReport() << std::endl;
}
//Body Prior (coef) //////////////////////////////////////////////////////////////////////////
CoeffsParameterNormDiff* cost_prior_body_coeffs = new CoeffsParameterNormDiff(TotalModel::NUM_SHAPE_COEFFICIENTS);
ceres::LossFunction* loss_weight_prior_body_coeffs = new ceres::ScaledLoss(NULL,
quan? 1e-5 : 1e-2,
ceres::TAKE_OWNERSHIP);
problem.AddResidualBlock(cost_prior_body_coeffs,
loss_weight_prior_body_coeffs,
frame_param.m_adam_coeffs.data());
//Body Prior (pose) //////////////////////////////////////////////////////////////////////////
AdamBodyPoseParamPriorDiff* cost_prior_body_pose = new AdamBodyPoseParamPriorDiff(TotalModel::NUM_POSE_PARAMETERS);
ceres::LossFunction* loss_weight_prior_body_pose = new ceres::ScaledLoss(NULL,
quan? 1e-2 : 1e-2,
ceres::TAKE_OWNERSHIP);
problem.AddResidualBlock(cost_prior_body_pose,
loss_weight_prior_body_pose,
frame_param.m_adam_pose.data());
if (!euler)
{
Eigen::Matrix<double, 72, TotalModel::NUM_POSE_PARAMETERS> prior_A; prior_A.setZero();
prior_A.block<72, 66>(0, 0) = adam.smpl_pose_prior_A.block<72, 66>(0, 0); // for body, use the prior from SMPL
Eigen::Matrix<double, TotalModel::NUM_POSE_PARAMETERS, 1> prior_mu; prior_mu.setZero();
prior_mu.block<66, 1>(0, 0) = -adam.smpl_pose_prior_mu.block<66, 1>(0, 0); // use the prior from SMPL
ceres::CostFunction *pose_reg = new ceres::NormalPrior(prior_A, prior_mu);
ceres::LossFunction* pose_reg_loss = new ceres::ScaledLoss(NULL,
10,
ceres::TAKE_OWNERSHIP);
problem.AddResidualBlock(pose_reg,
pose_reg_loss,
frame_param.m_adam_pose.data());
for (int i = 0; i < TotalModel::NUM_POSE_PARAMETERS; i++) cost_prior_body_pose->weight[i] = 0.0; // only use regularization for fingers
Eigen::MatrixXd hand_prior_A(120, TotalModel::NUM_POSE_PARAMETERS); hand_prior_A.setZero();
Eigen::Matrix<double, TotalModel::NUM_POSE_PARAMETERS, 1> hand_prior_mu; hand_prior_mu.setZero();
hand_prior_mu.block<60, 1>(66, 0) = -adam.hand_pose_prior_mu; hand_prior_mu.block<60, 1>(126, 0) = adam.hand_pose_prior_mu;
for (int i = 66; i < 126; i += 3) hand_prior_mu(i, 0) = -hand_prior_mu(i, 0);
hand_prior_A.block<60, 60>(0, 66) = -adam.hand_pose_prior_A; hand_prior_A.block<60, 60>(60, 126) = adam.hand_pose_prior_A;
for (int i = 66; i < 126; i += 3) hand_prior_A.col(i) = -hand_prior_A.col(i);
ceres::CostFunction *hand_pose_reg = new ceres::NormalPrior(hand_prior_A, hand_prior_mu);
ceres::LossFunction *hand_pose_reg_loss = new ceres::ScaledLoss(NULL,
10,
ceres::TAKE_OWNERSHIP);
problem.AddResidualBlock(hand_pose_reg,
hand_pose_reg_loss,
frame_param.m_adam_pose.data());
}
if (fit_face_exp)
{
ceres::NormalPrior *cost_prior_face_exp = new ceres::NormalPrior(adam.face_prior_A_exp.asDiagonal(), Eigen::Matrix<double, TotalModel::NUM_EXP_BASIS_COEFFICIENTS, 1>::Zero());
ceres::LossFunction *loss_weight_prior_face_exp = new ceres::ScaledLoss(NULL,
100, //original
ceres::TAKE_OWNERSHIP);
problem.AddResidualBlock(cost_prior_face_exp,
loss_weight_prior_face_exp,
frame_param.m_adam_facecoeffs_exp.data());
}
if (quan)
{
for (int i = 0; i < 12; i++) adam_cost->PAF_weight[i] = 50;
if(euler) std::fill(cost_prior_body_pose->weight.data() + 36, cost_prior_body_pose->weight.data() + TotalModel::NUM_POSE_PARAMETERS, 2.0);
}
else
{
if (regressor_type == 0)
{
// setting for make videos
// adam_cost->m_targetPts_weight[adam_cost->m_nCorrespond_adam2joints + 0] =
// adam_cost->m_targetPts_weight[adam_cost->m_nCorrespond_adam2joints + 1] =
// adam_cost->m_targetPts_weight[adam_cost->m_nCorrespond_adam2joints + 2] =
// adam_cost->m_targetPts_weight[adam_cost->m_nCorrespond_adam2joints + 4] =
// adam_cost->m_targetPts_weight[adam_cost->m_nCorrespond_adam2joints + 5] =
// adam_cost->m_targetPts_weight[adam_cost->m_nCorrespond_adam2joints + 6] =
// adam_cost->m_targetPts_weight[adam_cost->m_nCorrespond_adam2joints + 7] = 1.0;
std::fill(adam_cost->m_targetPts_weight.data() + 5 * adam_cost->m_nCorrespond_adam2joints,
adam_cost->m_targetPts_weight.data() + 5 * (adam_cost->m_nCorrespond_adam2joints + 8), 1.0);
// for (auto i = 0; i < adam.m_correspond_adam2face70_adamIdx.rows(); i++) // face starts from 8
// adam_cost->m_targetPts_weight[adam_cost->m_nCorrespond_adam2joints + 8] = 1.0;
std::fill(adam_cost->m_targetPts_weight.data() + 5 * (adam_cost->m_nCorrespond_adam2joints + 8),
adam_cost->m_targetPts_weight.data() + 5 * (adam_cost->m_nCorrespond_adam2joints + adam.m_correspond_adam2face70_adamIdx.rows()), 1.0);
}
else if (regressor_type == 2)
{
// set weight for all vertices
// for (auto i = 0; i < adam_cost->m_nCorrespond_adam2pts; ++i)
// {
// if (i < 8 + adam.m_correspond_adam2face70_adamIdx.rows()) adam_cost->m_targetPts_weight[adam_cost->m_nCorrespond_adam2joints + i] = 1.0;
// else adam_cost->m_targetPts_weight[adam_cost->m_nCorrespond_adam2joints + i] = 0.1; // remaining are surface constraints
// }
std::fill(adam_cost->m_targetPts_weight.data() + 5 * adam_cost->m_nCorrespond_adam2joints,
adam_cost->m_targetPts_weight.data() + 5 * (adam_cost->m_nCorrespond_adam2joints + 8 + adam.m_correspond_adam2face70_adamIdx.rows()), 1.0);
std::fill(adam_cost->m_targetPts_weight.data() + 5 * (adam_cost->m_nCorrespond_adam2joints + 8 + adam.m_correspond_adam2face70_adamIdx.rows()),
adam_cost->m_targetPts_weight.data() + 5 * (adam_cost->m_nCorrespond_adam2joints + adam_cost->m_nCorrespond_adam2pts), 0.1);
}
adam_cost->toggle_activate(true, false, false);
adam_cost->toggle_rigid_body(false);
ceres::Solve(options, &problem, &summary);
std::cout << summary.FullReport() << std::endl;
}
adam_cost->toggle_activate(true, true, true);
adam_cost->toggle_rigid_body(false);
if (!quan && regressor_type == 2)
{
adam_cost->PAF_weight[12] = 5;
adam_cost->PAF_weight[13] = 5;
// adam_cost->m_targetPts_weight[17] = 0.1;
// adam_cost->m_targetPts_weight[18] = 0.1;
std::fill(adam_cost->m_targetPts_weight.data() + 17 * 5, adam_cost->m_targetPts_weight.data() + 19 * 5, 0.1); // lower cost for ear, satisfy the face70 constraints
}
ceres::Solve(options, &problem, &summary);
std::cout << summary.FullReport() << std::endl;
const auto duration_solve = std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::high_resolution_clock::now() - start_solve).count();
std::cout << "2D solve time: " << duration_solve * 1e-6 << "\n";
const auto duration = std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::high_resolution_clock::now() - start).count();
std::cout << "Total fitting time: " << duration * 1e-6 << "\n";
}