in visualization/FitAdam/include/AdamFastCost.h [310:611]
void SetupCost()
{
using namespace cv;
using namespace Eigen;
if (regressor_type == 0)
{
// calculating the dim for 3D / 2D constraints
m_nCorrespond_adam2joints = fit_data_.adam.m_indices_jointConst_adamIdx.rows();
if(fit_data_.lHandJoints.size() > 0) m_nCorrespond_adam2joints += fit_data_.adam.m_correspond_adam2lHand_adamIdx.rows();
if(fit_data_.rHandJoints.size() > 0) m_nCorrespond_adam2joints += fit_data_.adam.m_correspond_adam2rHand_adamIdx.rows();
// These are all the vertices to compute
total_vertex.push_back(8130); //nose
total_vertex.push_back(6970); //left ear
total_vertex.push_back(10088); //right ear
total_vertex.push_back(1372); //head top
total_vertex.push_back(14328); //right bigtoe
total_vertex.push_back(14288); //right littletoe
total_vertex.push_back(14357); //right heel
total_vertex.push_back(14361); //right heel
total_vertex.push_back(12239); //left bigtoe
total_vertex.push_back(12289); //left littletoe
total_vertex.push_back(12368); //left heel
total_vertex.push_back(12357); //left heel
// face detector results
for (int r = 0; r < fit_data_.adam.m_correspond_adam2face70_adamIdx.rows(); ++r)
{
int adamVertexIdx = fit_data_.adam.m_correspond_adam2face70_adamIdx(r);
total_vertex.push_back(adamVertexIdx);
}
// setting up the vertex (specify the vertices with constraints to optimize)
corres_vertex2targetpt.push_back(std::make_pair(0, std::vector<double>(5)));
corres_vertex2targetpt.push_back(std::make_pair(1, std::vector<double>(5)));
corres_vertex2targetpt.push_back(std::make_pair(2, std::vector<double>(5)));
corres_vertex2targetpt.push_back(std::make_pair(3, std::vector<double>(5)));
corres_vertex2targetpt.push_back(std::make_pair(4, std::vector<double>(5)));
corres_vertex2targetpt.push_back(std::make_pair(5, std::vector<double>(5)));
corres_vertex2targetpt.push_back(std::make_pair(6, std::vector<double>(5)));
corres_vertex2targetpt.push_back(std::make_pair(7, std::vector<double>(5)));
corres_vertex2targetpt.push_back(std::make_pair(8, std::vector<double>(5)));
corres_vertex2targetpt.push_back(std::make_pair(9, std::vector<double>(5)));
corres_vertex2targetpt.push_back(std::make_pair(10, std::vector<double>(5)));
corres_vertex2targetpt.push_back(std::make_pair(11, std::vector<double>(5)));
for (int r = 0; r < fit_data_.adam.m_correspond_adam2face70_adamIdx.rows(); ++r)
{
corres_vertex2targetpt.push_back(std::make_pair(12 + r, std::vector<double>(5)));
}
if (fit_data_.fit_surface)
{
for(auto i = 0; i < fit_data_.surface_constraint.cols(); i++)
{
corres_vertex2targetpt.push_back(std::make_pair(total_vertex.size()-1, std::vector<double>(5)));
total_vertex.push_back(fit_data_.surface_constraint(5, i));
}
}
m_nCorrespond_adam2pts = corres_vertex2targetpt.size();
PAF_connection.resize(num_PAF_constraint);
PAF_connection = {{ {{0, 12, 0, 2}}, {{0, 2, 0, 5}}, {{0, 5, 0, 8}}, {{0, 12, 0, 1}}, {{0, 1, 0, 4}}, {{0, 4, 0, 7}},
{{0, 12, 0, 17}}, {{0, 17, 0, 19}}, {{0, 19, 0, 21}}, // right arm
{{0, 12, 0, 16}}, {{0, 16, 0, 18}}, {{0, 18, 0, 20}}, // left arm
{{0, 12, 1, 0}}, {{0, 12, 1, 3}}, // neck -> nose, neck -> head top
{{0, 17, 0, 21}}, {{0, 16, 0, 20}}, // shoulder -> wrist
{{0, 2, 0, 21}}, {{0, 1, 0, 20}}, // hip -> wrist
{{0, 2, 0, 8}}, {{0, 1, 0, 7}}, // hip -> ankle
{{0, 20, 0, 22}}, {{0, 22, 0, 23}}, {{0, 23, 0, 24}}, {{0, 24, 0, 25}}, // left hand
{{0, 20, 0, 26}}, {{0, 26, 0, 27}}, {{0, 27, 0, 28}}, {{0, 28, 0, 29}},
{{0, 20, 0, 30}}, {{0, 30, 0, 31}}, {{0, 31, 0, 32}}, {{0, 32, 0, 33}},
{{0, 20, 0, 34}}, {{0, 34, 0, 35}}, {{0, 35, 0, 36}}, {{0, 36, 0, 37}},
{{0, 20, 0, 38}}, {{0, 38, 0, 39}}, {{0, 39, 0, 40}}, {{0, 40, 0, 41}},
{{0, 21, 0, 22 + 20}}, {{0, 22 + 20, 0, 23 + 20}}, {{0, 23 + 20, 0, 24 + 20}}, {{0, 24 + 20, 0, 25 + 20}}, // right hand
{{0, 21, 0, 26 + 20}}, {{0, 26 + 20, 0, 27 + 20}}, {{0, 27 + 20, 0, 28 + 20}}, {{0, 28 + 20, 0, 29 + 20}},
{{0, 21, 0, 30 + 20}}, {{0, 30 + 20, 0, 31 + 20}}, {{0, 31 + 20, 0, 32 + 20}}, {{0, 32 + 20, 0, 33 + 20}},
{{0, 21, 0, 34 + 20}}, {{0, 34 + 20, 0, 35 + 20}}, {{0, 35 + 20, 0, 36 + 20}}, {{0, 36 + 20, 0, 37 + 20}},
{{0, 21, 0, 38 + 20}}, {{0, 38 + 20, 0, 39 + 20}}, {{0, 39 + 20, 0, 40 + 20}}, {{0, 40 + 20, 0, 41 + 20}},
}};
PAF_weight.resize(num_PAF_constraint);
PAF_weight = {{ 150., 150., 150., 150., 150., 150.,
150., 150., 150.,
150., 150., 150.,
0., 0.,
50., 50.,
50., 50.,
50., 50.,
50., 50., 50., 50., // left hand
50., 50., 50., 50.,
50., 50., 50., 50.,
50., 50., 50., 50.,
50., 50., 50., 50.,
50., 50., 50., 50., // right hand
50., 50., 50., 50.,
50., 50., 50., 50.,
50., 50., 50., 50.,
50., 50., 50., 50.
}};
}
else if (regressor_type == 1)
{
m_nCorrespond_adam2joints = fit_data_.adam.h36m_jointConst_smcIdx.size();
int count_vertex = 0;
for (int k = 0; k < fit_data_.adam.m_cocoplus_reg.outerSize(); ++k)
{
for (SparseMatrix<double>::InnerIterator it(fit_data_.adam.m_cocoplus_reg, k); it; ++it)
{
total_vertex.push_back(k);
count_vertex++;
break; // now this vertex is used, go to next vertex
}
}
corres_vertex2targetpt.clear();
m_nCorrespond_adam2pts = 0;
if (fit_data_.fit_surface)
{
for(auto i = 0; i < fit_data_.surface_constraint.cols(); i++)
{
corres_vertex2targetpt.push_back(std::make_pair(i, std::vector<double>(5)));
total_vertex.push_back(fit_data_.surface_constraint(5, i));
m_nCorrespond_adam2pts++;
}
}
PAF_connection.resize(num_PAF_constraint);
PAF_connection = {{ {{2, 12, 2, 2}}, {{2, 2, 2, 1}}, {{2, 1, 2, 0}}, {{2, 12, 2, 3}}, {{2, 3, 2, 4}}, {{2, 4, 2, 5}},
{{2, 12, 2, 8}}, {{2, 8, 2, 7}}, {{2, 7, 2, 6}}, // right arm
{{2, 12, 2, 9}}, {{2, 9, 2, 10}}, {{2, 10, 2, 11}}, // left arm
{{2, 12, 2, 14}}, {{2, 12, 2, 13}}, // neck -> nose, neck -> head top
{{2, 12, 2, 19}}, {{2, 19, 2, 2}}, {{2, 19, 2, 3}},
{{2, 8, 2, 6}}, {{2, 9, 2, 11}}, // shoulder -> wrist
{{2, 2, 2, 6}}, {{2, 3, 2, 11}}, // hip -> wrist
{{2, 2, 2, 0}}, {{2, 3, 2, 5}}, // hip -> ankle
{{0, 20, 0, 22}}, {{0, 22, 0, 23}}, {{0, 23, 0, 24}}, {{0, 24, 0, 25}}, // left hand
{{0, 20, 0, 26}}, {{0, 26, 0, 27}}, {{0, 27, 0, 28}}, {{0, 28, 0, 29}},
{{0, 20, 0, 30}}, {{0, 30, 0, 31}}, {{0, 31, 0, 32}}, {{0, 32, 0, 33}},
{{0, 20, 0, 34}}, {{0, 34, 0, 35}}, {{0, 35, 0, 36}}, {{0, 36, 0, 37}},
{{0, 20, 0, 38}}, {{0, 38, 0, 39}}, {{0, 39, 0, 40}}, {{0, 40, 0, 41}},
{{0, 21, 0, 22 + 20}}, {{0, 22 + 20, 0, 23 + 20}}, {{0, 23 + 20, 0, 24 + 20}}, {{0, 24 + 20, 0, 25 + 20}}, // right hand
{{0, 21, 0, 26 + 20}}, {{0, 26 + 20, 0, 27 + 20}}, {{0, 27 + 20, 0, 28 + 20}}, {{0, 28 + 20, 0, 29 + 20}},
{{0, 21, 0, 30 + 20}}, {{0, 30 + 20, 0, 31 + 20}}, {{0, 31 + 20, 0, 32 + 20}}, {{0, 32 + 20, 0, 33 + 20}},
{{0, 21, 0, 34 + 20}}, {{0, 34 + 20, 0, 35 + 20}}, {{0, 35 + 20, 0, 36 + 20}}, {{0, 36 + 20, 0, 37 + 20}},
{{0, 21, 0, 38 + 20}}, {{0, 38 + 20, 0, 39 + 20}}, {{0, 39 + 20, 0, 40 + 20}}, {{0, 40 + 20, 0, 41 + 20}},
}};
PAF_weight.resize(num_PAF_constraint);
PAF_weight = {{ 150., 150., 150., 150., 150., 150.,
150., 150., 150.,
150., 150., 150.,
50., 50., // neck to nose + neck to headtop
0., 0., 0.,
50., 50.,
50., 50.,
50., 50.,
50., 50., 50., 50., // left hand
50., 50., 50., 50.,
50., 50., 50., 50.,
50., 50., 50., 50.,
50., 50., 50., 50.,
50., 50., 50., 50., // right hand
50., 50., 50., 50.,
50., 50., 50., 50.,
50., 50., 50., 50.,
50., 50., 50., 50.
}};
map_regressor_to_constraint.clear();
for (int i = 0; i < m_nCorrespond_adam2joints; i++) map_regressor_to_constraint[i] = i;
}
else
{
assert(regressor_type == 2);
m_nCorrespond_adam2joints = fit_data_.adam.coco_jointConst_smcIdx.size() + 20 * 2; // COCO keypoints plus fingers
int count_vertex = 0;
for (int k = 0; k < fit_data_.adam.m_small_coco_reg.outerSize(); ++k)
{
for (SparseMatrix<double>::InnerIterator it(fit_data_.adam.m_small_coco_reg, k); it; ++it)
{
total_vertex.push_back(k);
count_vertex++;
break; // now this vertex is used, go to next vertex
}
}
// Set up vertex constraints
corres_vertex2targetpt.clear();
// total_vertex.push_back(14328); //right bigtoe
total_vertex.push_back(14238); //right bigtoe
corres_vertex2targetpt.push_back(std::make_pair(count_vertex++, std::vector<double>(5)));
total_vertex.push_back(14288); //right littletoe
corres_vertex2targetpt.push_back(std::make_pair(count_vertex++, std::vector<double>(5)));
total_vertex.push_back(14357); //right heel
corres_vertex2targetpt.push_back(std::make_pair(count_vertex++, std::vector<double>(5)));
total_vertex.push_back(14361); //right heel
corres_vertex2targetpt.push_back(std::make_pair(count_vertex++, std::vector<double>(5)));
total_vertex.push_back(12239); //left bigtoe
corres_vertex2targetpt.push_back(std::make_pair(count_vertex++, std::vector<double>(5)));
total_vertex.push_back(12289); //left littletoe
corres_vertex2targetpt.push_back(std::make_pair(count_vertex++, std::vector<double>(5)));
total_vertex.push_back(12368); //left heel
corres_vertex2targetpt.push_back(std::make_pair(count_vertex++, std::vector<double>(5)));
total_vertex.push_back(12357); //left heel
corres_vertex2targetpt.push_back(std::make_pair(count_vertex++, std::vector<double>(5)));
// face detector results
for (int r = 0; r < fit_data_.adam.m_correspond_adam2face70_adamIdx.rows(); ++r)
{
int adamVertexIdx = fit_data_.adam.m_correspond_adam2face70_adamIdx(r);
total_vertex.push_back(adamVertexIdx);
corres_vertex2targetpt.push_back(std::make_pair(count_vertex++, std::vector<double>(5)));
}
// surface constraints (e.g., densepose)
if (fit_data_.fit_surface)
{
for(auto i = 0; i < fit_data_.surface_constraint.cols(); i++)
{
total_vertex.push_back(fit_data_.surface_constraint(5, i));
corres_vertex2targetpt.push_back(std::make_pair(count_vertex++, std::vector<double>(5)));
}
}
m_nCorrespond_adam2pts = corres_vertex2targetpt.size();
PAF_connection.resize(num_PAF_constraint);
PAF_connection = {{ {{2, 12, 2, 2}}, {{2, 2, 2, 1}}, {{2, 1, 2, 0}}, {{2, 12, 2, 3}}, {{2, 3, 2, 4}}, {{2, 4, 2, 5}},
{{2, 12, 2, 8}}, {{2, 8, 2, 7}}, {{2, 7, 2, 6}}, // right arm
{{2, 12, 2, 9}}, {{2, 9, 2, 10}}, {{2, 10, 2, 11}}, // left arm
{{2, 12, 2, 14}}, {{2, 12, 2, 13}}, // neck -> nose, neck -> head top
{{2, 12, 2, 19}}, {{2, 19, 2, 2}}, {{2, 19, 2, 3}},
{{2, 8, 2, 6}}, {{2, 9, 2, 11}}, // shoulder -> wrist
{{2, 2, 2, 6}}, {{2, 3, 2, 11}}, // hip -> wrist
{{2, 2, 2, 0}}, {{2, 3, 2, 5}}, // hip -> ankle
{{0, 20, 0, 22}}, {{0, 22, 0, 23}}, {{0, 23, 0, 24}}, {{0, 24, 0, 25}}, // left hand
{{0, 20, 0, 26}}, {{0, 26, 0, 27}}, {{0, 27, 0, 28}}, {{0, 28, 0, 29}},
{{0, 20, 0, 30}}, {{0, 30, 0, 31}}, {{0, 31, 0, 32}}, {{0, 32, 0, 33}},
{{0, 20, 0, 34}}, {{0, 34, 0, 35}}, {{0, 35, 0, 36}}, {{0, 36, 0, 37}},
{{0, 20, 0, 38}}, {{0, 38, 0, 39}}, {{0, 39, 0, 40}}, {{0, 40, 0, 41}},
{{0, 21, 0, 22 + 20}}, {{0, 22 + 20, 0, 23 + 20}}, {{0, 23 + 20, 0, 24 + 20}}, {{0, 24 + 20, 0, 25 + 20}}, // right hand
{{0, 21, 0, 26 + 20}}, {{0, 26 + 20, 0, 27 + 20}}, {{0, 27 + 20, 0, 28 + 20}}, {{0, 28 + 20, 0, 29 + 20}},
{{0, 21, 0, 30 + 20}}, {{0, 30 + 20, 0, 31 + 20}}, {{0, 31 + 20, 0, 32 + 20}}, {{0, 32 + 20, 0, 33 + 20}},
{{0, 21, 0, 34 + 20}}, {{0, 34 + 20, 0, 35 + 20}}, {{0, 35 + 20, 0, 36 + 20}}, {{0, 36 + 20, 0, 37 + 20}},
{{0, 21, 0, 38 + 20}}, {{0, 38 + 20, 0, 39 + 20}}, {{0, 39 + 20, 0, 40 + 20}}, {{0, 40 + 20, 0, 41 + 20}},
}};
PAF_weight.resize(num_PAF_constraint);
PAF_weight = {{ 150., 150., 150., 150., 150., 150.,
150., 150., 150.,
150., 150., 150.,
50., 50., // neck to nose + neck to headtop
0., 0., 0.,
50., 50.,
50., 50.,
50., 50.,
50., 50., 50., 50., // left hand
50., 50., 50., 50.,
50., 50., 50., 50.,
50., 50., 50., 50.,
50., 50., 50., 50.,
50., 50., 50., 50., // right hand
50., 50., 50., 50.,
50., 50., 50., 50.,
50., 50., 50., 50.,
50., 50., 50., 50.
}};
map_regressor_to_constraint.clear();
for (int i = 0; i < 20; i++) map_regressor_to_constraint[i] = i;
}
m_targetPts.resize((m_nCorrespond_adam2joints + m_nCorrespond_adam2pts) * 5);
m_targetPts.setZero();
m_targetPts_weight.resize((m_nCorrespond_adam2joints + m_nCorrespond_adam2pts) * 5);
m_targetPts_weight_buffer.resize((m_nCorrespond_adam2joints + m_nCorrespond_adam2pts) * 5);
// copy the fitting target in place
SetupWeight();
UpdateTarget();
// start counting from PAF
int count_dim = (m_nCorrespond_adam2joints + m_nCorrespond_adam2pts) * res_dim;
start_PAF = count_dim;
if (fit_data_.fitPAF) count_dim += 3 * num_PAF_constraint;
// counting for inner constraints
start_inner = count_dim;
for (auto& n: inner_dim)
{
count_dim += n;
total_inner_dim += n;
}
// setting num_residuals
m_nResiduals = count_dim;
std::cout << "m_nCorrespond_adam2joints " << m_nCorrespond_adam2joints << std::endl;
std::cout << "m_nCorrespond_adam2pts " << m_nCorrespond_adam2pts << std::endl;
std::cout << "m_nResiduals " << m_nResiduals << std::endl;
std::cout << "res_dim " << res_dim << std::endl;
std::cout << "start_2d_dim " << start_2d_dim << std::endl;
std::cout << "start_PAF " << start_PAF << std::endl;
std::cout << "start_inner " << start_inner << std::endl;
CostFunction::set_num_residuals(m_nResiduals);
auto parameter_block_sizes = CostFunction::mutable_parameter_block_sizes();
parameter_block_sizes->clear();
parameter_block_sizes->push_back(3); // Translation
parameter_block_sizes->push_back(TotalModel::NUM_JOINTS * 3); // SMPL Pose
parameter_block_sizes->push_back(TotalModel::NUM_SHAPE_COEFFICIENTS); // SMPL Pose
if (fit_face_exp) parameter_block_sizes->push_back(TotalModel::NUM_EXP_BASIS_COEFFICIENTS); // facial expression
}