source/rig/RigAnalyzer.cpp (508 lines of code) (raw):
/**
* Copyright 2004-present Facebook. All Rights Reserved.
*
* This source code is licensed under the BSD-style license found in the
* LICENSE file in the root directory of this source tree.
*/
#include <fstream>
#include <iostream>
#include <gflags/gflags.h>
#include <glog/logging.h>
#include "source/util/Camera.h"
#include "source/util/SystemUtil.h"
using namespace fb360_dep;
const std::string kUsageMessage = R"(
- Miscellaneous analysis utilities for a rig. Various output formats are supported to
visualize the rig setup (e.g. equirect projection).
- Example:
./RigAnalyzer \
--rig=/path/to/rigs/rig.json \
--output_equirect=/path/to/output/equirect.png
)";
DEFINE_double(custom, -1, "custom angle away from north");
DEFINE_double(discard_poles, 0, "degrees from poles to ignore");
DEFINE_string(eulers, "", "create from eulers file");
DEFINE_double(min_distance, 0.50, "min distance to test");
DEFINE_double(
overlap_distance,
Camera::kNearInfinity,
"distance to visualize equirect overlap, default is INF");
DEFINE_bool(one_based_indexing, false, "enable to index cameras starting at 1 instead of 0");
DEFINE_string(output_camera, "", "path to output camera .ppm file");
DEFINE_string(output_camera_id, "", "output camera id");
DEFINE_string(output_cross_section, "", "path to output cross section .ppm file");
DEFINE_string(output_equirect, "", "path to output equirect .ppm file");
DEFINE_string(output_obj, "", "path to output rig .obj file");
DEFINE_string(output_rig, "", "path to output rig .json file");
DEFINE_bool(perturb_cameras, false, "");
DEFINE_double(perturb_focals, 0, "pertub focals");
DEFINE_double(perturb_positions, 0, "perturb positions (cm)");
DEFINE_double(perturb_principals, 0, "pertub principals (pixels)");
DEFINE_double(perturb_rotations, 0, "perturb rotations (radians)");
DEFINE_int32(perturb_seed, 1, "seed for perturb cameras. Default: 1, same as no seed");
DEFINE_double(radius, 0, "change rig radius");
DEFINE_string(
rearrange,
"",
"create specific arrangement (ballcam24, tetra, ring4, cube, carbon0, carbon1, diamond)");
DEFINE_string(revolve, "", "create from angle file");
DEFINE_string(rig, "", "path to rig .json file (required)");
DEFINE_string(rotate, "", "rotate rig by euler angles");
DEFINE_string(rotate_cam_z, "", "rotate camera to align with z");
DEFINE_int32(sample_count, 100000, "number of samples");
DEFINE_double(scale_resolution, 1, "scale camera resolutions");
DEFINE_bool(show_timing, false, "visualize time as well as spatial overlap");
DEFINE_bool(z_is_down, false, "modify rig from y-is-up to z-is-down");
DEFINE_bool(z_is_up, false, "modify rig from y-is-up to z-is-up");
DEFINE_double(scale_rig, 1, "scale rig space, e.g., by 1e-2 to convert from cm to m");
// generates a fairly uniform distribution that is very regular
std::vector<Camera::Vector3> getFibonacciUnits(int count) {
std::vector<Camera::Vector3> points;
for (int i = 0; i < count; ++i) {
Camera::Real y = (i + 0.5) / count * 2 - 1;
Camera::Real r = sqrt(1 - y * y);
Camera::Real phi = (1 + sqrt(5)) / 2;
Camera::Real roty = i / phi * 2 * M_PI;
points.emplace_back(sin(roty) * r, y, cos(roty) * r);
}
return points;
}
// discard points that are within radians of the z axis
// samples are assumed to be unit vectors
std::vector<Camera::Vector3> discardPoles(
const std::vector<Camera::Vector3>& samples,
const Camera::Real radians) {
const Camera::Real threshold = cos(radians);
std::vector<Camera::Vector3> result;
for (const Camera::Vector3& sample : samples) {
if (std::abs(sample.z()) < threshold) {
result.emplace_back(sample);
}
}
return result;
}
Camera::Matrix3 rotationMatrixFromEulers(const Camera::Vector3& euler, bool xyz = true) {
Eigen::AngleAxis<Camera::Real> x(euler.x(), Camera::Vector3::UnitX());
Eigen::AngleAxis<Camera::Real> y(euler.y(), Camera::Vector3::UnitY());
Eigen::AngleAxis<Camera::Real> z(euler.z(), Camera::Vector3::UnitZ());
return (xyz ? z * y * x : y * x * z).toRotationMatrix();
}
Camera::Rig
makeRigFromEulers(const Camera& model, const std::vector<Camera::Vector3>& eulers, bool xyz) {
// Note/instruction:
// - place camera at 0,0,1
// - pointing in direction 0,0,1 in landscape
// - i.e. sensor's wide direction is the 1,0,0 direction
// - and up is 0,1,0
// if xyz is false:
// - rotate around the z axis by the third number
// - then rotate around the x axis by the first number
// - finally rotate around the y axis by the second number
// - *please* note the order: z, x, y
// if xyz is true:
// - rotate around the x axis by the first number
// - then rotate around the y axis by the second number
// - finally rotate around the z axis by the third number
Camera::Rig result;
for (Camera::Vector3 euler : eulers) {
euler *= M_PI / 180;
Camera::Matrix3 xform = rotationMatrixFromEulers(euler, xyz);
Camera camera = model;
camera.setRotation(xform.col(2), xform.col(1), -xform.col(0));
camera.position = model.position.norm() * camera.forward();
camera.id = "cam" + std::to_string(result.size() + (FLAGS_one_based_indexing ? 1 : 0));
result.emplace_back(camera);
}
return result;
}
Camera::Rig revolveRig(const Camera::Rig& rig, const std::vector<Camera::Vector3>& eulers) {
Camera::Rig result;
for (int frame = 0; frame < int(eulers.size()); ++frame) {
const Camera::Vector3& euler = eulers[frame];
Eigen::AngleAxis<Camera::Real> x(euler.x(), Camera::Vector3::UnitX());
Eigen::AngleAxis<Camera::Real> y(euler.y(), Camera::Vector3::UnitY());
Eigen::AngleAxis<Camera::Real> z(euler.z(), Camera::Vector3::UnitZ());
Camera::Matrix3 xform = (z * y * x).toRotationMatrix();
for (Camera camera : rig) {
// transform rotation and position
Camera::Vector3 forward = camera.forward();
Camera::Vector3 up = camera.up();
Camera::Vector3 right = camera.right();
camera.setRotation(xform * forward, xform * up, xform * right);
camera.position = xform * camera.position;
if (eulers.size() > 1) {
camera.id.append('_' + std::to_string(frame));
}
result.push_back(camera);
}
}
return result;
}
Camera::Rig makeBallcam24(const Camera& model) {
return makeRigFromEulers(
model,
{{22.998, -36.1543, 132.267}, {-2.89381, -156.601, 168.482},
{-50.2907, -68.7384, 139.028}, {-80.2662, 172.721, 113.889},
{57.5173, 87.6811, 161.596}, {6.46204, 162.32, 70.7419},
{21.8577, 118.439, 114.195}, {77.4316, -95.0674, -100.379},
{-20.2739, 41.1554, -135.466}, {-38.2009, 172.776, -171.825},
{-0.841465, -110.909, 57.8619}, {-39.8563, -128.178, 46.3619},
{-54.3882, 8.6561, -13.3586}, {24.3104, 51.5133, -20.0308},
{35.7198, -82.6713, 160.228}, {-48.4447, 85.1941, 93.5637},
{48.4425, 165.464, 19.7297}, {-3.41527, 84.0526, 56.5226},
{-20.5666, -24.4286, 14.2745}, {35.8214, -139.006, -27.4138},
{-8.22831, -69.3313, -46.6214}, {51.5282, 4.18718, -133.303},
{6.61383, 8.24745, -72.7674}, {-22.4038, 126.995, 13.7087}},
false);
}
Camera::Rig makeTetraTilted(const Camera& model) {
// === arrangement 1280: mean = 2.14859, quality = 1.64213, stddev = 0.554116, min = 1, max = 3
return makeRigFromEulers(
model,
{{-35.2644, 45, -65.1818},
{-35.2644, -135, -137.834},
{35.2644, -45, -45.0048},
{35.2644, 135, -104.664}},
false);
}
Camera::Rig makeCarbon0(const Camera& model) {
// Lens: Izugar MKX22, 220 degree fov, 10 mm image diameter
// Sensor: Sony IMX264, 2448 x 2048 x 3.45 um
// === arrangement 789: mean = 3.22289, quality = 2.89455, stddev = 0.368871
return makeRigFromEulers(
model,
{{-35.2644, 3.89537e-15, 112.232},
{-35.2644, 120, -67.3096},
{-35.2644, -120, 155.867},
{35.2644, 180, 21.9328},
{35.2644, -60, 14.0236},
{35.2644, 60, 66.2737}},
false);
}
Camera::Rig makeCarbon1(const Camera& model) {
// Lens: Entaniya M12, 220 degree fov, 5.1 mm image diameter
// Sensor: Sony IMX377, 4000 x 3000 x 1.55 um
// === arrangement 449: mean = 3.93396, quality = 3.36985, stddev = 0.602308, min = 3, max = 5
return makeRigFromEulers(
model,
{{-35.2644, 1.94768e-15, 133.504},
{-35.2644, 120, -179.989},
{-35.2644, -120, -134.51},
{35.2644, 180, 89.7419},
{35.2644, -60, 43.7899},
{35.2644, 60, -45.1612}},
false);
}
Camera::Rig makeTetra(const Camera& model, double angle) {
Camera::Real a = angle == -1 ? acos(-1 / 3.0) * 180 / M_PI : angle;
// note: default is almost 110
// note: 140 is the angle at which everything below 3 is above the horizon
return makeRigFromEulers(model, {{a, 0, 0}, {a, 0, 120}, {a, 0, -120}, {0, 0, 0}}, true);
}
Camera::Rig makeCube(const Camera& model, double angle) {
Camera::Real a = (angle == -1 ? 90 : angle);
return makeRigFromEulers(
model, {{a, 0, 0}, {a, 0, 90}, {a, 0, 180}, {a, 0, 270}, {0, 0, 0}, {180, 0, 0}}, true);
}
Camera::Rig makeDiamond(const Camera& model, double angle) {
Camera::Real a = (angle == -1 ? 90 : angle);
return makeRigFromEulers(
model, {{a, 0, 0}, {a, 0, 120}, {a, 0, 240}, {0, 0, 0}, {180, 0, 0}}, true);
}
Camera::Rig makeRing4(const Camera& model, double angle) {
Camera::Real a = (angle == -1 ? 90 : angle);
return makeRigFromEulers(model, {{a, 0, 0}, {a, 0, 90}, {a, 0, 180}, {a, 0, 270}}, true);
}
Camera::Rig
makeNamedArrangement(const std::string& name, const Camera& model, const double custom) {
if (name == "ballcam24") {
return makeBallcam24(model);
} else if (name == "tetra") {
return makeTetra(model, custom);
} else if (name == "tetratilted") {
return makeTetraTilted(model);
} else if (name == "ring4") {
return makeRing4(model, custom);
} else if (name == "cube") {
return makeCube(model, custom);
} else if (name == "carbon0") {
return makeCarbon0(model);
} else if (name == "carbon1") {
return makeCarbon1(model);
}
CHECK_EQ(name, "diamond") << "unknown arrangement";
return makeDiamond(model, custom);
}
std::string getHistogram(const Eigen::VectorXd& coverages) {
std::string result;
int last = coverages.maxCoeff();
for (int i = 0; i <= last; ++i) {
int count = (coverages.array() == i).count();
result += "h[" + std::to_string(i) + "] = " + std::to_string(count) + ", ";
}
return result;
}
static void
writeVertexObj(std::ofstream& file, const Camera::Vector3& color, const Camera::Vector3& position) {
const Camera::Real kScale = 1000; // CAD wants mm, json is meters
file << "v";
for (int i = 0; i < int(position.size()); ++i) {
file << " " << kScale * position[i];
}
for (int i = 0; i < int(color.size()); ++i) {
file << " " << color[i];
}
file << "\n";
}
static void writeFaceObj(
std::ofstream& file,
const Camera::Vector3& color,
const std::vector<Camera::Vector3>& positions) {
// write each vertex
for (int i = 0; i < int(positions.size()); ++i) {
writeVertexObj(file, color, positions[i]);
}
// write the face in both orders to avoid backface culling
for (int order = 0; order < 2; ++order) {
file << "f";
for (int i = 0; i < int(positions.size()); ++i) {
int index = order ? -int(positions.size()) + i : -1 - i;
file << " " << index;
}
file << "\n";
}
}
static void writeArrowObj(
std::ofstream& file,
const Camera::Vector3& color,
const Camera::Vector3& base,
const Camera::Vector3& dir,
const Camera::Vector3& t0,
const Camera::Vector3& t1,
const Camera::Real length = 0.01,
const Camera::Real radius = 0.001) {
writeFaceObj(file, color, {base + length * dir, base + radius * t0, base - radius * t0});
writeFaceObj(file, color, {base + length * dir, base + radius * t1, base - radius * t1});
}
static void writeCameraObj(
std::ofstream& file,
const Camera::Vector3& p,
const Camera::Vector3& f,
const Camera::Vector3& r,
const Camera::Vector3& u) {
// a camera is represented by white forward, green right and blue up arrows
writeArrowObj(file, {1, 1, 1}, p, f, r, u, 0.02);
writeArrowObj(file, {0, 1, 0}, p, r, u, f);
writeArrowObj(file, {0, 0, 1}, p, u, f, r);
}
void saveRigObj(const std::string& filename, const Camera::Rig& rig) {
std::ofstream file(filename);
for (int i = 0; i < int(rig.size()); ++i) {
const Camera::Vector3& p = rig[i].position;
const Camera::Vector3& f = rig[i].forward();
const Camera::Vector3& r = rig[i].right();
const Camera::Vector3& u = rig[i].up();
writeCameraObj(file, p, f, r, u);
for (int tri = 0; tri < i; ++tri) { // output the index as i triangles
const double kSize = 0.002;
const Camera::Vector3 v = p - kSize * tri * r;
writeFaceObj(file, {1, 0, 0}, {v, v - kSize * r, v - kSize * u});
}
}
// draw arrow from {0,0,-1} to {0,0,0}
writeArrowObj(file, {1, 1, 0}, {0, 0, -1}, {0, 0, 1}, {1, 0, 0}, {0, 1, 0}, 1.0, 0.01);
}
void saveCamera(const std::string& filename, const std::string& camId, const Camera::Rig& rig) {
for (const Camera& cam : rig) {
if (cam.id != camId) {
continue;
}
const int kDimX = cam.resolution.x();
const int kDimY = cam.resolution.y();
std::ofstream file(filename, std::ios::binary);
file << "P2" << std::endl;
file << kDimX << " " << kDimY << std::endl;
file << rig.size() << std::endl;
for (int y = 0; y < kDimY; ++y) {
for (int x = 0; x < kDimX; ++x) {
const Camera::Vector2 p(x + 0.5, y + 0.5);
int count = 0;
if (!cam.isOutsideImageCircle(p)) {
const Camera::Vector3 pWorld = cam.rig(p, FLAGS_overlap_distance);
for (const Camera& camera : rig) {
if (camera.sees(pWorld)) {
++count;
}
}
}
file << count << " ";
}
file << std::endl;
}
}
}
void saveEquirect(const std::string& filename, const Camera::Rig& rig) {
const int kPixelsPerDegree = 5;
const int kDimX = 360 * kPixelsPerDegree;
const int kDimY = 180 * kPixelsPerDegree;
std::ofstream file(filename, std::ios::binary);
file << "P2" << std::endl;
file << kDimX << " " << kDimY << std::endl;
if (FLAGS_show_timing) {
file << 256 << std::endl;
} else {
file << rig.size() << std::endl;
}
double holes = 0;
double maxMin = 0;
double aveMin = 0;
for (int y = 0; y < kDimY; ++y) {
// latitude goes from pi/2 down to -pi/2
Camera::Real lat = M_PI / 2 - (y + 0.5) / kDimY * M_PI;
for (int x = 0; x < kDimX; ++x) {
// lon goes from -pi up to pi
Camera::Real lon = -M_PI + (x + 0.5) / kDimX * 2 * M_PI;
Camera::Vector3 direction(cos(lat) * cos(lon), cos(lat) * sin(lon), sin(lat));
std::vector<float> timing(rig.size());
int count = 0;
for (const Camera& camera : rig) {
Camera::Vector2 p;
if (camera.sees(direction * FLAGS_overlap_distance, p)) {
// Normalized timing distance assuming all the cameras'
// pixel clocks are synced and single line activated/reset rolling
// shutter.
timing[count] = p.y() / camera.resolution.y();
// Count this camera
++count;
}
}
double minTimingDiff = 1.0;
double maxTimingDiff = 0.0;
for (int i = 0; i < count; ++i) {
for (int j = i + 1; j < count; ++j) {
const double timingDiff = std::abs(timing[i] - timing[j]);
minTimingDiff = std::min(minTimingDiff, timingDiff);
maxTimingDiff = std::max(maxTimingDiff, timingDiff);
}
}
maxMin = std::max(maxMin, minTimingDiff);
aveMin += minTimingDiff;
if (FLAGS_show_timing) {
const int timeWeightedCount = int((1.0 - minTimingDiff) * 255.0);
file << timeWeightedCount << " ";
} else {
file << count << " ";
}
holes += (0 == count) ? 1 : 0;
}
file << std::endl;
}
const float kFrameRate = 60.0f; // 60 fps
const float kFrameTime = 1000.0f / kFrameRate; // in milliseconds
LOG(INFO) << "Holes found (in pixels) = " << holes;
LOG(INFO) << "Max of min timing distance = " << kFrameTime * maxMin << "ms";
LOG(INFO) << "Ave of min timing distance = " << kFrameTime * aveMin / (kDimX * kDimY) << "ms";
}
void saveCrossSection(const std::string& filename, const Camera::Rig& rig) {
const int kDim = 400;
std::ofstream file(filename, std::ios::binary);
file << "P2" << std::endl;
file << kDim << " " << kDim << std::endl;
file << rig.size() << std::endl;
for (int y = 0; y < kDim; ++y) {
for (int x = 0; x < kDim; ++x) {
// points are distributed within +/-0.5 * (kDim, kDim)
Camera::Vector3 point(x + 0.5 - 0.5 * kDim, y + 0.5 - 0.5 * kDim, 0);
int count = 0;
for (const Camera& camera : rig) {
if (camera.sees(point)) {
++count;
}
}
file << count << " ";
}
file << std::endl;
}
}
std::vector<Camera::Vector3> readVectorFile(const std::string& filename) {
std::vector<Camera::Vector3> result;
std::ifstream file(filename);
for (std::string line; getline(file, line);) {
if (line.find("===") == 0) { // ignore lines beginning with '==='
continue;
}
std::istringstream s(line);
Camera::Vector3 angles;
s >> angles[0] >> angles[1] >> angles[2];
CHECK(s) << "bad line <" << line << "> in file " << filename;
result.push_back(angles);
}
return result;
}
int main(int argc, char* argv[]) {
system_util::initDep(argc, argv, kUsageMessage);
CHECK_NE(FLAGS_rig, "");
// Read the cameras
Camera::Rig rig = Camera::loadRig(FLAGS_rig);
// Modify rig
if (!FLAGS_rearrange.empty()) { // clone rig[0] into named configuration
rig = makeNamedArrangement(FLAGS_rearrange, rig[0], FLAGS_custom);
} else if (!FLAGS_eulers.empty()) { // clone first camera according to eulers
rig = makeRigFromEulers(rig[0], readVectorFile(FLAGS_eulers), false);
} else if (!FLAGS_revolve.empty()) {
rig = revolveRig(rig, readVectorFile(FLAGS_revolve));
} else if (FLAGS_perturb_cameras) {
std::srand(FLAGS_perturb_seed);
Camera::perturbCameras(
rig,
FLAGS_perturb_positions,
FLAGS_perturb_rotations,
FLAGS_perturb_principals,
FLAGS_perturb_focals);
}
if (!FLAGS_rotate_cam_z.empty()) {
const Camera& zCam = Camera::findCameraById(FLAGS_rotate_cam_z, rig);
const Camera::Vector3 z = Eigen::Vector3d::UnitZ();
Camera::Real angle = acos(zCam.position.dot(z));
Camera::Vector3 axis = zCam.position.cross(z);
axis.normalize();
Eigen::AngleAxis<Camera::Real> alignCamToZ(angle, axis);
Camera::Matrix3 rot = alignCamToZ.toRotationMatrix();
for (Camera& camera : rig) {
LOG(INFO) << camera.forward();
LOG(INFO) << rot * camera.forward();
camera.position = rot * camera.position;
camera.setRotation(rot * camera.forward(), rot * camera.up(), rot * camera.right());
}
}
if (FLAGS_z_is_up || FLAGS_z_is_down || !FLAGS_rotate.empty()) {
Camera::Matrix3 m;
if (FLAGS_z_is_up) {
m << 1, 0, 0, 0, 0, -1, 0, 1, 0;
} else if (FLAGS_z_is_down) {
m << 1, 0, 0, 0, 0, 1, 0, -1, 0;
} else {
Camera::Vector3 euler;
std::istringstream s(FLAGS_rotate);
s >> euler.x() >> euler.y() >> euler.z();
CHECK(s.eof() && !s.fail()) << "bad --rotate vector " << FLAGS_rotate;
m = rotationMatrixFromEulers(euler);
}
for (Camera& camera : rig) {
camera.position = m * camera.position;
camera.setRotation(m * camera.forward(), m * camera.up(), m * camera.right());
}
}
if (FLAGS_scale_rig != 1) {
LOG(INFO) << "scaling rig by " << FLAGS_scale_rig;
for (Camera& camera : rig) {
camera.position *= FLAGS_scale_rig;
}
}
if (FLAGS_radius > 0) {
for (Camera& camera : rig) {
camera.position = FLAGS_radius * camera.position.normalized();
}
}
if (FLAGS_scale_resolution != 1) {
for (Camera& camera : rig) {
camera = camera.rescale(FLAGS_scale_resolution * camera.resolution);
}
}
// Generate the directions that we want to test
std::vector<Camera::Vector3> samples = getFibonacciUnits(FLAGS_sample_count);
samples = discardPoles(samples, FLAGS_discard_poles * M_PI / 180);
// Go through N distances from min_distance to kNearInfinity
const int kN = 20;
for (int i = 0; i < kN; ++i) {
Camera::Real frac = i / Camera::Real(kN);
Camera::Real distance = FLAGS_min_distance / (1 - frac);
// Compute coverage for each sample
Eigen::VectorXd coverages(samples.size());
for (int i = 0; i < int(samples.size()); ++i) {
int coverage = 0;
for (const Camera& camera : rig) {
if (camera.sees(distance * samples[i])) {
++coverage;
}
}
coverages[i] = coverage;
}
// Report results
const int minC = coverages.minCoeff();
double quality = minC + (coverages.array() >= minC + 1).count() / double(coverages.size());
std::cout << folly::format(
"distance: {:.2f} quality: {:.2f} samples: {} {}",
distance,
quality,
ssize(coverages),
getHistogram(coverages))
<< std::endl;
}
// Write the cameras
if (FLAGS_output_rig != "") {
Camera::saveRig(FLAGS_output_rig, rig, {"command line:", gflags::GetArgv()});
}
if (FLAGS_output_obj != "") {
saveRigObj(FLAGS_output_obj, rig);
}
if (FLAGS_output_equirect != "") {
saveEquirect(FLAGS_output_equirect, rig);
}
if (FLAGS_output_camera != "" && FLAGS_output_camera_id != "") {
saveCamera(FLAGS_output_camera, FLAGS_output_camera_id, rig);
}
if (FLAGS_output_cross_section != "") {
saveCrossSection(FLAGS_output_cross_section, rig);
}
return 0;
}