void DepthVideoImporter::importColmapRecon()

in lib/Importer.cpp [290:388]


void DepthVideoImporter::importColmapRecon(
    DepthVideo& video, const std::string &colmapFile,
    const int stream, const bool silent) {
  DepthStream& ds = video.depthStream(stream);

  float scale = loadScale(video.path());

  // Load indices of frames that have been reconstructed.
  std::vector<int> frameIndices;
  const std::string depthPath = ds.path() + "/depth";
  for (auto& entry :
      boost::make_iterator_range(fs::directory_iterator(depthPath), {})) {
    std::string name = entry.path().stem().string();
    if (name.length() != 12 || name.substr(0, 6) != "frame_") {
      throw std::runtime_error(
          "Depth file name does not have expected format.");
    }

    const int index = std::stoi(name.substr(6));
    frameIndices.push_back(index);
  }
  std::sort(frameIndices.begin(), frameIndices.end());

  // Initially disable all frames, later enable the ones that are reconstructed.
  for (int i = 0; i < video.numFrames(); ++i) {
    ds.frame(i).enabled = false;
  }

  // Load metadata.
  cnpy::npz_t meta = cnpy::npz_load(colmapFile);
  cnpy::NpyArray extrArr = meta["extrinsics"];
  cnpy::NpyArray intrArr = meta["intrinsics"];
  const std::vector<size_t>& extrShape = extrArr.shape;
  const std::vector<size_t>& intrShape = intrArr.shape;
  constexpr int DIM = 3;
  CHECK_EQ(extrShape[0], intrShape[0]);
  CHECK_EQ(extrShape[0], frameIndices.size());
  CHECK_EQ(extrShape.size(), 3);
  CHECK_EQ(extrShape[1], DIM);
  CHECK_EQ(extrShape[2], DIM + 1);
  CHECK_EQ(intrShape.size(), 2);
  CHECK_EQ(intrShape[1], 4);

  // We only support row-major storage and double values for now.
  constexpr int DOUBLE_SIZE = 8;
  CHECK(!extrArr.fortran_order);
  CHECK(!intrArr.fortran_order);
  CHECK_EQ(extrArr.word_size, DOUBLE_SIZE);
  CHECK_EQ(intrArr.word_size, DOUBLE_SIZE);

  using dtype_t = double;
  using dst_dtype_t = float;

  // Load extrinsics. The coordinate system for both the bundles here and the
  // numpy metadata file is +x pointing to the right, +y pointing up and camera
  // facing at -z direction.
  if (!silent) {
    LOG(INFO) << "Loading Extrinsics...";
  }
  using Pose = Matrix<dtype_t, DIM, DIM + 1, RowMajor>;
  const dtype_t* extrPtr = extrArr.data<dtype_t>();
  int extrElementSize = extrShape[1] * extrShape[2];
  for (int i = 0; i < extrShape[0]; ++i, extrPtr += extrElementSize) {
    const Pose pose = Map<const Pose>(extrPtr);
    DepthPhoto::Extrinsics extr;
    extr.position = pose.col(DIM).cast<dst_dtype_t>() / scale;
    extr.orientation = Quaternionf(
      pose.block<DIM, DIM>(0, 0).cast<dst_dtype_t>());
    DepthFrame& df = video.depthFrame(stream, frameIndices[i]);
    df.enabled = true;
    df.extrinsics = extr;
    if (!silent) {
      LOG(INFO) << "  Frame" << i;
      LOG(INFO) << "    " << extr.position;
      LOG(INFO) << "    " << pose.block<DIM, DIM>(0, 0).cast<dst_dtype_t>();
      LOG(INFO) << "    " << extr.orientation.coeffs();
    }
  }

  // set intrinsics
  if (!silent) {
    LOG(INFO) << "Loading Intrinsics...";
  }
  const dtype_t* intrPtr = intrArr.data<dtype_t>();
  cv::Size imSize(ds.width(), ds.height());
  for (int i = 0; i < intrShape[0]; ++i, intrPtr += intrShape[1]) {
    const Vector2d fxy = Map<const Vector2d>(intrPtr);
    DepthPhoto::Intrinsics intr;
    intr.hFov = 2 * atan2(imSize.width / 2.0, fxy.x());
    intr.vFov = 2 * atan2(imSize.height / 2.0, fxy.y());
    video.depthFrame(stream, frameIndices[i]).intrinsics = intr;

    if (!silent) {
      LOG(INFO) << "  Frame " << i;
      LOG(INFO) << "    Fxy " << fxy;
      intr.printParams();
    }
  }
}