void DepthVideoProcessor::bilateralFilter()

in lib/Processor.cpp [183:313]


void DepthVideoProcessor::bilateralFilter(const Params& params) {
  const float depthSigma2 = sqr(params.depthSigma);
  const float colorSigma2 = sqr(params.colorSigma);

  const bool useDepthRange = (params.depthSigma > 0.f);
  const bool useColorRange = (params.colorSigma > 0.f);

  const DepthStream& ds = video_->depthStream(0);
  const ColorStream& cs = video_->colorStream("down");

  for (int frame : params.frameRange) {
    LOG(INFO) << "Processing frame " << frame <<
        " of range " << params.frameRange.toString() << "...";

    // This is the image we're filtering.
    const Mat1f& referenceDepth = *ds.frame(frame).depth();
    const int w = referenceDepth.cols;
    const int h = referenceDepth.rows;

    // We have a reference color image, too.
    const Mat* referenceColor = cs.frame(frame).image();
    CHECK_EQ(referenceColor->cols, w);
    CHECK_EQ(referenceColor->rows, h);

    // The output image.
    Mat1f filteredDepth(h, w);

    // Fetching a stack of (2 * frameRadius + 1) *guidance images*, temporally
    // centered around the current frame.
    const int f0 = std::max(0, frame - params.frameRadius);
    const int f1 = std::min(video_->numFrames()-1, frame + params.frameRadius);
    const int temporalWindowSize = f1 - f0 + 1;
    std::vector<const Mat1f*> guideDepthStack(temporalWindowSize);
    std::vector<const Mat3f*> guideColorStack(temporalWindowSize);
    for (int i = 0; i < temporalWindowSize; ++i) {
      guideDepthStack[i] = ds.frame(f0 + i).depth();
      CHECK_EQ(guideDepthStack[i]->cols, w);
      CHECK_EQ(guideDepthStack[i]->rows, h);
      guideColorStack[i] = cs.frame(f0 + i).image3f();
      CHECK_EQ(guideColorStack[i]->cols, w);
      CHECK_EQ(guideColorStack[i]->rows, h);
    }

    // Vector of samples for the median filter
    std::vector<std::pair<float, float>> depthWeightSamples;

    // Looping over the output image pixels.
    for (int y = 0; y < h; ++y) {
      const int y0 = std::max(0, y - params.spatialRadius);
      const int y1 = std::min(h - 1, y + params.spatialRadius);

      const float* const refDepthPtr =
          (const float* const)referenceDepth.ptr(y);
      const Vec3f* const refColorPtr =
          (const Vec3f* const)referenceColor->ptr(y);
      float* const dstPtr = (float* const)filteredDepth.ptr(y);

      for (int x = 0; x < w; ++x) {
        const int x0 = std::max(0, x - params.spatialRadius);
        const int x1 = std::min(w - 1, x + params.spatialRadius);

        const float referenceDepth = refDepthPtr[x];
        const Vec3f& referenceColor = refColorPtr[x];

        float sumDepth = 0.f;
        float sumWeight = 0.f;

        if (params.median) {
          depthWeightSamples.clear();
        }

        // "Inner loop" over the spatio-temporal kernel.
        for (int wf = 0; wf < temporalWindowSize; ++wf) {
          for (int wy = y0; wy <= y1; ++wy) {
            const float* const guideDepthPtr =
                (const float* const)guideDepthStack[wf]->ptr(wy);
            const Vec3f* const guideColorPtr =
                (const Vec3f* const)guideColorStack[wf]->ptr(wy);

            for (int wx = x0; wx <= x1; ++wx) {
              const float depth = guideDepthPtr[wx];
              float exponent = 0.f;

              if (useDepthRange) {
                const float diff2 = sqr(depth - referenceDepth);
                exponent += -diff2 / depthSigma2;
              }

              if (useColorRange) {
                const Vec3f& color = guideColorPtr[wx];
                float diff2 =
                    sqr(color(0) - referenceColor(0)) +
                    sqr(color(1) - referenceColor(1)) +
                    sqr(color(2) - referenceColor(2));
                exponent += -diff2 / colorSigma2;
              }

              const float weight = (exponent != 0.f ? expf(exponent) : 1.f);

              if (params.median) {
                depthWeightSamples.emplace_back(depth, weight);
              } else {
                sumDepth += depth * weight;
              }
              sumWeight += weight;
            }
          }
        }

        if (params.median) {
          float halfWeight = sumWeight / 2.f;
          std::sort(depthWeightSamples.begin(), depthWeightSamples.end());
          const int N = depthWeightSamples.size();
          float cumWeight = 0.f;
          for (int i = 0; i < N; ++i) {
            const auto& dw = depthWeightSamples[i];
            cumWeight += dw.second;
            if (cumWeight >= halfWeight) {
              dstPtr[x] = dw.first;
              break;
            }
          }
        } else {
          dstPtr[x] = (sumWeight > 0.f ? sumDepth / sumWeight : 0.f);
        }
      }
    }

    video_->depthFrame(params.depthStream, frame).setDepth(filteredDepth);
  }
}