in src/dewrapper/dewrapper.c [181:415]
static int depth_engine_thread(void *param)
{
dewrapper_context_t *dewrapper = (dewrapper_context_t *)param;
k4a_result_t result = K4A_RESULT_SUCCEEDED;
size_t depth_engine_output_buffer_size;
int depth_engine_max_compute_time_ms;
bool received_valid_image = false;
result = TRACE_CALL(depth_engine_start_helper(dewrapper,
dewrapper->fps,
dewrapper->depth_mode,
&depth_engine_max_compute_time_ms,
&depth_engine_output_buffer_size));
// The Start routine is blocked waiting for this thread to complete startup, so we signal it here and share our
// startup status.
Lock(dewrapper->lock);
dewrapper->thread_started = true;
dewrapper->thread_start_result = result;
Condition_Post(dewrapper->condition);
Unlock(dewrapper->lock);
// NOTE: Failures after this point are reported to the user via the k4a_device_get_capture()
while (result != K4A_RESULT_FAILED && dewrapper->thread_stop == false)
{
k4a_capture_t capture = NULL;
k4a_capture_t capture_raw = NULL;
k4a_image_t image_raw = NULL;
k4a_depth_engine_output_frame_info_t outputCaptureInfo = { 0 };
uint8_t *capture_byte_ptr = NULL;
bool cleanup_capture_byte_ptr = true;
shared_image_context_t *shared_image_context = NULL;
uint8_t *raw_image_buffer = NULL;
size_t raw_image_buffer_size = 0;
bool dropped = false;
k4a_wait_result_t wresult = queue_pop(dewrapper->queue, K4A_WAIT_INFINITE, &capture_raw);
if (wresult != K4A_WAIT_RESULT_SUCCEEDED)
{
result = K4A_RESULT_FAILED;
}
if (K4A_SUCCEEDED(result))
{
image_raw = capture_get_ir_image(capture_raw);
result = K4A_RESULT_FROM_BOOL(image_raw != NULL);
}
if (K4A_SUCCEEDED(result))
{
raw_image_buffer = image_get_buffer(image_raw);
raw_image_buffer_size = image_get_size(image_raw);
// Allocate 1 buffer for depth engine to write depth and IR images to
assert(depth_engine_output_buffer_size != 0);
capture_byte_ptr = allocator_alloc(ALLOCATION_SOURCE_DEPTH, depth_engine_output_buffer_size);
if (capture_byte_ptr == NULL)
{
LOG_ERROR("Depth streaming callback failed to allocate output buffer", 0);
result = K4A_RESULT_FAILED;
}
}
if (K4A_SUCCEEDED(result))
{
tickcounter_ms_t start_time = 0;
tickcounter_ms_t stop_time = 0;
tickcounter_get_current_ms(dewrapper->tick, &start_time);
k4a_depth_engine_result_code_t deresult =
deloader_depth_engine_process_frame(dewrapper->depth_engine,
raw_image_buffer,
raw_image_buffer_size,
K4A_DEPTH_ENGINE_OUTPUT_TYPE_Z_DEPTH,
capture_byte_ptr,
depth_engine_output_buffer_size,
&outputCaptureInfo,
NULL);
tickcounter_get_current_ms(dewrapper->tick, &stop_time);
if (deresult == K4A_DEPTH_ENGINE_RESULT_FATAL_ERROR_WAIT_PROCESSING_COMPLETE_FAILED ||
deresult == K4A_DEPTH_ENGINE_RESULT_FATAL_ERROR_GPU_TIMEOUT)
{
LOG_ERROR("Timeout during depth engine process frame.", 0);
LOG_ERROR("SDK should be restarted since it looks like GPU has encountered an unrecoverable error.", 0);
dropped = true;
result = K4A_RESULT_FAILED;
}
else if (deresult != K4A_DEPTH_ENGINE_RESULT_SUCCEEDED)
{
LOG_ERROR("Depth engine process frame failed with error code: %d.", deresult);
result = K4A_RESULT_FAILED;
}
else if ((stop_time - start_time) > (unsigned)depth_engine_max_compute_time_ms)
{
LOG_WARNING("Depth image processing is too slow at %lldms (this may be transient).",
stop_time - start_time);
}
}
if (K4A_SUCCEEDED(result) && received_valid_image && outputCaptureInfo.center_of_exposure_in_ticks == 0)
{
// We drop samples with a timestamp of zero when starting up.
LOG_WARNING("Dropping depth image due to bad timestamp at startup", 0);
dropped = true;
result = K4A_RESULT_FAILED;
}
if (K4A_SUCCEEDED(result))
{
shared_image_context = (shared_image_context_t *)malloc(sizeof(shared_image_context_t));
result = K4A_RESULT_FROM_BOOL(shared_image_context != NULL);
}
if (K4A_SUCCEEDED(result))
{
shared_image_context->ref = 0;
shared_image_context->buffer = capture_byte_ptr;
result = TRACE_CALL(capture_create(&capture));
}
bool depth16_present = (dewrapper->depth_mode == K4A_DEPTH_MODE_NFOV_2X2BINNED ||
dewrapper->depth_mode == K4A_DEPTH_MODE_NFOV_UNBINNED ||
dewrapper->depth_mode == K4A_DEPTH_MODE_WFOV_2X2BINNED ||
dewrapper->depth_mode == K4A_DEPTH_MODE_WFOV_UNBINNED);
if (K4A_SUCCEEDED(result) & depth16_present)
{
k4a_image_t image;
int stride_bytes = (int)outputCaptureInfo.output_width * (int)sizeof(uint16_t);
result = TRACE_CALL(image_create_from_buffer(K4A_IMAGE_FORMAT_DEPTH16,
outputCaptureInfo.output_width,
outputCaptureInfo.output_height,
stride_bytes,
capture_byte_ptr,
(size_t)stride_bytes * (size_t)outputCaptureInfo.output_height,
free_shared_depth_image,
shared_image_context,
&image));
if (K4A_SUCCEEDED(result))
{
cleanup_capture_byte_ptr = false; // buffer is now owned by image;
INC_REF_VAR(shared_image_context->ref);
image_set_device_timestamp_usec(image,
K4A_90K_HZ_TICK_TO_USEC(outputCaptureInfo.center_of_exposure_in_ticks));
image_set_system_timestamp_nsec(image, image_get_system_timestamp_nsec(image_raw));
capture_set_depth_image(capture, image);
image_dec_ref(image);
}
}
if (K4A_SUCCEEDED(result))
{
k4a_image_t image;
int stride_bytes = (int)outputCaptureInfo.output_width * (int)sizeof(uint16_t);
uint8_t *image_buf = capture_byte_ptr;
if (depth16_present)
{
image_buf = image_buf + stride_bytes * outputCaptureInfo.output_height;
}
result = TRACE_CALL(image_create_from_buffer(K4A_IMAGE_FORMAT_IR16,
outputCaptureInfo.output_width,
outputCaptureInfo.output_height,
stride_bytes,
image_buf,
(size_t)stride_bytes * (size_t)outputCaptureInfo.output_height,
free_shared_depth_image,
shared_image_context,
&image));
if (K4A_SUCCEEDED(result))
{
cleanup_capture_byte_ptr = false; // buffer is now owned by image;
INC_REF_VAR(shared_image_context->ref);
image_set_device_timestamp_usec(image,
K4A_90K_HZ_TICK_TO_USEC(outputCaptureInfo.center_of_exposure_in_ticks));
image_set_system_timestamp_nsec(image, image_get_system_timestamp_nsec(image_raw));
capture_set_ir_image(capture, image);
image_dec_ref(image);
}
}
if (K4A_SUCCEEDED(result))
{
// set capture attributes
capture_set_temperature_c(capture, outputCaptureInfo.sensor_temp);
received_valid_image = true;
dewrapper->capture_ready_cb(result, capture, dewrapper->capture_ready_cb_context);
}
if (shared_image_context && shared_image_context->ref == 0)
{
// It didn't get used due to a failure
free(shared_image_context);
}
if (capture)
{
capture_dec_ref(capture);
}
if (capture_raw)
{
capture_dec_ref(capture_raw);
}
if (image_raw)
{
image_dec_ref(image_raw);
image_raw = NULL;
}
if (capture_byte_ptr && cleanup_capture_byte_ptr)
{
allocator_free(capture_byte_ptr);
}
if (dropped)
{
// It is not a fatal error when we drop a frame, so we reset 'result' so that we can continue to run.
result = K4A_RESULT_SUCCEEDED;
}
}
if (K4A_FAILED(result))
{
dewrapper->capture_ready_cb(result, NULL, dewrapper->capture_ready_cb_context);
}
depth_engine_stop_helper(dewrapper);
// This will always return failure, because stop is trigged by the queue being disabled
return (int)result;
}