in cpp/acs_agent_client.cc [123:172]
absl::Status AcsAgentClient::AddRequest(Request& request) {
absl::Status latest_send_request_status = absl::OkStatus();
// TODO: Make the retry parameters configurable.
for (int i = 0; i < 5; ++i) {
// Generate a new message id for each attempt.
{
absl::MutexLock lock(&reactor_mtx_);
request.set_message_id(CreateMessageUuid());
}
latest_send_request_status = AddRequestAndWaitForResponse(request);
if (latest_send_request_status.ok()) {
return absl::OkStatus();
}
if (latest_send_request_status.code() == absl::StatusCode::kAlreadyExists) {
// Retry to send the request because the request was not even buffered in
// the reactor.
ABSL_VLOG(2) << absl::StrFormat(
"Failed to add message with id: %s to reactor as the ongoing write "
"takes. Retrying with a sleep delay.",
request.message_id());
absl::SleepFor(absl::Seconds(1));
continue;
}
if (latest_send_request_status.code() ==
absl::StatusCode::kDeadlineExceeded) {
// Give up retrying to send the request because the wait for the response
// timed out. This generally means the server is not responding, and the
// client should re-create the connection.
ABSL_VLOG(1) << absl::StrFormat(
"Successfully added message with id: %s to reactor, but timed out "
"waiting for response from server. Please re-create the connection.",
request.message_id());
break;
}
if (latest_send_request_status.code() ==
absl::StatusCode::kResourceExhausted) {
// Retry to send the request because the resource exhausted with backoff.
ABSL_VLOG(1) << absl::StrFormat(
"Successfully added message with id: %s to reactor, but get a "
"resource exhausted error. Retrying with a sleep delay.",
request.message_id());
int delayMillis = std::min(250 * (i + 1), 1000);
absl::SleepFor(absl::Milliseconds(delayMillis));
continue;
}
// Stop retrying if the send failed due to any other error.
break;
}
return latest_send_request_status;
}