in object_store/src/client/retry.rs [157:257]
fn send_retry(self, config: &RetryConfig) -> BoxFuture<'static, Result<Response>> {
let mut backoff = Backoff::new(&config.backoff);
let max_retries = config.max_retries;
let retry_timeout = config.retry_timeout;
async move {
let mut retries = 0;
let now = Instant::now();
loop {
let s = self.try_clone().expect("request body must be cloneable");
match s.send().await {
Ok(r) => match r.error_for_status_ref() {
Ok(_) if r.status().is_success() => return Ok(r),
Ok(r) if r.status() == StatusCode::NOT_MODIFIED => {
return Err(Error{
message: "not modified".to_string(),
retries,
status: Some(r.status()),
source: None,
})
}
Ok(r) => {
let is_bare_redirect = r.status().is_redirection() && !r.headers().contains_key(LOCATION);
let message = match is_bare_redirect {
true => "Received redirect without LOCATION, this normally indicates an incorrectly configured region".to_string(),
// Not actually sure if this is reachable, but here for completeness
false => format!("request unsuccessful: {}", r.status()),
};
return Err(Error{
message,
retries,
status: Some(r.status()),
source: None,
})
}
Err(e) => {
let status = r.status();
if retries == max_retries
|| now.elapsed() > retry_timeout
|| !status.is_server_error() {
// Get the response message if returned a client error
let message = match status.is_client_error() {
true => match r.text().await {
Ok(message) if !message.is_empty() => message,
Ok(_) => "No Body".to_string(),
Err(e) => format!("error getting response body: {e}")
}
false => status.to_string(),
};
return Err(Error{
message,
retries,
status: Some(status),
source: Some(e),
})
}
let sleep = backoff.next();
retries += 1;
info!("Encountered server error, backing off for {} seconds, retry {} of {}", sleep.as_secs_f32(), retries, max_retries);
tokio::time::sleep(sleep).await;
}
},
Err(e) =>
{
let mut do_retry = false;
if let Some(source) = e.source() {
if let Some(e) = source.downcast_ref::<hyper::Error>() {
if e.is_connect() || e.is_closed() || e.is_incomplete_message() {
do_retry = true;
}
}
}
if retries == max_retries
|| now.elapsed() > retry_timeout
|| !do_retry {
return Err(Error{
retries,
message: "request error".to_string(),
status: e.status(),
source: Some(e),
})
}
let sleep = backoff.next();
retries += 1;
info!("Encountered request error ({}) backing off for {} seconds, retry {} of {}", e, sleep.as_secs_f32(), retries, max_retries);
tokio::time::sleep(sleep).await;
}
}
}
}
.boxed()
}