in azurelinuxagent/common/utils/restutil.py [0:0]
def http_request(method,
url, data, timeout,
headers=None,
use_proxy=False,
max_retry=None,
retry_codes=None,
retry_delay=DELAY_IN_SECONDS,
throttle_delay=THROTTLE_DELAY_IN_SECONDS,
redact_data=False,
return_raw_response=False):
"""
NOTE: This method provides some logic to handle errors in the HTTP request, including checking the HTTP status of the response
and handling some exceptions. If return_raw_response is set to True all the error handling will be skipped and the
method will return the actual HTTP response and bubble up any exceptions while issuing the request. Also note that if
return_raw_response is True no retries will be done.
"""
if max_retry is None:
max_retry = DEFAULT_RETRIES
if retry_codes is None:
retry_codes = RETRY_CODES
global SECURE_WARNING_EMITTED # pylint: disable=W0603
host, port, secure, rel_uri = _parse_url(url)
# Use the HTTP(S) proxy
proxy_host, proxy_port = (None, None)
if use_proxy and not bypass_proxy(host):
proxy_host, proxy_port = _get_http_proxy(secure=secure)
if proxy_host or proxy_port:
logger.verbose("HTTP proxy: [{0}:{1}]", proxy_host, proxy_port)
# If httplib module is not built with ssl support,
# fallback to HTTP if allowed
if secure and not hasattr(httpclient, "HTTPSConnection"):
if not conf.get_allow_http():
raise HttpError("HTTPS is unavailable and required")
secure = False
if not SECURE_WARNING_EMITTED:
logger.warn("Python does not include SSL support")
SECURE_WARNING_EMITTED = True
# If httplib module doesn't support HTTPS tunnelling,
# fallback to HTTP if allowed
if secure and \
proxy_host is not None and \
proxy_port is not None \
and not hasattr(httpclient.HTTPSConnection, "set_tunnel"):
if not conf.get_allow_http():
raise HttpError("HTTPS tunnelling is unavailable and required")
secure = False
if not SECURE_WARNING_EMITTED:
logger.warn("Python does not support HTTPS tunnelling")
SECURE_WARNING_EMITTED = True
msg = ''
attempt = 0
delay = 0
was_throttled = False
while attempt < max_retry:
if attempt > 0:
# Compute the request delay
# -- Use a fixed delay if the server ever rate-throttles the request
# (with a safe, minimum number of retry attempts)
# -- Otherwise, compute a delay that is the product of the next
# item in the Fibonacci series and the initial delay value
if was_throttled:
delay = throttle_delay
else:
delay = _compute_delay(retry_attempt=attempt, delay=retry_delay)
logger.verbose("[HTTP Retry] "
"Attempt {0} of {1} will delay {2} seconds: {3}",
attempt+1,
max_retry,
delay,
msg)
time.sleep(delay)
attempt += 1
try:
resp = _http_request(method,
host,
rel_uri,
timeout,
port=port,
data=data,
secure=secure,
headers=headers,
proxy_host=proxy_host,
proxy_port=proxy_port,
redact_data=redact_data)
logger.verbose("[HTTP Response] Status Code {0}", resp.status)
if return_raw_response: # skip all error handling
return resp
if request_failed(resp):
if _is_retry_status(resp.status, retry_codes=retry_codes):
msg = '[HTTP Retry] {0} {1} -- Status Code {2}'.format(method, url, resp.status)
# Note if throttled and ensure a safe, minimum number of
# retry attempts
if _is_throttle_status(resp.status):
was_throttled = True
# Today, THROTTLE_RETRIES is set to a large number (26) for retries, as opposed to backing off and attempting fewer retries.
# However, for telemetry calls (due to throttle limit 15 calls per 15 seconds), we use max_retry set by the caller for overall retry attempts instead of THROTTLE_RETRIES.
if not _is_telemetry_req(url):
max_retry = max(max_retry, THROTTLE_RETRIES)
continue
# If we got a 410 (resource gone) for any reason, raise an exception. The caller will handle it by
# forcing a goal state refresh and retrying the call.
if resp.status in RESOURCE_GONE_CODES:
response_error = read_response_error(resp)
raise ResourceGoneError(response_error)
# If we got a 400 (bad request) because the container id is invalid, it could indicate a stale goal
# state. The caller will handle this exception by forcing a goal state refresh and retrying the call.
if resp.status == httpclient.BAD_REQUEST:
response_error = read_response_error(resp)
if INVALID_CONTAINER_CONFIGURATION in response_error:
raise InvalidContainerError(response_error)
return resp
except httpclient.HTTPException as e:
if return_raw_response: # skip all error handling
raise
clean_url = _trim_url_parameters(url)
msg = '[HTTP Failed] {0} {1} -- HttpException {2}'.format(method, clean_url, e)
if _is_retry_exception(e):
continue
break
except IOError as e:
if return_raw_response: # skip all error handling
raise
IOErrorCounter.increment(host=host, port=port)
clean_url = _trim_url_parameters(url)
msg = '[HTTP Failed] {0} {1} -- IOError {2}'.format(method, clean_url, e)
continue
raise HttpError("{0} -- {1} attempts made".format(msg, attempt))