def batch_execute_all()

in gcpdiag/queries/apis_utils.py [0:0]

• 59 lines of code
• 16 McCabe index (conditional complexity)

def batch_execute_all(api, requests: list):
  """Execute all `requests` using the batch API and yield (request,response,exception)
  tuples."""
  # results: (request, result, exception) tuples
  results: List[Tuple[Any, Optional[Any], Optional[Exception]]] = []
  requests_todo = requests
  requests_in_flight: List = []
  retry_count = 0

  def fetch_all_cb(request_id, response, exception):
    try:
      request = requests_in_flight[int(request_id)]
    except (IndexError, ValueError, TypeError):
      logging.debug(
          'BUG: Cannot find request %r in list of pending requests, dropping request.',
          request_id)
      return

    if exception:
      if isinstance(exception, googleapiclient.errors.HttpError) and \
        should_retry(exception.status_code) and \
        retry_count < config.API_RETRIES:
        logging.debug('received HTTP error status code %d from API, retrying',
                      exception.status_code)
        requests_todo.append(request)
      else:
        results.append((request, None, utils.GcpApiError(exception)))
      return

    if not response:
      return

    results.append((request, response, None))

  while True:
    requests_in_flight = requests_todo
    requests_todo = []
    results = []

    # Do the batch API request
    try:
      batch = api.new_batch_http_request()
      for i, req in enumerate(requests_in_flight):
        batch.add(req, callback=fetch_all_cb, request_id=str(i))
      batch.execute()
    except (googleapiclient.errors.HttpError, httplib2.HttpLib2Error) as err:
      if isinstance(err, googleapiclient.errors.HttpError):
        error_msg = f'received HTTP error status code {err.status_code} from Batch API, retrying'
      else:
        error_msg = f'received exception from Batch API: {err}, retrying'
      if (not isinstance(err, googleapiclient.errors.HttpError) or \
          should_retry(err.status_code)) \
          and retry_count < config.API_RETRIES:
        logging.debug(error_msg)
        requests_todo = requests_in_flight
        results = []
      else:
        raise utils.GcpApiError(err) from err

    # Yield results
    yield from results

    # If no requests_todo, means we are done.
    if not requests_todo:
      break

    # for example: retry delay: 20% is random, progression: 1, 1.4, 2.0, 2.7, ... 28.9 (10 retries)
    sleep_time = get_nth_exponential_random_retry(
        n=retry_count,
        random_pct=config.API_RETRY_SLEEP_RANDOMNESS_PCT,
        multiplier=config.API_RETRY_SLEEP_MULTIPLIER)
    logging.debug('sleeping %.2f seconds before retry #%d', sleep_time,
                  retry_count + 1)
    time.sleep(sleep_time)
    retry_count += 1