def _get()

in psaw/PushshiftAPI.py [0:0]

• 23 lines of code
• 7 McCabe index (conditional complexity)

    def _get(self, url, payload={}):
        log.debug('URL: %s' % url)
        log.debug('Payload: %s' % payload)
        i, success = 0, False
        while (not success) and (i<self.max_retries):
            if i > 0:
                warnings.warn("Unable to connect to pushshift.io. Retrying after backoff.")
            self._impose_rate_limit(i)
            i+=1
            try:
                response = requests.get(url, params=payload, proxies=self.proxies,
                                        timeout=self.timeout)
                log.info(response.url)
                log.debug('Response status code: %s' % response.status_code)
            except (requests.ConnectionError, requests.Timeout) as e:
                warnings.warn(f"Network Error: {e}, Retrying ({i + 1}/{self.max_retries}")
                continue
            success = response.status_code == 200
            if not success:
                warnings.warn("Response Code [%s] %s" % (response.status_code, response.reason))

        if not success:
            raise Exception("Unable to connect to pushshift.io. Max retries exceeded.")
        return json.loads(response.text)