def get_log_events()

in Solutions/Auth0/Data Connectors/Auth0Connector/main.py [0:0]

• 99 lines of code
• 23 McCabe index (conditional complexity)

    def get_log_events(self, script_start_time, config: dict) -> Tuple[str, List]:
        self.token = self._get_token()
        logging.info(f'Token provided.')
        self.header = self._get_header()
        last_log_id = self._get_last_log_id(config)
        # last_log_id = "90020230126121002244690048186607762971591195832157732866"
        logging.info(f'\tLast log id extracted: {last_log_id}.')
        if last_log_id is None:
            # return '', []
            self.update_statemarker_file(config, '', [])
            return
        # first request
        params = {'from': last_log_id, 'take': '100'}
        count = 0
        error = True
        while error:
            try:
                error = False
                resp = requests.get(
                    self.uri, headers=self.header, params=params)
                if not resp.json():
                    self.update_statemarker_file(config, last_log_id, [])
                    return
                events = resp.json()
                if 'statusCode' in events:
                    raise Exception(events['error'])
            except Exception as err:
                error = True
                count += 1
                if (err == 'Too Many Requests'):
                    time.sleep(1)
                logging.error(
                    "Something wrong. Exception error text: {}".format(err))
                if count > self.retry:
                    logging.error("Exceeded maximum Retries")
                    break
        logging.info('\tFirst request executed.')
        if not resp.json():
            # return last_log_id, []
            self.update_statemarker_file(config, last_log_id, [])
            return
        events = resp.json()
        logging.info(f'Response Object : {events}')
        events.sort(key=lambda item: item['date'], reverse=True)
        last_log_id = events[0]['log_id']
        for el in events:
            self.customize_event(el)
            self.sentinel.send(el)
        self.sentinel.flush()
        logging.info('Events sent to Sentinel.')
        self.update_statemarker_file(config, last_log_id, events)

        if "Link" in resp.headers:
            next_link = resp.headers['Link']
            next_uri = next_link[next_link.index('<') + 1:next_link.index('>')]
            page_num = 1
            while resp.json() and len(events) != 0:
                count = 0
                error = True
                while error:
                    try:
                        error = False
                        resp = requests.get(next_uri, headers=self.header)
                        events = resp.json()
                        if 'statusCode' in events:
                            raise Exception(events['error'])
                    except Exception as err:
                        error = True
                        count += 1
                        if (err == 'Too Many Requests'):
                            time.sleep(1)
                        logging.error(
                            "Something wrong. Exception error text: {}".format(err))
                        if count > self.retry:
                            logging.error("Exceeded maximum Retries")
                            break
                # logging.info(f'\t Response message {resp.headers}')
                try:
                    next_link = resp.headers['Link']
                    next_uri = next_link[next_link.index(
                        '<') + 1:next_link.index('>')]
                    events = resp.json()
                    logging.info(f'\t#{page_num} extracted')
                    page_num += 1
                    if page_num % 9 == 0:
                        time.sleep(1)
                    if len(events) != 0:
                        events.sort(
                            key=lambda item: item['date'], reverse=True)
                        last_log_id = events[0]['log_id']

                        for el in events:
                            self.customize_event(el)
                            self.sentinel.send(el)
                        self.sentinel.flush()

                        self.update_statemarker_file(
                            config, last_log_id, events)

                    if self.check_if_script_runs_too_long(script_start_time):
                        logging.info(
                            f'Script is running too long. Stop processing new events. Finish script.')
                        break
                except Exception as err:
                    logging.error(
                        "Something wrong. Exception error text: {}".format(err))
                    break
        # logging.info(f'\t New last log id: {last_log_id}\n at date {events[0]["date"]}. Events extracted.')
        return last_log_id, events