def _search()

in psaw/PushshiftAPI.py [0:0]

• 43 lines of code
• 15 McCabe index (conditional complexity)

    def _search(self,
                kind,
                stop_condition=lambda x: False,
                return_batch=False,
                dataset='reddit',
                **kwargs):
        self.metadata_ = {}
        self.payload = copy.deepcopy(kwargs)
        endpoint = '{dataset}/{kind}/search'.format(dataset=dataset, kind=kind)
        url = self.base_url.format(endpoint=endpoint)
        for response in self._handle_paging(url):
            if 'aggs' in response:
                yield response['aggs']
                # Aggs responses are unreliable in subsequent batches with
                # current search paging implementation. Enforce aggs result
                # is only returned once.
                self.payload.pop('aggs')
            self.metadata_ = response.get('metadata', {})
            log.debug('Metadata: %s' % self.metadata_)
            results = response['data']
            
            shards_down_message = "Not all PushShift shards are active. Query results may be incomplete"
            if self.shards_are_down and (self.shards_down_behavior is not None) :
                if self.shards_down_behavior == 'warn':
                    warnings.warn(shards_down_message)
                if self.shards_down_behavior == 'stop':
                    raise RuntimeError(shards_down_message)
            
            if len(results) == 0:
                return
            if return_batch:
                batch = []

            for thing in results:
                thing = self._wrap_thing(thing, kind)

                if return_batch:
                    batch.append(thing)
                else:
                    yield thing

                if stop_condition(thing):
                    if return_batch:
                        return batch
                    return

            if return_batch:
                yield batch

            # For paging.
            if self.payload.get('sort') == 'desc':
                self.payload['before'] = thing.created_utc
            else:
                self.payload['after'] = thing.created_utc