def score_passages()

in paq/generation/passage_scorer/scorer.py [0:0]

• 19 lines of code
• 6 McCabe index (conditional complexity)

    def score_passages(self, passages_to_label, disable_tqdm=False):

        def _run_batch(batch):
            inputs = self._tokenize([b['passage'] for b in batch])
            scores = self.model(**inputs)
            log_probs = torch.log_softmax(scores.logits, dim=-1)[:, 1].cpu().tolist()
            for s, b in zip(log_probs, batch):
                b['metadata']['ps_score'] = float(s)
            return scores

        batch, outputs = [], []
        for passage in tqdm(passages_to_label, disable=disable_tqdm):
            batch.append(passage)

            if len(batch) == self.batch_size:
                _run_batch(batch)
                outputs += batch
                batch = []

        if len(batch) != 0:
            _run_batch(batch)
            outputs += batch

        return outputs