def _process_row()

in distant_supervision/squad_ner_creator.py [0:0]

• 14 lines of code
• 4 McCabe index (conditional complexity)

    def _process_row(self, question):
        """
        1. Run NER on `context` field
        2. Set `ner_category` for answers
        3. Discard answers that are not entities
        """

        # this returns a list, e.g. [('today', 'DATE'), ('Patrick', 'PERSON')]
        ent_lst = self.text_preprocessor.compute_ner(question.context)

        ent_dct = {}
        for ent_str, ner_category in ent_lst:
            # perform uncasing
            ent_dct[ent_str.lower()] = ner_category

        new_answers = []
        for ans_struct in question.answers:
            ans_text = ans_struct['text'].lower()  # perform uncasing
            if ans_text not in ent_dct:
                continue

            ans_struct['ner_category'] = ent_dct[ans_text]
            new_answers.append(ans_struct)

        question.answers = new_answers
        return question