kilt/datasets/natural_questions.py [98:144]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
                            (
                                paragraph_id,
                                start_character,
                                end_character,
                                bleu,
                            ) = utils.match_answer(
                                answer_span, page, nlp=self.nlp, debug=False
                            )

                            kilt_record_output.append(
                                {
                                    # answer in textual form
                                    "answer": answer_span,
                                    "provenance": [
                                        # list of relevant WikipediaPages / Spans as provenance for the answer from the ks
                                        {
                                            "wikipedia_id": page[
                                                "wikipedia_id"
                                            ],  # *mandatory* - ID Wikipedia Page
                                            "title": page[
                                                "wikipedia_title"
                                            ],  # *mandatory* - Title Wikipedia Page
                                            "start_paragraph_id": paragraph_id,  # start paragraph id with relevant info
                                            "start_character": start_character,
                                            "end_paragraph_id": paragraph_id,  # end paragraph id
                                            "end_character": end_character,
                                            "bleu_score": bleu,  # 1.0 when gold data is exactly matched, lower for fuzzy matches
                                            "meta": {  # dataset/task specific
                                                "yes_no_answer": annotations[0][
                                                    "yes_no_answer"
                                                ],
                                                "annotation_id": annotations[0][
                                                    "annotation_id"
                                                ],
                                            },
                                        }
                                    ],
                                }
                            )

                            if bleu == 1:
                                local_sem += 1
                            elif bleu < 1 and bleu >= 0:
                                local_sfm += 1
                            else:
                                print("ERROR: invalid bleu: {}".format(bleu))
                                sys.exit(-1)
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



kilt/datasets/natural_questions.py [155:201]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
                        (
                            paragraph_id,
                            start_character,
                            end_character,
                            bleu,
                        ) = utils.match_answer(
                            answer_span, page, nlp=self.nlp, debug=False
                        )

                        kilt_record_output.append(
                            {
                                # answer in textual form
                                "answer": answer_span,
                                "provenance": [
                                    # list of relevant WikipediaPages / Spans as provenance for the answer from the ks
                                    {
                                        "wikipedia_id": page[
                                            "wikipedia_id"
                                        ],  # *mandatory* - ID Wikipedia Page
                                        "title": page[
                                            "wikipedia_title"
                                        ],  # *mandatory* - Title Wikipedia Page
                                        "start_paragraph_id": paragraph_id,  # start paragraph id with relevant info
                                        "start_character": start_character,
                                        "end_paragraph_id": paragraph_id,  # end paragraph id
                                        "end_character": end_character,
                                        "bleu_score": bleu,  # 1.0 when gold data is exactly matched, lower for fuzzy matches
                                        "meta": {  # dataset/task specific
                                            "yes_no_answer": annotations[0][
                                                "yes_no_answer"
                                            ],
                                            "annotation_id": annotations[0][
                                                "annotation_id"
                                            ],
                                        },
                                    }
                                ],
                            }
                        )

                        if bleu == 1:
                            local_sem += 1
                        elif bleu < 1 and bleu >= 0:
                            local_sfm += 1
                        else:
                            print("ERROR: invalid bleu: {}".format(bleu))
                            sys.exit(-1)
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -