XLM/src/evaluation/evaluator.py [639:664]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
                bad = False
                if hyp.count('?') == 2:
                    l, r, _ = hyp.split('?')
                    l = l + '?'
                    r = r + '?'
                    if l == r:
                        doubles += 1
                        bad = True  # Unnecessary to use doubles for the "bad" criteria
                    l_toks = l.split()
                    r_toks = r.split()
                    inp_toks = inp.split()
                    for subq_toks in [l_toks, r_toks]:
                        if set(inp_toks).issubset(set(subq_toks)):
                            all_q_words_in_subq += 1
                            bad = True
                            break
                    for subq_toks in [l_toks, r_toks]:
                        if len(subq_toks) >= len(inp_toks):
                            subq_longer_than_q += 1
                            bad = True
                            break
                elif hyp.count('?') < 2:
                    too_few_qs += 1
                    bad = True
                else:
                    too_many_qs += 1
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



pytorch-transformers/add_umt_subqs_subas_to_q_squad_format_new.py [44:69]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
            bad = False
            if hyp.count('?') == 2:
                l, r, _ = hyp.split('?')
                l = l + '?'
                r = r + '?'
                if l == r:
                    doubles += 1
                    bad = True  # Unnecessary to use doubles for the "bad" criteria
                l_toks = l.split()
                r_toks = r.split()
                inp_toks = inp.split()
                for subq_toks in [l_toks, r_toks]:
                    if set(inp_toks).issubset(set(subq_toks)):
                        all_q_words_in_subq += 1
                        bad = True
                        break
                for subq_toks in [l_toks, r_toks]:
                    if len(subq_toks) >= len(inp_toks):
                        subq_longer_than_q += 1
                        bad = True
                        break
            elif hyp.count('?') < 2:
                too_few_qs += 1
                bad = True
            else:
                too_many_qs += 1
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -