problem.py [523:541]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
                        if not branch in length_appended_set:
                            lengths[branch]['sentence_length'].append(len(tokens))
                            length_appended_set.add(branch)
                        else:
                            if len(tokens) != lengths[branch]['sentence_length'][-1]:
                                # logging.warning(
                                #     "The length of inputs are not consistent. Ingore now. %s" % line)
                                cnt_illegal += 1
                                if cnt_illegal / cnt_all > 0.33:
                                    raise PreprocessError(
                                        "The illegal data is too much. Please check the number of data columns or text token version.")
                                lengths[branch]['sentence_length'].pop()
                                true_len = len(lengths[branch]['sentence_length'])
                                # need delete the last example
                                check_list = ['data', 'lengths', 'target']
                                for single_check in check_list:
                                    single_check = eval(single_check)
                                    self.delete_example(single_check, true_len)
                                break
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -



problem.py [560:578]:
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
                        if not branch in length_appended_set:
                            lengths[branch]['sentence_length'].append(len(tokens))
                            length_appended_set.add(branch)
                        else:
                            if len(tokens) != lengths[branch]['sentence_length'][-1]:
                                # logging.warning(
                                #     "The length of inputs are not consistent. Ingore now. %s" % line)
                                cnt_illegal += 1
                                if cnt_illegal / cnt_all > 0.33:
                                    raise PreprocessError(
                                        "The illegal data is too much. Please check the number of data columns or text token version.")
                                lengths[branch]['sentence_length'].pop()
                                true_len = len(lengths[branch]['sentence_length'])
                                # need delete the last example
                                check_list = ['data', 'lengths', 'target']
                                for single_check in check_list:
                                    single_check = eval(single_check)
                                    self.delete_example(single_check, true_len)
                                break
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -