in learn_bpe.py [0:0]
def __init__(self, fp_ins, is_dict=False):
self.forward_index = {}
self.inverse_index = []
self.counter = []
for fp in fp_ins:
for l in fp:
fields = l.strip('\r\n ').split(' ')
if is_dict:
assert len(fields) == 2
self._add_word(fields[0], count=int(fields[1]))
else:
for w in fields:
if w:
self._add_word(w)