in torchmoji/word_generator.py [0:0]
def convert_unicode_punctuation(self, word):
word_converted_punct = []
for c in word:
decoded_c = unidecode(c).lower()
if len(decoded_c) == 0:
# Cannot decode to anything reasonable
word_converted_punct.append(c)
else:
# Check if all punctuation and therefore fine
# to include unidecoded version
allowed_punct = punct_word(
decoded_c,
punctuation=ALLOWED_CONVERTED_UNICODE_PUNCTUATION)
if allowed_punct:
word_converted_punct.append(decoded_c)
else:
word_converted_punct.append(c)
return ''.join(word_converted_punct)