in src/utils/cutZhWords.ts [18:72]
function cut(subToken: string, carry: WrappedTerm): void {
let matchedLastIndex = 0;
let matched = false;
for (const words of zhDictionary) {
if (subToken.substr(0, words.length) === words) {
const nextCarry = {
missed: carry.missed,
term: carry.term.concat({
value: words,
}),
};
if (subToken.length > words.length) {
cut(subToken.substr(words.length), nextCarry);
} else {
wrappedTerms.push(nextCarry);
}
matched = true;
} else {
for (
let lastIndex = words.length - 1;
lastIndex > matchedLastIndex;
lastIndex -= 1
) {
const subWords = words.substr(0, lastIndex);
if (subToken.substr(0, lastIndex) === subWords) {
matchedLastIndex = lastIndex;
const nextCarry = {
missed: carry.missed,
term: carry.term.concat({
value: subWords,
trailing: true,
}),
};
if (subToken.length > lastIndex) {
cut(subToken.substr(lastIndex), nextCarry);
} else {
wrappedTerms.push(nextCarry);
}
matched = true;
break;
}
}
}
}
if (!matched) {
if (subToken.length > 0) {
cut(subToken.substr(1), {
missed: carry.missed + 1,
term: carry.term,
});
} else if (carry.term.length > 0) {
wrappedTerms.push(carry);
}
}
}