in Sources/Tokenizers/TokenLattice.swift [63:97]
func viterbi() -> [TokenLatticeNode] {
for offset in 0...count {
guard beginNodes[offset].count > 0 else { return [] }
for rnode in beginNodes[offset] {
rnode.prev = nil
var bestScore: Float = 0
var bestNode: TokenLatticeNode?
for lnode in endNodes[offset] {
let score = lnode.backtraceScore + rnode.score
if bestNode == nil || score > bestScore {
bestNode = lnode.clone()
bestScore = score
}
}
if bestNode != nil {
rnode.prev = bestNode
rnode.backtraceScore = bestScore
}
}
}
let root = beginNodes[count][0]
guard let prev = root.prev else { return [] }
// TODO: the reference implementations have a few more clones here: verify
var result: [TokenLatticeNode] = []
var node = prev // .clone()
while node.prev != nil {
result.append(node.clone())
node = node.prev! // .clone()
}
return result.reversed()
}