in Sources/Tokenizers/Tokenizer.swift [655:693]
func maybeUpdatePostProcessor(tokenizerConfig: Config, processorConfig: Config?) throws -> Config? {
// If it's already a Template processor (instead of a ByteLevel one), assume it's correct
let postProcessor = PostProcessorFactory.fromConfig(config: processorConfig)
guard !(postProcessor is TemplateProcessing) else { return nil }
let addBosToken = tokenizerConfig.addBosToken.boolean(or: false)
let bosToken = addedTokenAsString(tokenizerConfig.bosToken)
if addBosToken, bosToken == nil {
throw TokenizerError.mismatchedConfig("add_bos_token is True but bos_token is nil")
}
let addEosToken = tokenizerConfig.addEosToken.boolean(or: false)
let eosToken = addedTokenAsString(tokenizerConfig.eosToken)
if addEosToken, eosToken == nil {
throw TokenizerError.mismatchedConfig("add_eos_token is True but eos_token is nil")
}
// alt implementation
var single: [[String: Any]] = []
if addBosToken {
single = single + [["SpecialToken": ["id": bosToken!, "type_id": 0]]]
}
single = single + [["Sequence": ["id": "A", "type_id": 0]]]
if addEosToken {
single = single + [["SpecialToken": ["id": eosToken!, "type_id": 0]]]
}
var pair: [[String: Any]] = single
if addBosToken {
pair = pair + [["SpecialToken": ["id": bosToken!, "type_id": 1]]]
}
pair = pair + [["Sequence": ["id": "B", "type_id": 1]]]
if addEosToken {
pair = pair + [["SpecialToken": ["id": eosToken!, "type_id": 1]]]
}
let postProcessorConfig = Config(["type": PostProcessorType.TemplateProcessing.rawValue, "single": single, "pair": pair])
return postProcessorConfig
}