public TokenizerModel()

in Runtime/Tokenizers/Tokenizers/Tokenizers.cs [53:68]

• 11 lines of code
• 1 McCabe index (conditional complexity)

        public TokenizerModel(JObject config, JObject tokenizerData)
        {
            Config = config;
            TokenizerData = tokenizerData;
            Vocab = new List<string>();

            // Dict of tokens to ids Dict<string, number>
            TokensToIds = new Dictionary<string, int>();

            UnkTokenId = null;
            UnkToken = null;
            EndOfWordSuffix = null;

            // Whether to fuse unknown tokens when encoding (default to false)
            FuseUnk = false;
        }