BPETokenizer

constructor(vocab: Map<String, Int>, merges: List<Pair<String, String>>, specialTokens: Map<String, Int> = emptyMap(), addedTokens: Map<String, Int> = emptyMap(), preTokenizerPattern: String = GPT2_PRE_TOKENIZER_PATTERN, addPrefixSpace: Boolean = false, normalization: String? = null, ignoreMerges: Boolean = false, unkToken: String? = null, prefixTokens: IntArray = IntArray(0), suffixTokens: IntArray = IntArray(0))(source)