BPETokenizer
class BPETokenizer(vocab: Map<String, Int>, merges: List<Pair<String, String>>, specialTokens: Map<String, Int> = emptyMap(), addedTokens: Map<String, Int> = emptyMap(), val preTokenizerPattern: String = GPT2_PRE_TOKENIZER_PATTERN, val addPrefixSpace: Boolean = false, val normalization: String? = null, val ignoreMerges: Boolean = false, val unkToken: String? = null, prefixTokens: IntArray = IntArray(0), suffixTokens: IntArray = IntArray(0)) : Tokenizer(source)
Constructors
Link copied to clipboard
constructor(vocab: Map<String, Int>, merges: List<Pair<String, String>>, specialTokens: Map<String, Int> = emptyMap(), addedTokens: Map<String, Int> = emptyMap(), preTokenizerPattern: String = GPT2_PRE_TOKENIZER_PATTERN, addPrefixSpace: Boolean = false, normalization: String? = null, ignoreMerges: Boolean = false, unkToken: String? = null, prefixTokens: IntArray = IntArray(0), suffixTokens: IntArray = IntArray(0))