BPETokenizer

class BPETokenizer(vocab: Map<String, Int>, merges: List<Pair<String, String>>, specialTokens: Map<String, Int> = emptyMap(), addedTokens: Map<String, Int> = emptyMap(), val preTokenizerPattern: String = GPT2_PRE_TOKENIZER_PATTERN, val addPrefixSpace: Boolean = false, val normalization: String? = null, val ignoreMerges: Boolean = false, val unkToken: String? = null, prefixTokens: IntArray = IntArray(0), suffixTokens: IntArray = IntArray(0)) : Tokenizer(source)

Constructors

Link copied to clipboard
constructor(vocab: Map<String, Int>, merges: List<Pair<String, String>>, specialTokens: Map<String, Int> = emptyMap(), addedTokens: Map<String, Int> = emptyMap(), preTokenizerPattern: String = GPT2_PRE_TOKENIZER_PATTERN, addPrefixSpace: Boolean = false, normalization: String? = null, ignoreMerges: Boolean = false, unkToken: String? = null, prefixTokens: IntArray = IntArray(0), suffixTokens: IntArray = IntArray(0))

Types

Link copied to clipboard
object Companion

Properties

Link copied to clipboard
Link copied to clipboard
Link copied to clipboard
Link copied to clipboard
Link copied to clipboard
val imEndId: Int?
Link copied to clipboard
Link copied to clipboard
Link copied to clipboard
Link copied to clipboard
Link copied to clipboard
Link copied to clipboard
Link copied to clipboard
Link copied to clipboard
Link copied to clipboard
open override val vocabularySize: Int

Functions

Link copied to clipboard
open override fun decode(tokens: IntArray): String
fun decode(tokens: IntArray, skipSpecialTokens: Boolean): String
Link copied to clipboard
open override fun encode(text: String): IntArray
fun encode(text: String, addSpecialTokens: Boolean): IntArray
Link copied to clipboard
Link copied to clipboard
fun token(id: Int): String?
Link copied to clipboard
fun tokenId(token: String): Int?