bpe

Ordering

Visibility

class BartTokenizer extends Gpt2Tokenizer
class BertTokenizer extends BasicTokenizer
class CLIPTokenizer extends Gpt2Tokenizer
class Florence2Tokenizer extends BpeTokenizer
class Gemma3Tokenizer extends BpeTokenizer
class Gpt2Tokenizer extends BpeTokenizer
class InternVLTokenizer extends Gpt2Tokenizer
class JanusTokenizer extends BpeTokenizer
class LLAMA3Tokenizer extends BpeTokenizer
class LLAVATokenizer extends BpeTokenizer
class MLLamaTokenizer extends BpeTokenizer
class OLMoTokenizer extends Gpt2Tokenizer
class PaliGemmaTokenizer extends BpeTokenizer
class Phi2Tokenizer extends Gpt2Tokenizer
class Phi3VisionTokenizer extends BpeTokenizer
class Qwen2VLTokenizer extends BpeTokenizer
class QwenTokenizer extends Gpt2Tokenizer
class RobertaTokenizer extends Gpt2Tokenizer
class SmolVLMTokenizer extends BpeTokenizer
case class SpecialToken(content: String, id: Int, singleWord: Boolean = false, lstrip: Boolean = false, rstrip: Boolean = false) extends Product with Serializable
class StarCoderTokenizer extends Gpt2Tokenizer
class WhisperTokenDecoder extends Gpt2Tokenizer
Class used by Whisper model to decode tokens.
Class used by Whisper model to decode tokens. Does not require merges and is therefore omitted.
Note that this means this class cannot tokenize strings.

Packages