Value Members

1.  final def !=(arg0: Any): Boolean

   Definition Classes

   AnyRef → Any

2.  final def ##(): Int

   Definition Classes

   AnyRef → Any

3.  final def ==(arg0: Any): Boolean

   Definition Classes

   AnyRef → Any

4.  val addPrefixSpaceToSentence: Boolean

   Definition Classes

   BpeTokenizer

5.  val alwaysAddPrefix: Boolean

   Definition Classes

   BpeTokenizer

6.  final def asInstanceOf[T0]: T0

   Definition Classes

   Any

7.  def bpe(indToken: IndexedToken): Array[TokenPiece]

   CLIP Specific tokenization.

   CLIP Specific tokenization. We append "<\w>" to word ends.

   returns

   Array of TokenPieces, corresponding to encoded token

   Attributes

   protected

   Definition Classes

   CLIPTokenizer → BpeTokenizer

8.  val bpeRanks: Map[(String, String), Int]

   Attributes

   protected

   Definition Classes

   BpeTokenizer

9.  val bytesToUnicodeMapping: Map[Int, String]

   Mapping for bytes to a different set of unicode characters (especially white spaces).

   Mapping for bytes to a different set of unicode characters (especially white spaces). This improved model performance for gpt-2

   Attributes

   protected

   Definition Classes

   Gpt2Tokenizer

10.  val cache: Map[String, Array[String]]

    cache for already encoded tokens

    cache for already encoded tokens

    Attributes

    protected

    Definition Classes

    BpeTokenizer

11.  def clone(): AnyRef

    Attributes

    protected[lang]

    Definition Classes

    AnyRef

    Annotations

    @throws( ... ) @native()

12.  def decodeTokens(tokens: Array[Int]): String

    Definition Classes

    Gpt2Tokenizer

13.  val decoderVocab: Map[Int, String]

    Attributes

    protected

    Definition Classes

    Gpt2Tokenizer

14.  def encode(indTokens: Array[IndexedToken]): Array[TokenPiece]

    Definition Classes

    BpeTokenizer

15.  def encode(indToken: IndexedToken): Array[TokenPiece]

    Definition Classes

    BpeTokenizer

16.  final def eq(arg0: AnyRef): Boolean

    Definition Classes

    AnyRef

17.  def equals(arg0: Any): Boolean

    Definition Classes

    AnyRef → Any

18.  def finalize(): Unit

    Attributes

    protected[lang]

    Definition Classes

    AnyRef

    Annotations

    @throws( classOf[java.lang.Throwable] )

19.  def getBpeRanking: ((String, String)) ⇒ Int

    Rankings for the byte pairs.

    Rankings for the byte pairs. Derived from merges.txt

    Attributes

    protected

    Definition Classes

    BpeTokenizer

20.  def getBytePairs(word: Array[String]): Array[(String, String)]

    Create a sequence of byte-pairs of the word

    Create a sequence of byte-pairs of the word

    Attributes

    protected

    Definition Classes

    BpeTokenizer

21.  final def getClass(): Class[_]

    Definition Classes

    AnyRef → Any

    Annotations

    @native()

22.  def getTokenPieces(indToken: IndexedToken, word: Array[String]): Array[TokenPiece]

    Attributes

    protected

    Definition Classes

    BpeTokenizer

23.  def hashCode(): Int

    Definition Classes

    AnyRef → Any

    Annotations

    @native()

24.  final def isInstanceOf[T0]: Boolean

    Definition Classes

    Any

25.  val merges: Map[(String, String), Int]

    Definition Classes

    BpeTokenizer

26.  final def ne(arg0: AnyRef): Boolean

    Definition Classes

    AnyRef

27.  final def notify(): Unit

    Definition Classes

    AnyRef

    Annotations

    @native()

28.  final def notifyAll(): Unit

    Definition Classes

    AnyRef

    Annotations

    @native()

29.  val padWithSequenceTokens: Boolean

    Definition Classes

    BpeTokenizer

30.  def performMerges(wordChars: Array[String], charPairs: Array[(String, String)]): Array[String]

    Attributes

    protected

    Definition Classes

    BpeTokenizer

31.  def preProcessTokenForBpe(token: String): String

    Definition Classes

    Gpt2Tokenizer → BpeTokenizer

32.  val prefixForPieceId: Option[String]

    Definition Classes

    Gpt2Tokenizer → BpeTokenizer

33.  val sentencePadding: (String, String)

    Special tokens of the model for processing

    Special tokens of the model for processing

    Definition Classes

    BpeTokenizer

34.  val specialTokens: SpecialTokens

    Definition Classes

    BpeTokenizer

35.  def splitOnSpecialToken(specialToken: SpecialToken, text: String): ListBuffer[String]

    Split the the individual sub texts on special tokens, e.g.

    Split the the individual sub texts on special tokens, e.g. masking etc.

    Attributes

    protected

    Definition Classes

    BpeTokenizer

36.  val splitPattern: Regex

    Definition Classes

    CLIPTokenizer → Gpt2Tokenizer

37.  val suffixForPieceId: Option[String]

    Attributes

    protected

    Definition Classes

    BpeTokenizer

38.  final def synchronized[T0](arg0: ⇒ T0): T0

    Definition Classes

    AnyRef

39.  def toString(): String

    Definition Classes

    AnyRef → Any

40.  def tokenize(sentence: Sentence): Array[IndexedToken]

    Tokenize considering special tokens and split algorithm

    Tokenize considering special tokens and split algorithm

    Definition Classes

    BpeTokenizer

41.  def tokenizeSubText(text: String, indexOffset: Int): Array[IndexedToken]

    Needs to be implemented

    Needs to be implemented

    Definition Classes

    CLIPTokenizer → Gpt2Tokenizer → BpeTokenizer

42.  val unicodeToByteMapping: Map[String, Int]

    Attributes

    protected

    Definition Classes

    Gpt2Tokenizer

43.  val vocab: Map[String, Int]

    Definition Classes

    BpeTokenizer

44.  final def wait(): Unit

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

45.  final def wait(arg0: Long, arg1: Int): Unit

    Definition Classes

    AnyRef

    Annotations

    @throws( ... )

46.  final def wait(arg0: Long): Unit

    Definition Classes

    AnyRef

    Annotations

    @throws( ... ) @native()