Packages

class BiEncoderMultimodalEmbeddings extends Transformer with HasInputAnnotationCols with HasOutputAnnotatorType with HasOutputAnnotationCol with HasImageFeatureProperties with HasEngine with WriteOnnxModel

Dual-encoder multimodal embeddings annotator.

The output is written to two derived columns based on outputCol: ${outputCol}_doc_embeddings and ${outputCol}_image_embeddings.

Linear Supertypes
WriteOnnxModel, HasEngine, HasImageFeatureProperties, ParamsAndFeaturesWritable, HasFeatures, DefaultParamsWritable, MLWritable, HasOutputAnnotationCol, HasOutputAnnotatorType, HasInputAnnotationCols, Transformer, PipelineStage, Logging, Params, Serializable, Serializable, Identifiable, AnyRef, Any
Ordering
  1. Grouped
  2. Alphabetic
  3. By Inheritance
Inherited
  1. BiEncoderMultimodalEmbeddings
  2. WriteOnnxModel
  3. HasEngine
  4. HasImageFeatureProperties
  5. ParamsAndFeaturesWritable
  6. HasFeatures
  7. DefaultParamsWritable
  8. MLWritable
  9. HasOutputAnnotationCol
  10. HasOutputAnnotatorType
  11. HasInputAnnotationCols
  12. Transformer
  13. PipelineStage
  14. Logging
  15. Params
  16. Serializable
  17. Serializable
  18. Identifiable
  19. AnyRef
  20. Any
  1. Hide All
  2. Show All
Visibility
  1. Public
  2. All

Instance Constructors

  1. new BiEncoderMultimodalEmbeddings()
  2. new BiEncoderMultimodalEmbeddings(uid: String)

Type Members

  1. type AnnotatorType = String
    Definition Classes
    HasOutputAnnotatorType

Value Members

  1. final def !=(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  2. final def ##(): Int
    Definition Classes
    AnyRef → Any
  3. final def $[T](param: Param[T]): T
    Attributes
    protected
    Definition Classes
    Params
  4. def $$[T](feature: StructFeature[T]): T
    Attributes
    protected
    Definition Classes
    HasFeatures
  5. def $$[K, V](feature: MapFeature[K, V]): Map[K, V]
    Attributes
    protected
    Definition Classes
    HasFeatures
  6. def $$[T](feature: SetFeature[T]): Set[T]
    Attributes
    protected
    Definition Classes
    HasFeatures
  7. def $$[T](feature: ArrayFeature[T]): Array[T]
    Attributes
    protected
    Definition Classes
    HasFeatures
  8. final def ==(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  9. val addedTokens: MapFeature[String, Int]
  10. final def asInstanceOf[T0]: T0
    Definition Classes
    Any
  11. val batchSize: IntParam
  12. val bosTokenId: IntParam
  13. final def checkSchema(schema: StructType, inputAnnotatorType: String): Boolean
    Attributes
    protected
    Definition Classes
    HasInputAnnotationCols
  14. final def clear(param: Param[_]): BiEncoderMultimodalEmbeddings.this.type
    Definition Classes
    Params
  15. def clone(): AnyRef
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( ... ) @native()
  16. def copy(extra: ParamMap): Transformer
    Definition Classes
    BiEncoderMultimodalEmbeddings → Transformer → PipelineStage → Params
  17. def copyValues[T <: Params](to: T, extra: ParamMap): T
    Attributes
    protected
    Definition Classes
    Params
  18. final def defaultCopy[T <: Params](extra: ParamMap): T
    Attributes
    protected
    Definition Classes
    Params
  19. val doNormalize: BooleanParam

    Whether or not to normalize the input with mean and standard deviation

    Whether or not to normalize the input with mean and standard deviation

    Definition Classes
    HasImageFeatureProperties
  20. val doResize: BooleanParam

    Whether to resize the input to a certain size

    Whether to resize the input to a certain size

    Definition Classes
    HasImageFeatureProperties
  21. val engine: Param[String]

    This param is set internally once via loadSavedModel.

    This param is set internally once via loadSavedModel. That's why there is no setter

    Definition Classes
    HasEngine
  22. val eosTokenId: IntParam
  23. final def eq(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  24. def equals(arg0: Any): Boolean
    Definition Classes
    AnyRef → Any
  25. def explainParam(param: Param[_]): String
    Definition Classes
    Params
  26. def explainParams(): String
    Definition Classes
    Params
  27. final val extraInputCols: StringArrayParam
    Attributes
    protected
    Definition Classes
    HasInputAnnotationCols
  28. final def extractParamMap(): ParamMap
    Definition Classes
    Params
  29. final def extractParamMap(extra: ParamMap): ParamMap
    Definition Classes
    Params
  30. val featureExtractorType: Param[String]

    Name of model's architecture for feature extraction

    Name of model's architecture for feature extraction

    Definition Classes
    HasImageFeatureProperties
  31. val features: ArrayBuffer[Feature[_, _, _]]
    Definition Classes
    HasFeatures
  32. def finalize(): Unit
    Attributes
    protected[lang]
    Definition Classes
    AnyRef
    Annotations
    @throws( classOf[java.lang.Throwable] )
  33. def get[T](feature: StructFeature[T]): Option[T]
    Attributes
    protected
    Definition Classes
    HasFeatures
  34. def get[K, V](feature: MapFeature[K, V]): Option[Map[K, V]]
    Attributes
    protected
    Definition Classes
    HasFeatures
  35. def get[T](feature: SetFeature[T]): Option[Set[T]]
    Attributes
    protected
    Definition Classes
    HasFeatures
  36. def get[T](feature: ArrayFeature[T]): Option[Array[T]]
    Attributes
    protected
    Definition Classes
    HasFeatures
  37. final def get[T](param: Param[T]): Option[T]
    Definition Classes
    Params
  38. def getAddedTokens: Map[String, Int]
  39. def getBatchSize: Int
  40. def getBosTokenId: Int
  41. final def getClass(): Class[_]
    Definition Classes
    AnyRef → Any
    Annotations
    @native()
  42. final def getDefault[T](param: Param[T]): Option[T]
    Definition Classes
    Params
  43. def getDoNormalize: Boolean

    Definition Classes
    HasImageFeatureProperties
  44. def getDoResize: Boolean

    Definition Classes
    HasImageFeatureProperties
  45. def getEngine: String

    Definition Classes
    HasEngine
  46. def getEosTokenId: Int
  47. def getFeatureExtractorType: String

    Definition Classes
    HasImageFeatureProperties
  48. def getImageDataFile: Option[String]
  49. def getImageMean: Array[Double]

    Definition Classes
    HasImageFeatureProperties
  50. def getImageModelFile: String
  51. def getImagePromptFixTokenIds: Array[Int]
  52. def getImageStd: Array[Double]

    Definition Classes
    HasImageFeatureProperties
  53. def getImageTokenId: Int
  54. def getInputCols: Array[String]

    returns

    input annotations columns currently used

    Definition Classes
    HasInputAnnotationCols
  55. def getInstruction: String
  56. def getMaxPixels: Int
  57. def getMerges: Map[(String, String), Int]
  58. def getMinPixels: Int
  59. final def getOrDefault[T](param: Param[T]): T
    Definition Classes
    Params
  60. final def getOutputCol: String

    Gets annotation column name going to generate

    Gets annotation column name going to generate

    Definition Classes
    HasOutputAnnotationCol
  61. def getOutputDocEmbeddingsCol: String
  62. def getOutputImageEmbeddingsCol: String
  63. def getPadTokenId: Int
  64. def getParam(paramName: String): Param[Any]
    Definition Classes
    Params
  65. def getPatchSize: Int
  66. def getResample: Int

    Definition Classes
    HasImageFeatureProperties
  67. def getSize: Int

    Definition Classes
    HasImageFeatureProperties
  68. def getSpatialMergeSize: Int
  69. def getTemporalPatchSize: Int
  70. def getTextDataFile: Option[String]
  71. def getTextModelFile: String
  72. def getVocabulary: Map[String, Int]
  73. final def hasDefault[T](param: Param[T]): Boolean
    Definition Classes
    Params
  74. def hasParam(paramName: String): Boolean
    Definition Classes
    Params
  75. def hashCode(): Int
    Definition Classes
    AnyRef → Any
    Annotations
    @native()
  76. val imageDataFile: Param[String]
  77. val imageMean: DoubleArrayParam

    The sequence of means for each channel, to be used when normalizing images

    The sequence of means for each channel, to be used when normalizing images

    Definition Classes
    HasImageFeatureProperties
  78. val imageModelFile: Param[String]
  79. val imagePromptFixTokenIds: IntArrayParam
  80. val imageStd: DoubleArrayParam

    The sequence of standard deviations for each channel, to be used when normalizing images

    The sequence of standard deviations for each channel, to be used when normalizing images

    Definition Classes
    HasImageFeatureProperties
  81. val imageTokenId: IntParam
  82. def initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean
    Attributes
    protected
    Definition Classes
    Logging
  83. def initializeLogIfNecessary(isInterpreter: Boolean): Unit
    Attributes
    protected
    Definition Classes
    Logging
  84. val inputAnnotatorTypes: Array[String]

    Annotator reference id.

    Annotator reference id. Used to identify elements in metadata or to refer to this annotator type

    Definition Classes
    BiEncoderMultimodalEmbeddings → HasInputAnnotationCols
  85. final val inputCols: StringArrayParam

    columns that contain annotations necessary to run this annotator AnnotatorType is used both as input and output columns if not specified

    columns that contain annotations necessary to run this annotator AnnotatorType is used both as input and output columns if not specified

    Attributes
    protected
    Definition Classes
    HasInputAnnotationCols
  86. val instruction: Param[String]
  87. final def isDefined(param: Param[_]): Boolean
    Definition Classes
    Params
  88. final def isInstanceOf[T0]: Boolean
    Definition Classes
    Any
  89. final def isSet(param: Param[_]): Boolean
    Definition Classes
    Params
  90. def isTraceEnabled(): Boolean
    Attributes
    protected
    Definition Classes
    Logging
  91. def log: Logger
    Attributes
    protected
    Definition Classes
    Logging
  92. def logDebug(msg: ⇒ String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  93. def logDebug(msg: ⇒ String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  94. def logError(msg: ⇒ String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  95. def logError(msg: ⇒ String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  96. def logInfo(msg: ⇒ String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  97. def logInfo(msg: ⇒ String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  98. def logName: String
    Attributes
    protected
    Definition Classes
    Logging
  99. def logTrace(msg: ⇒ String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  100. def logTrace(msg: ⇒ String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  101. def logWarning(msg: ⇒ String, throwable: Throwable): Unit
    Attributes
    protected
    Definition Classes
    Logging
  102. def logWarning(msg: ⇒ String): Unit
    Attributes
    protected
    Definition Classes
    Logging
  103. val maxPixels: IntParam
  104. val merges: MapFeature[(String, String), Int]
  105. val minPixels: IntParam
  106. def msgHelper(schema: StructType): String
    Attributes
    protected
    Definition Classes
    HasInputAnnotationCols
  107. final def ne(arg0: AnyRef): Boolean
    Definition Classes
    AnyRef
  108. final def notify(): Unit
    Definition Classes
    AnyRef
    Annotations
    @native()
  109. final def notifyAll(): Unit
    Definition Classes
    AnyRef
    Annotations
    @native()
  110. def onWrite(path: String, spark: SparkSession): Unit
    Definition Classes
    BiEncoderMultimodalEmbeddings → ParamsAndFeaturesWritable
  111. val optionalInputAnnotatorTypes: Array[String]
    Definition Classes
    HasInputAnnotationCols
  112. val outputAnnotatorType: AnnotatorType
    Definition Classes
    BiEncoderMultimodalEmbeddings → HasOutputAnnotatorType
  113. final val outputCol: Param[String]
    Attributes
    protected
    Definition Classes
    HasOutputAnnotationCol
  114. val padTokenId: IntParam
  115. lazy val params: Array[Param[_]]
    Definition Classes
    Params
  116. val patchSize: IntParam
  117. val resample: IntParam

    An optional resampling filter.

    An optional resampling filter. This can be one of PIL.Image.NEAREST, PIL.Image.BOX, PIL.Image.BILINEAR, PIL.Image.HAMMING, PIL.Image.BICUBIC or PIL.Image.LANCZOS. Only has an effect if do_resize is set to True

    Definition Classes
    HasImageFeatureProperties
  118. def save(path: String): Unit
    Definition Classes
    MLWritable
    Annotations
    @Since( "1.6.0" ) @throws( ... )
  119. def set[T](feature: StructFeature[T], value: T): BiEncoderMultimodalEmbeddings.this.type
    Attributes
    protected
    Definition Classes
    HasFeatures
  120. def set[K, V](feature: MapFeature[K, V], value: Map[K, V]): BiEncoderMultimodalEmbeddings.this.type
    Attributes
    protected
    Definition Classes
    HasFeatures
  121. def set[T](feature: SetFeature[T], value: Set[T]): BiEncoderMultimodalEmbeddings.this.type
    Attributes
    protected
    Definition Classes
    HasFeatures
  122. def set[T](feature: ArrayFeature[T], value: Array[T]): BiEncoderMultimodalEmbeddings.this.type
    Attributes
    protected
    Definition Classes
    HasFeatures
  123. final def set(paramPair: ParamPair[_]): BiEncoderMultimodalEmbeddings.this.type
    Attributes
    protected
    Definition Classes
    Params
  124. final def set(param: String, value: Any): BiEncoderMultimodalEmbeddings.this.type
    Attributes
    protected
    Definition Classes
    Params
  125. final def set[T](param: Param[T], value: T): BiEncoderMultimodalEmbeddings.this.type
    Definition Classes
    Params
  126. def setAddedTokens(value: Map[String, Int]): BiEncoderMultimodalEmbeddings.this.type
  127. def setBatchSize(value: Int): BiEncoderMultimodalEmbeddings.this.type
  128. def setBosTokenId(value: Int): BiEncoderMultimodalEmbeddings.this.type
  129. def setDefault[T](feature: StructFeature[T], value: () ⇒ T): BiEncoderMultimodalEmbeddings.this.type
    Attributes
    protected
    Definition Classes
    HasFeatures
  130. def setDefault[K, V](feature: MapFeature[K, V], value: () ⇒ Map[K, V]): BiEncoderMultimodalEmbeddings.this.type
    Attributes
    protected
    Definition Classes
    HasFeatures
  131. def setDefault[T](feature: SetFeature[T], value: () ⇒ Set[T]): BiEncoderMultimodalEmbeddings.this.type
    Attributes
    protected
    Definition Classes
    HasFeatures
  132. def setDefault[T](feature: ArrayFeature[T], value: () ⇒ Array[T]): BiEncoderMultimodalEmbeddings.this.type
    Attributes
    protected
    Definition Classes
    HasFeatures
  133. final def setDefault(paramPairs: ParamPair[_]*): BiEncoderMultimodalEmbeddings.this.type
    Attributes
    protected
    Definition Classes
    Params
  134. final def setDefault[T](param: Param[T], value: T): BiEncoderMultimodalEmbeddings.this.type
    Attributes
    protected[org.apache.spark.ml]
    Definition Classes
    Params
  135. def setDoNormalize(value: Boolean): BiEncoderMultimodalEmbeddings.this.type

    Definition Classes
    HasImageFeatureProperties
  136. def setDoResize(value: Boolean): BiEncoderMultimodalEmbeddings.this.type

    Definition Classes
    HasImageFeatureProperties
  137. def setEosTokenId(value: Int): BiEncoderMultimodalEmbeddings.this.type
  138. def setExtraInputCols(value: Array[String]): BiEncoderMultimodalEmbeddings.this.type
    Definition Classes
    HasInputAnnotationCols
  139. def setFeatureExtractorType(value: String): BiEncoderMultimodalEmbeddings.this.type

    Definition Classes
    HasImageFeatureProperties
  140. def setImageMean(value: Array[Double]): BiEncoderMultimodalEmbeddings.this.type

    Definition Classes
    HasImageFeatureProperties
  141. def setImagePromptFixTokenIds(value: Array[Int]): BiEncoderMultimodalEmbeddings.this.type
  142. def setImageStd(value: Array[Double]): BiEncoderMultimodalEmbeddings.this.type

    Definition Classes
    HasImageFeatureProperties
  143. def setImageTokenId(value: Int): BiEncoderMultimodalEmbeddings.this.type
  144. final def setInputCols(value: String*): BiEncoderMultimodalEmbeddings.this.type
    Definition Classes
    HasInputAnnotationCols
  145. def setInputCols(value: Array[String]): BiEncoderMultimodalEmbeddings.this.type

    Overrides required annotators column if different than default

    Overrides required annotators column if different than default

    Definition Classes
    HasInputAnnotationCols
  146. def setInstruction(value: String): BiEncoderMultimodalEmbeddings.this.type
  147. def setMaxPixels(value: Int): BiEncoderMultimodalEmbeddings.this.type
  148. def setMerges(value: Map[(String, String), Int]): BiEncoderMultimodalEmbeddings.this.type
  149. def setMinPixels(value: Int): BiEncoderMultimodalEmbeddings.this.type
  150. final def setOutputCol(value: String): BiEncoderMultimodalEmbeddings.this.type

    Overrides annotation column name when transforming

    Overrides annotation column name when transforming

    Definition Classes
    HasOutputAnnotationCol
  151. def setPadTokenId(value: Int): BiEncoderMultimodalEmbeddings.this.type
  152. def setPatchSize(value: Int): BiEncoderMultimodalEmbeddings.this.type
  153. def setResample(value: Int): BiEncoderMultimodalEmbeddings.this.type

    Definition Classes
    HasImageFeatureProperties
  154. def setSize(value: Int): BiEncoderMultimodalEmbeddings.this.type

    Definition Classes
    HasImageFeatureProperties
  155. def setSpatialMergeSize(value: Int): BiEncoderMultimodalEmbeddings.this.type
  156. def setTemporalPatchSize(value: Int): BiEncoderMultimodalEmbeddings.this.type
  157. def setVocabulary(value: Map[String, Int]): BiEncoderMultimodalEmbeddings.this.type
  158. val size: IntParam

    Resize the input to the given size.

    Resize the input to the given size. If a tuple is provided, it should be (width, height). If only an integer is provided, then the input will be resized to (size, size). Only has an effect if do_resize is set to True.

    Definition Classes
    HasImageFeatureProperties
  159. val spatialMergeSize: IntParam
  160. final def synchronized[T0](arg0: ⇒ T0): T0
    Definition Classes
    AnyRef
  161. val temporalPatchSize: IntParam
  162. val textDataFile: Param[String]
  163. val textModelFile: Param[String]
  164. def toString(): String
    Definition Classes
    Identifiable → AnyRef → Any
  165. def transform(dataset: Dataset[_]): DataFrame
    Definition Classes
    BiEncoderMultimodalEmbeddings → Transformer
  166. def transform(dataset: Dataset[_], paramMap: ParamMap): DataFrame
    Definition Classes
    Transformer
    Annotations
    @Since( "2.0.0" )
  167. def transform(dataset: Dataset[_], firstParamPair: ParamPair[_], otherParamPairs: ParamPair[_]*): DataFrame
    Definition Classes
    Transformer
    Annotations
    @Since( "2.0.0" ) @varargs()
  168. def transformSchema(schema: StructType): StructType
    Definition Classes
    BiEncoderMultimodalEmbeddings → PipelineStage
  169. def transformSchema(schema: StructType, logging: Boolean): StructType
    Attributes
    protected
    Definition Classes
    PipelineStage
    Annotations
    @DeveloperApi()
  170. val uid: String
    Definition Classes
    BiEncoderMultimodalEmbeddings → Identifiable
  171. val vocabulary: MapFeature[String, Int]
  172. final def wait(): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )
  173. final def wait(arg0: Long, arg1: Int): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... )
  174. final def wait(arg0: Long): Unit
    Definition Classes
    AnyRef
    Annotations
    @throws( ... ) @native()
  175. def write: MLWriter
    Definition Classes
    ParamsAndFeaturesWritable → DefaultParamsWritable → MLWritable
  176. def writeOnnxModel(path: String, spark: SparkSession, onnxWrapper: OnnxWrapper, suffix: String, fileName: String): Unit
    Definition Classes
    WriteOnnxModel
  177. def writeOnnxModels(path: String, spark: SparkSession, onnxWrappersWithNames: Seq[(OnnxWrapper, String)], suffix: String): Unit
    Definition Classes
    WriteOnnxModel

Inherited from WriteOnnxModel

Inherited from HasEngine

Inherited from HasImageFeatureProperties

Inherited from ParamsAndFeaturesWritable

Inherited from HasFeatures

Inherited from DefaultParamsWritable

Inherited from MLWritable

Inherited from HasOutputAnnotationCol

Inherited from HasOutputAnnotatorType

Inherited from HasInputAnnotationCols

Inherited from Transformer

Inherited from PipelineStage

Inherited from Logging

Inherited from Params

Inherited from Serializable

Inherited from Serializable

Inherited from Identifiable

Inherited from AnyRef

Inherited from Any

getParam

param

setParam

Ungrouped