class WordEmbeddings extends AnnotatorApproach[WordEmbeddingsModel] with HasStorage with HasEmbeddingsProperties
Word Embeddings lookup annotator that maps tokens to vectors.
For instantiated/pretrained models, see WordEmbeddingsModel.
A custom token lookup dictionary for embeddings can be set with setStoragePath. Each line of
the provided file needs to have a token, followed by their vector representation, delimited by
a spaces.
... are 0.39658191506190343 0.630968081620067 0.5393722253731201 0.8428180123359783 were 0.7535235923631415 0.9699218875629833 0.10397182122983872 0.11833962569383116 stress 0.0492683418305907 0.9415954572751959 0.47624463167525755 0.16790967216778263 induced 0.1535748762292387 0.33498936903209897 0.9235178224122094 0.1158772920395934 ...
If a token is not found in the dictionary, then the result will be a zero vector of the same dimension. Statistics about the rate of converted tokens, can be retrieved with WordEmbeddingsModel.withCoverageColumn and WordEmbeddingsModel.overallCoverage.
For extended examples of usage, see the Examples and the WordEmbeddingsTestSpec.
Example
In this example, the file random_embeddings_dim4.txt has the form of the content above.
import spark.implicits._ import com.johnsnowlabs.nlp.base.DocumentAssembler import com.johnsnowlabs.nlp.annotators.Tokenizer import com.johnsnowlabs.nlp.embeddings.WordEmbeddings import com.johnsnowlabs.nlp.util.io.ReadAs import com.johnsnowlabs.nlp.EmbeddingsFinisher import org.apache.spark.ml.Pipeline val documentAssembler = new DocumentAssembler() .setInputCol("text") .setOutputCol("document") val tokenizer = new Tokenizer() .setInputCols(Array("document")) .setOutputCol("token") val embeddings = new WordEmbeddings() .setStoragePath("src/test/resources/random_embeddings_dim4.txt", ReadAs.TEXT) .setStorageRef("glove_4d") .setDimension(4) .setInputCols("document", "token") .setOutputCol("embeddings") val embeddingsFinisher = new EmbeddingsFinisher() .setInputCols("embeddings") .setOutputCols("finished_embeddings") .setOutputAsVector(true) .setCleanAnnotations(false) val pipeline = new Pipeline() .setStages(Array( documentAssembler, tokenizer, embeddings, embeddingsFinisher )) val data = Seq("The patient was diagnosed with diabetes.").toDF("text") val result = pipeline.fit(data).transform(data) result.selectExpr("explode(finished_embeddings) as result").show(false) +----------------------------------------------------------------------------------+ |result | +----------------------------------------------------------------------------------+ |[0.9439099431037903,0.4707513153553009,0.806300163269043,0.16176554560661316] | |[0.7966810464859009,0.5551124811172485,0.8861005902290344,0.28284206986427307] | |[0.025029370561242104,0.35177749395370483,0.052506182342767715,0.1887107789516449]| |[0.08617766946554184,0.8399239182472229,0.5395117998123169,0.7864698767662048] | |[0.6599600911140442,0.16109347343444824,0.6041093468666077,0.8913561105728149] | |[0.5955275893211365,0.01899011991918087,0.4397728443145752,0.8911281824111938] | |[0.9840458631515503,0.7599489092826843,0.9417727589607239,0.8624503016471863] | +----------------------------------------------------------------------------------+
- See also
 SentenceEmbeddings to combine embeddings into a sentence-level representation
Annotators Main Page for a list of transformer based embeddings
- Grouped
 - Alphabetic
 - By Inheritance
 
- WordEmbeddings
 - HasEmbeddingsProperties
 - HasProtectedParams
 - HasStorage
 - HasCaseSensitiveProperties
 - HasStorageOptions
 - HasStorageRef
 - ParamsAndFeaturesWritable
 - HasFeatures
 - AnnotatorApproach
 - CanBeLazy
 - DefaultParamsWritable
 - MLWritable
 - HasOutputAnnotatorType
 - HasOutputAnnotationCol
 - HasInputAnnotationCols
 - Estimator
 - PipelineStage
 - Logging
 - Params
 - Serializable
 - Serializable
 - Identifiable
 - AnyRef
 - Any
 
- Hide All
 - Show All
 
- Public
 - All
 
Instance Constructors
Type Members
- 
      
      
      
        
      
    
      
        implicit 
        class
      
      
        ProtectedParam[T] extends Param[T]
      
      
      
- Definition Classes
 - HasProtectedParams
 
 - 
      
      
      
        
      
    
      
        
        type
      
      
        AnnotatorType = String
      
      
      
- Definition Classes
 - HasOutputAnnotatorType
 
 
Value Members
- 
      
      
      
        
      
    
      
        final 
        def
      
      
        !=(arg0: Any): Boolean
      
      
      
- Definition Classes
 - AnyRef → Any
 
 - 
      
      
      
        
      
    
      
        final 
        def
      
      
        ##(): Int
      
      
      
- Definition Classes
 - AnyRef → Any
 
 - 
      
      
      
        
      
    
      
        final 
        def
      
      
        $[T](param: Param[T]): T
      
      
      
- Attributes
 - protected
 - Definition Classes
 - Params
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        $$[T](feature: StructFeature[T]): T
      
      
      
- Attributes
 - protected
 - Definition Classes
 - HasFeatures
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        $$[K, V](feature: MapFeature[K, V]): Map[K, V]
      
      
      
- Attributes
 - protected
 - Definition Classes
 - HasFeatures
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        $$[T](feature: SetFeature[T]): Set[T]
      
      
      
- Attributes
 - protected
 - Definition Classes
 - HasFeatures
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        $$[T](feature: ArrayFeature[T]): Array[T]
      
      
      
- Attributes
 - protected
 - Definition Classes
 - HasFeatures
 
 - 
      
      
      
        
      
    
      
        final 
        def
      
      
        ==(arg0: Any): Boolean
      
      
      
- Definition Classes
 - AnyRef → Any
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        _fit(dataset: Dataset[_], recursiveStages: Option[PipelineModel]): WordEmbeddingsModel
      
      
      
- Attributes
 - protected
 - Definition Classes
 - AnnotatorApproach
 
 - 
      
      
      
        
      
    
      
        final 
        def
      
      
        asInstanceOf[T0]: T0
      
      
      
- Definition Classes
 - Any
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        beforeTraining(spark: SparkSession): Unit
      
      
      
- Definition Classes
 - AnnotatorApproach
 
 - 
      
      
      
        
      
    
      
        
        val
      
      
        caseSensitive: BooleanParam
      
      
      
Whether to ignore case in index lookups (Default depends on model)
Whether to ignore case in index lookups (Default depends on model)
- Definition Classes
 - HasCaseSensitiveProperties
 
 - 
      
      
      
        
      
    
      
        final 
        def
      
      
        checkSchema(schema: StructType, inputAnnotatorType: String): Boolean
      
      
      
- Attributes
 - protected
 - Definition Classes
 - HasInputAnnotationCols
 
 - 
      
      
      
        
      
    
      
        final 
        def
      
      
        clear(param: Param[_]): WordEmbeddings.this.type
      
      
      
- Definition Classes
 - Params
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        clone(): AnyRef
      
      
      
- Attributes
 - protected[lang]
 - Definition Classes
 - AnyRef
 - Annotations
 - @throws( ... ) @native()
 
 - 
      
      
      
        
      
    
      
        final 
        def
      
      
        copy(extra: ParamMap): Estimator[WordEmbeddingsModel]
      
      
      
- Definition Classes
 - AnnotatorApproach → Estimator → PipelineStage → Params
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        copyValues[T <: Params](to: T, extra: ParamMap): T
      
      
      
- Attributes
 - protected
 - Definition Classes
 - Params
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        createDatabaseConnection(database: Name): RocksDBConnection
      
      
      
- Definition Classes
 - HasStorageRef
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        createWriter(database: Name, connection: RocksDBConnection): StorageWriter[_]
      
      
      
- Attributes
 - protected
 - Definition Classes
 - WordEmbeddings → HasStorage
 
 - 
      
      
      
        
      
    
      
        
        val
      
      
        databases: Array[Name]
      
      
      
- Definition Classes
 - WordEmbeddings → HasStorage
 
 - 
      
      
      
        
      
    
      
        final 
        def
      
      
        defaultCopy[T <: Params](extra: ParamMap): T
      
      
      
- Attributes
 - protected
 - Definition Classes
 - Params
 
 - 
      
      
      
        
      
    
      
        
        val
      
      
        description: String
      
      
      
Word Embeddings lookup annotator that maps tokens to vectors
Word Embeddings lookup annotator that maps tokens to vectors
- Definition Classes
 - WordEmbeddings → AnnotatorApproach
 
 - 
      
      
      
        
      
    
      
        
        val
      
      
        dimension: ProtectedParam[Int]
      
      
      
Number of embedding dimensions (Default depends on model)
Number of embedding dimensions (Default depends on model)
- Definition Classes
 - HasEmbeddingsProperties
 
 - 
      
      
      
        
      
    
      
        
        val
      
      
        enableInMemoryStorage: BooleanParam
      
      
      
- Definition Classes
 - HasStorageOptions
 
 - 
      
      
      
        
      
    
      
        final 
        def
      
      
        eq(arg0: AnyRef): Boolean
      
      
      
- Definition Classes
 - AnyRef
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        equals(arg0: Any): Boolean
      
      
      
- Definition Classes
 - AnyRef → Any
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        explainParam(param: Param[_]): String
      
      
      
- Definition Classes
 - Params
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        explainParams(): String
      
      
      
- Definition Classes
 - Params
 
 - 
      
      
      
        
      
    
      
        final 
        def
      
      
        extractParamMap(): ParamMap
      
      
      
- Definition Classes
 - Params
 
 - 
      
      
      
        
      
    
      
        final 
        def
      
      
        extractParamMap(extra: ParamMap): ParamMap
      
      
      
- Definition Classes
 - Params
 
 - 
      
      
      
        
      
    
      
        
        val
      
      
        features: ArrayBuffer[Feature[_, _, _]]
      
      
      
- Definition Classes
 - HasFeatures
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        finalize(): Unit
      
      
      
- Attributes
 - protected[lang]
 - Definition Classes
 - AnyRef
 - Annotations
 - @throws( classOf[java.lang.Throwable] )
 
 - 
      
      
      
        
      
    
      
        final 
        def
      
      
        fit(dataset: Dataset[_]): WordEmbeddingsModel
      
      
      
- Definition Classes
 - AnnotatorApproach → Estimator
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        fit(dataset: Dataset[_], paramMaps: Seq[ParamMap]): Seq[WordEmbeddingsModel]
      
      
      
- Definition Classes
 - Estimator
 - Annotations
 - @Since( "2.0.0" )
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        fit(dataset: Dataset[_], paramMap: ParamMap): WordEmbeddingsModel
      
      
      
- Definition Classes
 - Estimator
 - Annotations
 - @Since( "2.0.0" )
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        fit(dataset: Dataset[_], firstParamPair: ParamPair[_], otherParamPairs: ParamPair[_]*): WordEmbeddingsModel
      
      
      
- Definition Classes
 - Estimator
 - Annotations
 - @Since( "2.0.0" ) @varargs()
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        get[T](feature: StructFeature[T]): Option[T]
      
      
      
- Attributes
 - protected
 - Definition Classes
 - HasFeatures
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        get[K, V](feature: MapFeature[K, V]): Option[Map[K, V]]
      
      
      
- Attributes
 - protected
 - Definition Classes
 - HasFeatures
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        get[T](feature: SetFeature[T]): Option[Set[T]]
      
      
      
- Attributes
 - protected
 - Definition Classes
 - HasFeatures
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        get[T](feature: ArrayFeature[T]): Option[Array[T]]
      
      
      
- Attributes
 - protected
 - Definition Classes
 - HasFeatures
 
 - 
      
      
      
        
      
    
      
        final 
        def
      
      
        get[T](param: Param[T]): Option[T]
      
      
      
- Definition Classes
 - Params
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        getCaseSensitive: Boolean
      
      
      
- Definition Classes
 - HasCaseSensitiveProperties
 
 - 
      
      
      
        
      
    
      
        final 
        def
      
      
        getClass(): Class[_]
      
      
      
- Definition Classes
 - AnyRef → Any
 - Annotations
 - @native()
 
 - 
      
      
      
        
      
    
      
        final 
        def
      
      
        getDefault[T](param: Param[T]): Option[T]
      
      
      
- Definition Classes
 - Params
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        getDimension: Int
      
      
      
- Definition Classes
 - HasEmbeddingsProperties
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        getEnableInMemoryStorage: Boolean
      
      
      
- Definition Classes
 - HasStorageOptions
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        getIncludeStorage: Boolean
      
      
      
- Definition Classes
 - HasStorageOptions
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        getInputCols: Array[String]
      
      
      
- returns
 input annotations columns currently used
- Definition Classes
 - HasInputAnnotationCols
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        getLazyAnnotator: Boolean
      
      
      
- Definition Classes
 - CanBeLazy
 
 - 
      
      
      
        
      
    
      
        final 
        def
      
      
        getOrDefault[T](param: Param[T]): T
      
      
      
- Definition Classes
 - Params
 
 - 
      
      
      
        
      
    
      
        final 
        def
      
      
        getOutputCol: String
      
      
      
Gets annotation column name going to generate
Gets annotation column name going to generate
- Definition Classes
 - HasOutputAnnotationCol
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        getParam(paramName: String): Param[Any]
      
      
      
- Definition Classes
 - Params
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        getStoragePath: Option[ExternalResource]
      
      
      
- Definition Classes
 - HasStorage
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        getStorageRef: String
      
      
      
- Definition Classes
 - HasStorageRef
 
 - 
      
      
      
        
      
    
      
        final 
        def
      
      
        hasDefault[T](param: Param[T]): Boolean
      
      
      
- Definition Classes
 - Params
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        hasParam(paramName: String): Boolean
      
      
      
- Definition Classes
 - Params
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        hashCode(): Int
      
      
      
- Definition Classes
 - AnyRef → Any
 - Annotations
 - @native()
 
 - 
      
      
      
        
      
    
      
        
        val
      
      
        includeStorage: BooleanParam
      
      
      
- Definition Classes
 - HasStorageOptions
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        index(fitDataset: Dataset[_], storageSourcePath: Option[String], readAs: Option[util.io.ReadAs.Value], writers: Map[Name, StorageWriter[_]], readOptions: Option[Map[String, String]]): Unit
      
      
      
- Attributes
 - protected
 - Definition Classes
 - WordEmbeddings → HasStorage
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        indexStorage(fitDataset: Dataset[_], resource: Option[ExternalResource]): Unit
      
      
      
- Definition Classes
 - HasStorage
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean
      
      
      
- Attributes
 - protected
 - Definition Classes
 - Logging
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        initializeLogIfNecessary(isInterpreter: Boolean): Unit
      
      
      
- Attributes
 - protected
 - Definition Classes
 - Logging
 
 - 
      
      
      
        
      
    
      
        
        val
      
      
        inputAnnotatorTypes: Array[String]
      
      
      
Input annotation type : DOCUMENT, TOKEN
Input annotation type : DOCUMENT, TOKEN
- Definition Classes
 - WordEmbeddings → HasInputAnnotationCols
 
 - 
      
      
      
        
      
    
      
        final 
        val
      
      
        inputCols: StringArrayParam
      
      
      
columns that contain annotations necessary to run this annotator AnnotatorType is used both as input and output columns if not specified
columns that contain annotations necessary to run this annotator AnnotatorType is used both as input and output columns if not specified
- Attributes
 - protected
 - Definition Classes
 - HasInputAnnotationCols
 
 - 
      
      
      
        
      
    
      
        final 
        def
      
      
        isDefined(param: Param[_]): Boolean
      
      
      
- Definition Classes
 - Params
 
 - 
      
      
      
        
      
    
      
        final 
        def
      
      
        isInstanceOf[T0]: Boolean
      
      
      
- Definition Classes
 - Any
 
 - 
      
      
      
        
      
    
      
        final 
        def
      
      
        isSet(param: Param[_]): Boolean
      
      
      
- Definition Classes
 - Params
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        isTraceEnabled(): Boolean
      
      
      
- Attributes
 - protected
 - Definition Classes
 - Logging
 
 - 
      
      
      
        
      
    
      
        
        val
      
      
        lazyAnnotator: BooleanParam
      
      
      
- Definition Classes
 - CanBeLazy
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        log: Logger
      
      
      
- Attributes
 - protected
 - Definition Classes
 - Logging
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        logDebug(msg: ⇒ String, throwable: Throwable): Unit
      
      
      
- Attributes
 - protected
 - Definition Classes
 - Logging
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        logDebug(msg: ⇒ String): Unit
      
      
      
- Attributes
 - protected
 - Definition Classes
 - Logging
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        logError(msg: ⇒ String, throwable: Throwable): Unit
      
      
      
- Attributes
 - protected
 - Definition Classes
 - Logging
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        logError(msg: ⇒ String): Unit
      
      
      
- Attributes
 - protected
 - Definition Classes
 - Logging
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        logInfo(msg: ⇒ String, throwable: Throwable): Unit
      
      
      
- Attributes
 - protected
 - Definition Classes
 - Logging
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        logInfo(msg: ⇒ String): Unit
      
      
      
- Attributes
 - protected
 - Definition Classes
 - Logging
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        logName: String
      
      
      
- Attributes
 - protected
 - Definition Classes
 - Logging
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        logTrace(msg: ⇒ String, throwable: Throwable): Unit
      
      
      
- Attributes
 - protected
 - Definition Classes
 - Logging
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        logTrace(msg: ⇒ String): Unit
      
      
      
- Attributes
 - protected
 - Definition Classes
 - Logging
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        logWarning(msg: ⇒ String, throwable: Throwable): Unit
      
      
      
- Attributes
 - protected
 - Definition Classes
 - Logging
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        logWarning(msg: ⇒ String): Unit
      
      
      
- Attributes
 - protected
 - Definition Classes
 - Logging
 
 - 
      
      
      
        
      
    
      
        
        val
      
      
        missingRefMsg: String
      
      
      
Error message
Error message
- Attributes
 - protected
 - Definition Classes
 - WordEmbeddings → HasStorage
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        msgHelper(schema: StructType): String
      
      
      
- Attributes
 - protected
 - Definition Classes
 - HasInputAnnotationCols
 
 - 
      
      
      
        
      
    
      
        final 
        def
      
      
        ne(arg0: AnyRef): Boolean
      
      
      
- Definition Classes
 - AnyRef
 
 - 
      
      
      
        
      
    
      
        final 
        def
      
      
        notify(): Unit
      
      
      
- Definition Classes
 - AnyRef
 - Annotations
 - @native()
 
 - 
      
      
      
        
      
    
      
        final 
        def
      
      
        notifyAll(): Unit
      
      
      
- Definition Classes
 - AnyRef
 - Annotations
 - @native()
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        onTrained(model: WordEmbeddingsModel, spark: SparkSession): Unit
      
      
      
- Definition Classes
 - AnnotatorApproach
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        onWrite(path: String, spark: SparkSession): Unit
      
      
      
- Attributes
 - protected
 - Definition Classes
 - ParamsAndFeaturesWritable
 
 - 
      
      
      
        
      
    
      
        
        val
      
      
        optionalInputAnnotatorTypes: Array[String]
      
      
      
- Definition Classes
 - HasInputAnnotationCols
 
 - 
      
      
      
        
      
    
      
        
        val
      
      
        outputAnnotatorType: AnnotatorType
      
      
      
Output annotation type : WORD_EMBEDDINGS
Output annotation type : WORD_EMBEDDINGS
- Definition Classes
 - WordEmbeddings → HasOutputAnnotatorType
 
 - 
      
      
      
        
      
    
      
        final 
        val
      
      
        outputCol: Param[String]
      
      
      
- Attributes
 - protected
 - Definition Classes
 - HasOutputAnnotationCol
 
 - 
      
      
      
        
      
    
      
        
        lazy val
      
      
        params: Array[Param[_]]
      
      
      
- Definition Classes
 - Params
 
 - 
      
      
      
        
      
    
      
        
        val
      
      
        readCacheSize: IntParam
      
      
      
Cache size for items retrieved from storage.
Cache size for items retrieved from storage. Increase for performance but higher memory consumption
 - 
      
      
      
        
      
    
      
        
        def
      
      
        save(path: String): Unit
      
      
      
- Definition Classes
 - MLWritable
 - Annotations
 - @Since( "1.6.0" ) @throws( ... )
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        set[T](param: ProtectedParam[T], value: T): WordEmbeddings.this.type
      
      
      
Sets the value for a protected Param.
Sets the value for a protected Param.
If the parameter was already set, it will not be set again. Default values do not count as a set value and can be overridden.
- T
 Type of the parameter
- param
 Protected parameter to set
- value
 Value for the parameter
- returns
 This object
- Definition Classes
 - HasProtectedParams
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        set[T](feature: StructFeature[T], value: T): WordEmbeddings.this.type
      
      
      
- Attributes
 - protected
 - Definition Classes
 - HasFeatures
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        set[K, V](feature: MapFeature[K, V], value: Map[K, V]): WordEmbeddings.this.type
      
      
      
- Attributes
 - protected
 - Definition Classes
 - HasFeatures
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        set[T](feature: SetFeature[T], value: Set[T]): WordEmbeddings.this.type
      
      
      
- Attributes
 - protected
 - Definition Classes
 - HasFeatures
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        set[T](feature: ArrayFeature[T], value: Array[T]): WordEmbeddings.this.type
      
      
      
- Attributes
 - protected
 - Definition Classes
 - HasFeatures
 
 - 
      
      
      
        
      
    
      
        final 
        def
      
      
        set(paramPair: ParamPair[_]): WordEmbeddings.this.type
      
      
      
- Attributes
 - protected
 - Definition Classes
 - Params
 
 - 
      
      
      
        
      
    
      
        final 
        def
      
      
        set(param: String, value: Any): WordEmbeddings.this.type
      
      
      
- Attributes
 - protected
 - Definition Classes
 - Params
 
 - 
      
      
      
        
      
    
      
        final 
        def
      
      
        set[T](param: Param[T], value: T): WordEmbeddings.this.type
      
      
      
- Definition Classes
 - Params
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        setCaseSensitive(value: Boolean): WordEmbeddings.this.type
      
      
      
- Definition Classes
 - HasCaseSensitiveProperties
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        setDefault[T](feature: StructFeature[T], value: () ⇒ T): WordEmbeddings.this.type
      
      
      
- Attributes
 - protected
 - Definition Classes
 - HasFeatures
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        setDefault[K, V](feature: MapFeature[K, V], value: () ⇒ Map[K, V]): WordEmbeddings.this.type
      
      
      
- Attributes
 - protected
 - Definition Classes
 - HasFeatures
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        setDefault[T](feature: SetFeature[T], value: () ⇒ Set[T]): WordEmbeddings.this.type
      
      
      
- Attributes
 - protected
 - Definition Classes
 - HasFeatures
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        setDefault[T](feature: ArrayFeature[T], value: () ⇒ Array[T]): WordEmbeddings.this.type
      
      
      
- Attributes
 - protected
 - Definition Classes
 - HasFeatures
 
 - 
      
      
      
        
      
    
      
        final 
        def
      
      
        setDefault(paramPairs: ParamPair[_]*): WordEmbeddings.this.type
      
      
      
- Attributes
 - protected
 - Definition Classes
 - Params
 
 - 
      
      
      
        
      
    
      
        final 
        def
      
      
        setDefault[T](param: Param[T], value: T): WordEmbeddings.this.type
      
      
      
- Attributes
 - protected[org.apache.spark.ml]
 - Definition Classes
 - Params
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        setDimension(value: Int): WordEmbeddings.this.type
      
      
      
- Definition Classes
 - HasEmbeddingsProperties
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        setEnableInMemoryStorage(value: Boolean): WordEmbeddings.this.type
      
      
      
- Definition Classes
 - HasStorageOptions
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        setIncludeStorage(value: Boolean): WordEmbeddings.this.type
      
      
      
- Definition Classes
 - HasStorageOptions
 
 - 
      
      
      
        
      
    
      
        final 
        def
      
      
        setInputCols(value: String*): WordEmbeddings.this.type
      
      
      
- Definition Classes
 - HasInputAnnotationCols
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        setInputCols(value: Array[String]): WordEmbeddings.this.type
      
      
      
Overrides required annotators column if different than default
Overrides required annotators column if different than default
- Definition Classes
 - HasInputAnnotationCols
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        setLazyAnnotator(value: Boolean): WordEmbeddings.this.type
      
      
      
- Definition Classes
 - CanBeLazy
 
 - 
      
      
      
        
      
    
      
        final 
        def
      
      
        setOutputCol(value: String): WordEmbeddings.this.type
      
      
      
Overrides annotation column name when transforming
Overrides annotation column name when transforming
- Definition Classes
 - HasOutputAnnotationCol
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        setReadCacheSize(value: Int): WordEmbeddings.this.type
      
      
      
Cache size for items retrieved from storage.
Cache size for items retrieved from storage. Increase for performance but higher memory consumption.
 - 
      
      
      
        
      
    
      
        
        def
      
      
        setStoragePath(path: String, readAs: util.io.ReadAs.Value): WordEmbeddings.this.type
      
      
      
- Definition Classes
 - HasStorage
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        setStoragePath(path: String, readAs: String): WordEmbeddings.this.type
      
      
      
- Definition Classes
 - HasStorage
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        setStorageRef(value: String): WordEmbeddings.this.type
      
      
      
- Definition Classes
 - HasStorageRef
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        setWriteBufferSize(value: Int): WordEmbeddings.this.type
      
      
      
Buffer size limit before dumping to disk storage while writing.
 - 
      
      
      
        
      
    
      
        
        val
      
      
        storagePath: ExternalResourceParam
      
      
      
Path to the external resource.
Path to the external resource.
- Definition Classes
 - HasStorage
 
 - 
      
      
      
        
      
    
      
        
        val
      
      
        storageRef: Param[String]
      
      
      
Unique identifier for storage (Default:
this.uid)Unique identifier for storage (Default:
this.uid)- Definition Classes
 - HasStorageRef
 
 - 
      
      
      
        
      
    
      
        final 
        def
      
      
        synchronized[T0](arg0: ⇒ T0): T0
      
      
      
- Definition Classes
 - AnyRef
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        toString(): String
      
      
      
- Definition Classes
 - Identifiable → AnyRef → Any
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        train(dataset: Dataset[_], recursivePipeline: Option[PipelineModel]): WordEmbeddingsModel
      
      
      
- Definition Classes
 - WordEmbeddings → AnnotatorApproach
 
 - 
      
      
      
        
      
    
      
        final 
        def
      
      
        transformSchema(schema: StructType): StructType
      
      
      
requirement for pipeline transformation validation.
requirement for pipeline transformation validation. It is called on fit()
- Definition Classes
 - AnnotatorApproach → PipelineStage
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        transformSchema(schema: StructType, logging: Boolean): StructType
      
      
      
- Attributes
 - protected
 - Definition Classes
 - PipelineStage
 - Annotations
 - @DeveloperApi()
 
 - 
      
      
      
        
      
    
      
        
        val
      
      
        uid: String
      
      
      
- Definition Classes
 - WordEmbeddings → Identifiable
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        validate(schema: StructType): Boolean
      
      
      
takes a Dataset and checks to see if all the required annotation types are present.
takes a Dataset and checks to see if all the required annotation types are present.
- schema
 to be validated
- returns
 True if all the required types are present, else false
- Attributes
 - protected
 - Definition Classes
 - AnnotatorApproach
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        validateStorageRef(dataset: Dataset[_], inputCols: Array[String], annotatorType: String): Unit
      
      
      
- Definition Classes
 - HasStorageRef
 
 - 
      
      
      
        
      
    
      
        final 
        def
      
      
        wait(): Unit
      
      
      
- Definition Classes
 - AnyRef
 - Annotations
 - @throws( ... )
 
 - 
      
      
      
        
      
    
      
        final 
        def
      
      
        wait(arg0: Long, arg1: Int): Unit
      
      
      
- Definition Classes
 - AnyRef
 - Annotations
 - @throws( ... )
 
 - 
      
      
      
        
      
    
      
        final 
        def
      
      
        wait(arg0: Long): Unit
      
      
      
- Definition Classes
 - AnyRef
 - Annotations
 - @throws( ... ) @native()
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        wrapEmbeddingsMetadata(col: Column, embeddingsDim: Int, embeddingsRef: Option[String] = None): Column
      
      
      
- Attributes
 - protected
 - Definition Classes
 - HasEmbeddingsProperties
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        wrapSentenceEmbeddingsMetadata(col: Column, embeddingsDim: Int, embeddingsRef: Option[String] = None): Column
      
      
      
- Attributes
 - protected
 - Definition Classes
 - HasEmbeddingsProperties
 
 - 
      
      
      
        
      
    
      
        
        def
      
      
        write: MLWriter
      
      
      
- Definition Classes
 - ParamsAndFeaturesWritable → DefaultParamsWritable → MLWritable
 
 - 
      
      
      
        
      
    
      
        
        val
      
      
        writeBufferSize: IntParam
      
      
      
Buffer size limit before dumping to disk storage while writing
 
Inherited from HasEmbeddingsProperties
Inherited from HasProtectedParams
Inherited from HasStorage
Inherited from HasCaseSensitiveProperties
Inherited from HasStorageOptions
Inherited from HasStorageRef
Inherited from ParamsAndFeaturesWritable
Inherited from HasFeatures
Inherited from AnnotatorApproach[WordEmbeddingsModel]
Inherited from CanBeLazy
Inherited from DefaultParamsWritable
Inherited from MLWritable
Inherited from HasOutputAnnotatorType
Inherited from HasOutputAnnotationCol
Inherited from HasInputAnnotationCols
Inherited from Estimator[WordEmbeddingsModel]
Inherited from PipelineStage
Inherited from Logging
Inherited from Params
Inherited from Serializable
Inherited from Serializable
Inherited from Identifiable
Inherited from AnyRef
Inherited from Any
Parameters
A list of (hyper-)parameter keys this annotator can take. Users can set and get the parameter values through setters and getters, respectively.
Annotator types
Required input and expected output annotator types