class Reader2Image extends Transformer with DefaultParamsWritable with HasOutputAnnotatorType with HasOutputAnnotationCol with HasBinaryReaderProperties with HasReaderContent
The Reader2Image annotator allows you to use the reading files with images more smoothly within existing Spark NLP workflows, enabling seamless reuse of your pipelines. Reader2Image can be used for extracting structured image content from various document types using Spark NLP readers. It supports reading from many files types and returns parsed output as a structured Spark DataFrame.
Supported formats include HTML and Markdown
Example
import com.johnsnowlabs.reader.Reader2Image import com. johnsnowlabs.nlp.base.DocumentAssembler import org.apache.spark.ml.Pipeline val reader2Image = new Reader2Image() .setContentType("text/html") .setContentPath("./example-images.html") .setOutputCol("image") val pipeline = new Pipeline() .setStages(Array(reader2Image)) val pipelineModel = pipeline.fit(emptyDataSet) val resultDf = pipelineModel.transform(emptyDataSet) resultDf.show() +-------------------+--------------------+ | fileName| image| +-------------------+--------------------+ |example-images.html|[{image, example-...| |example-images.html|[{image, example-...| +-------------------+--------------------+ resultDf.printSchema() root |-- fileName: string (nullable = true) |-- image: array (nullable = false) | |-- element: struct (containsNull = true) | | |-- annotatorType: string (nullable = true) | | |-- origin: string (nullable = true) | | |-- height: integer (nullable = false) | | |-- width: integer (nullable = false) | | |-- nChannels: integer (nullable = false) | | |-- mode: integer (nullable = false) | | |-- result: binary (nullable = true) | | |-- metadata: map (nullable = true) | | | |-- key: string | | | |-- value: string (valueContainsNull = true) | | |-- text: string (nullable = true)
Linear Supertypes
Ordering
- Alphabetic
- By Inheritance
Inherited
- Reader2Image
- HasReaderContent
- HasTagsReaderProperties
- HasXmlReaderProperties
- HasHTMLReaderProperties
- HasReaderProperties
- HasBinaryReaderProperties
- HasPowerPointProperties
- HasPdfReaderProperties
- HasExcelReaderProperties
- HasEmailReaderProperties
- ParamsAndFeaturesWritable
- HasFeatures
- HasOutputAnnotationCol
- HasOutputAnnotatorType
- DefaultParamsWritable
- MLWritable
- Transformer
- PipelineStage
- Logging
- Params
- Serializable
- Serializable
- Identifiable
- AnyRef
- Any
- Hide All
- Show All
Visibility
- Public
- All
Type Members
-
type
AnnotatorType = String
- Definition Classes
- HasOutputAnnotatorType
Value Members
-
final
def
!=(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
-
final
def
##(): Int
- Definition Classes
- AnyRef → Any
-
final
def
$[T](param: Param[T]): T
- Attributes
- protected
- Definition Classes
- Params
-
def
$$[T](feature: StructFeature[T]): T
- Attributes
- protected
- Definition Classes
- HasFeatures
-
def
$$[K, V](feature: MapFeature[K, V]): Map[K, V]
- Attributes
- protected
- Definition Classes
- HasFeatures
-
def
$$[T](feature: SetFeature[T]): Set[T]
- Attributes
- protected
- Definition Classes
- HasFeatures
-
def
$$[T](feature: ArrayFeature[T]): Array[T]
- Attributes
- protected
- Definition Classes
- HasFeatures
-
final
def
==(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
-
val
addAttachmentContent: Param[Boolean]
- Definition Classes
- HasEmailReaderProperties
- def afterAnnotate(dataset: DataFrame): DataFrame
-
val
appendCells: Param[Boolean]
- Definition Classes
- HasExcelReaderProperties
-
final
def
asInstanceOf[T0]: T0
- Definition Classes
- Any
-
def
buildEmptyDataFrame(dataset: Dataset[_]): DataFrame
- Definition Classes
- HasReaderContent
-
def
buildErrorDataFrame(dataset: Dataset[_], contentPath: String, ext: String): DataFrame
- Definition Classes
- HasReaderContent
-
val
cellSeparator: Param[String]
- Definition Classes
- HasExcelReaderProperties
-
final
def
clear(param: Param[_]): Reader2Image.this.type
- Definition Classes
- Params
-
def
clone(): AnyRef
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws( ... ) @native()
-
val
contentPath: Param[String]
- Definition Classes
- HasReaderProperties
-
val
contentType: Param[String]
- Definition Classes
- HasReaderProperties
-
def
copy(extra: ParamMap): Transformer
- Definition Classes
- Reader2Image → Transformer → PipelineStage → Params
-
def
copyValues[T <: Params](to: T, extra: ParamMap): T
- Attributes
- protected
- Definition Classes
- Params
- val customPromptTemplate: Param[String]
-
final
def
defaultCopy[T <: Params](extra: ParamMap): T
- Attributes
- protected
- Definition Classes
- Params
-
final
def
eq(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
-
def
equals(arg0: Any): Boolean
- Definition Classes
- AnyRef → Any
-
def
explainParam(param: Param[_]): String
- Definition Classes
- Params
-
def
explainParams(): String
- Definition Classes
- Params
-
val
explodeDocs: BooleanParam
- Definition Classes
- HasReaderProperties
-
final
def
extractParamMap(): ParamMap
- Definition Classes
- Params
-
final
def
extractParamMap(extra: ParamMap): ParamMap
- Definition Classes
- Params
-
val
features: ArrayBuffer[Feature[_, _, _]]
- Definition Classes
- HasFeatures
-
def
finalize(): Unit
- Attributes
- protected[lang]
- Definition Classes
- AnyRef
- Annotations
- @throws( classOf[java.lang.Throwable] )
-
val
flattenOutput: BooleanParam
- Definition Classes
- HasReaderProperties
-
def
get[T](feature: StructFeature[T]): Option[T]
- Attributes
- protected
- Definition Classes
- HasFeatures
-
def
get[K, V](feature: MapFeature[K, V]): Option[Map[K, V]]
- Attributes
- protected
- Definition Classes
- HasFeatures
-
def
get[T](feature: SetFeature[T]): Option[Set[T]]
- Attributes
- protected
- Definition Classes
- HasFeatures
-
def
get[T](feature: ArrayFeature[T]): Option[Array[T]]
- Attributes
- protected
- Definition Classes
- HasFeatures
-
final
def
get[T](param: Param[T]): Option[T]
- Definition Classes
- Params
-
final
def
getClass(): Class[_]
- Definition Classes
- AnyRef → Any
- Annotations
- @native()
-
def
getContentType: String
- Definition Classes
- HasReaderContent
-
final
def
getDefault[T](param: Param[T]): Option[T]
- Definition Classes
- Params
-
val
getFileName: UserDefinedFunction
- Definition Classes
- HasReaderContent
-
final
def
getInputCol: String
- Definition Classes
- HasReaderProperties
-
final
def
getOrDefault[T](param: Param[T]): T
- Definition Classes
- Params
-
final
def
getOutputCol: String
Gets annotation column name going to generate
Gets annotation column name going to generate
- Definition Classes
- HasOutputAnnotationCol
-
def
getParam(paramName: String): Param[Any]
- Definition Classes
- Params
-
final
def
hasDefault[T](param: Param[T]): Boolean
- Definition Classes
- Params
-
def
hasParam(paramName: String): Boolean
- Definition Classes
- Params
-
def
hashCode(): Int
- Definition Classes
- AnyRef → Any
- Annotations
- @native()
-
val
headers: Param[Map[String, String]]
- Definition Classes
- HasHTMLReaderProperties
-
val
ignoreExceptions: BooleanParam
- Definition Classes
- HasReaderProperties
-
val
includePageBreaks: Param[Boolean]
- Definition Classes
- HasReaderProperties
-
val
includeSlideNotes: Param[Boolean]
- Definition Classes
- HasPowerPointProperties
-
val
includeTitleTag: Param[Boolean]
- Definition Classes
- HasHTMLReaderProperties
-
val
inferTableStructure: Param[Boolean]
- Definition Classes
- HasReaderProperties
-
def
initializeLogIfNecessary(isInterpreter: Boolean, silent: Boolean): Boolean
- Attributes
- protected
- Definition Classes
- Logging
-
def
initializeLogIfNecessary(isInterpreter: Boolean): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
final
val
inputCol: Param[String]
- Attributes
- protected
- Definition Classes
- HasReaderProperties
-
final
def
isDefined(param: Param[_]): Boolean
- Definition Classes
- Params
-
final
def
isInstanceOf[T0]: Boolean
- Definition Classes
- Any
-
final
def
isSet(param: Param[_]): Boolean
- Definition Classes
- Params
-
def
isTraceEnabled(): Boolean
- Attributes
- protected
- Definition Classes
- Logging
-
def
listAllFilesRecursively(dir: File): Seq[File]
- Definition Classes
- HasReaderContent
-
def
log: Logger
- Attributes
- protected
- Definition Classes
- Logging
-
def
logDebug(msg: ⇒ String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logDebug(msg: ⇒ String): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logError(msg: ⇒ String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logError(msg: ⇒ String): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logInfo(msg: ⇒ String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logInfo(msg: ⇒ String): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logName: String
- Attributes
- protected
- Definition Classes
- Logging
-
def
logTrace(msg: ⇒ String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logTrace(msg: ⇒ String): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logWarning(msg: ⇒ String, throwable: Throwable): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
def
logWarning(msg: ⇒ String): Unit
- Attributes
- protected
- Definition Classes
- Logging
-
final
def
ne(arg0: AnyRef): Boolean
- Definition Classes
- AnyRef
-
final
def
notify(): Unit
- Definition Classes
- AnyRef
- Annotations
- @native()
-
final
def
notifyAll(): Unit
- Definition Classes
- AnyRef
- Annotations
- @native()
-
def
onWrite(path: String, spark: SparkSession): Unit
- Attributes
- protected
- Definition Classes
- ParamsAndFeaturesWritable
-
val
onlyLeafNodes: Param[Boolean]
- Definition Classes
- HasXmlReaderProperties
-
val
outputAnnotatorType: AnnotatorType
- Definition Classes
- Reader2Image → HasOutputAnnotatorType
-
final
val
outputCol: Param[String]
- Attributes
- protected
- Definition Classes
- HasOutputAnnotationCol
-
val
outputFormat: Param[String]
- Definition Classes
- HasHTMLReaderProperties
-
lazy val
params: Array[Param[_]]
- Definition Classes
- Params
-
def
partitionBuilder: Partition
- Attributes
- protected
-
def
partitionContent(partition: Partition, contentPath: String, isText: Boolean, dataset: Dataset[_]): DataFrame
- Definition Classes
- Reader2Image → HasReaderContent
-
def
partitionContentFromPath(partition: Partition, contentPath: String, isText: Boolean, dataset: Dataset[_]): DataFrame
- Definition Classes
- HasReaderContent
-
def
partitionMixedContent(dataset: Dataset[_], dirPath: String, partitionParams: Map[String, String]): DataFrame
- Definition Classes
- HasReaderContent
- def partitionToAnnotation: UserDefinedFunction
- val promptTemplate: Param[String]
-
final
val
readAsImage: BooleanParam
- Definition Classes
- HasPdfReaderProperties
-
def
retrieveFileName(path: String): String
- Definition Classes
- HasReaderContent
-
def
save(path: String): Unit
- Definition Classes
- MLWritable
- Annotations
- @Since( "1.6.0" ) @throws( ... )
-
def
set[T](feature: StructFeature[T], value: T): Reader2Image.this.type
- Attributes
- protected
- Definition Classes
- HasFeatures
-
def
set[K, V](feature: MapFeature[K, V], value: Map[K, V]): Reader2Image.this.type
- Attributes
- protected
- Definition Classes
- HasFeatures
-
def
set[T](feature: SetFeature[T], value: Set[T]): Reader2Image.this.type
- Attributes
- protected
- Definition Classes
- HasFeatures
-
def
set[T](feature: ArrayFeature[T], value: Array[T]): Reader2Image.this.type
- Attributes
- protected
- Definition Classes
- HasFeatures
-
final
def
set(paramPair: ParamPair[_]): Reader2Image.this.type
- Attributes
- protected
- Definition Classes
- Params
-
final
def
set(param: String, value: Any): Reader2Image.this.type
- Attributes
- protected
- Definition Classes
- Params
-
final
def
set[T](param: Param[T], value: T): Reader2Image.this.type
- Definition Classes
- Params
-
def
setAddAttachmentContent(value: Boolean): Reader2Image.this.type
- Definition Classes
- HasEmailReaderProperties
-
def
setAppendCells(value: Boolean): Reader2Image.this.type
- Definition Classes
- HasExcelReaderProperties
-
def
setCellSeparator(value: String): Reader2Image.this.type
- Definition Classes
- HasExcelReaderProperties
-
def
setContentPath(value: String): Reader2Image.this.type
- Definition Classes
- HasReaderProperties
-
def
setContentType(value: String): Reader2Image.this.type
- Definition Classes
- HasReaderProperties
- def setCustomPromptTemplate(value: String): Reader2Image.this.type
-
def
setDefault[T](feature: StructFeature[T], value: () ⇒ T): Reader2Image.this.type
- Attributes
- protected
- Definition Classes
- HasFeatures
-
def
setDefault[K, V](feature: MapFeature[K, V], value: () ⇒ Map[K, V]): Reader2Image.this.type
- Attributes
- protected
- Definition Classes
- HasFeatures
-
def
setDefault[T](feature: SetFeature[T], value: () ⇒ Set[T]): Reader2Image.this.type
- Attributes
- protected
- Definition Classes
- HasFeatures
-
def
setDefault[T](feature: ArrayFeature[T], value: () ⇒ Array[T]): Reader2Image.this.type
- Attributes
- protected
- Definition Classes
- HasFeatures
-
final
def
setDefault(paramPairs: ParamPair[_]*): Reader2Image.this.type
- Attributes
- protected
- Definition Classes
- Params
-
final
def
setDefault[T](param: Param[T], value: T): Reader2Image.this.type
- Attributes
- protected[org.apache.spark.ml]
- Definition Classes
- Params
-
def
setExplodeDocs(value: Boolean): Reader2Image.this.type
- Definition Classes
- HasReaderProperties
-
def
setFlattenOutput(value: Boolean): Reader2Image.this.type
- Definition Classes
- HasReaderProperties
-
def
setHeaders(value: Map[String, String]): Reader2Image.this.type
- Definition Classes
- HasHTMLReaderProperties
-
def
setHeadersPython(headers: Map[String, String]): Reader2Image.this.type
- Definition Classes
- HasHTMLReaderProperties
-
def
setIgnoreExceptions(value: Boolean): Reader2Image.this.type
- Definition Classes
- HasReaderProperties
-
def
setIncludePageBreaks(value: Boolean): Reader2Image.this.type
- Definition Classes
- HasReaderProperties
-
def
setIncludeSlideNotes(value: Boolean): Reader2Image.this.type
- Definition Classes
- HasPowerPointProperties
-
def
setIncludeTitleTag(value: Boolean): Reader2Image.this.type
- Definition Classes
- HasHTMLReaderProperties
-
def
setInferTableStructure(value: Boolean): Reader2Image.this.type
- Definition Classes
- HasReaderProperties
-
final
def
setInputCol(value: String): Reader2Image.this.type
- Definition Classes
- HasReaderProperties
-
def
setOnlyLeafNodes(value: Boolean): Reader2Image.this.type
- Definition Classes
- HasXmlReaderProperties
-
final
def
setOutputCol(value: String): Reader2Image.this.type
Overrides annotation column name when transforming
Overrides annotation column name when transforming
- Definition Classes
- HasOutputAnnotationCol
-
def
setOutputFormat(value: String): Reader2Image.this.type
- Definition Classes
- HasHTMLReaderProperties
- def setPromptTemplate(value: String): Reader2Image.this.type
-
def
setStoreContent(value: Boolean): Reader2Image.this.type
- Definition Classes
- HasReaderProperties
-
def
setTimeout(value: Int): Reader2Image.this.type
- Definition Classes
- HasHTMLReaderProperties
-
def
setTitleFontSize(value: Int): Reader2Image.this.type
- Definition Classes
- HasReaderProperties
-
def
setTitleThreshold(value: Double): Reader2Image.this.type
- Definition Classes
- HasPdfReaderProperties
- def setUserMessage(value: String): Reader2Image.this.type
-
def
setXmlKeepTags(value: Boolean): Reader2Image.this.type
- Definition Classes
- HasXmlReaderProperties
-
val
storeContent: Param[Boolean]
- Definition Classes
- HasReaderProperties
-
val
supportedTypes: Map[String, (String, Boolean)]
- Definition Classes
- Reader2Image → HasReaderContent
-
final
def
synchronized[T0](arg0: ⇒ T0): T0
- Definition Classes
- AnyRef
-
val
timeout: Param[Int]
- Definition Classes
- HasHTMLReaderProperties
-
val
titleFontSize: Param[Int]
- Definition Classes
- HasReaderProperties
-
val
titleThreshold: Param[Double]
- Definition Classes
- HasPdfReaderProperties
-
def
toString(): String
- Definition Classes
- Identifiable → AnyRef → Any
-
def
transform(dataset: Dataset[_]): DataFrame
- Definition Classes
- Reader2Image → Transformer
-
def
transform(dataset: Dataset[_], paramMap: ParamMap): DataFrame
- Definition Classes
- Transformer
- Annotations
- @Since( "2.0.0" )
-
def
transform(dataset: Dataset[_], firstParamPair: ParamPair[_], otherParamPairs: ParamPair[_]*): DataFrame
- Definition Classes
- Transformer
- Annotations
- @Since( "2.0.0" ) @varargs()
-
def
transformSchema(schema: StructType): StructType
- Definition Classes
- Reader2Image → PipelineStage
-
def
transformSchema(schema: StructType, logging: Boolean): StructType
- Attributes
- protected
- Definition Classes
- PipelineStage
- Annotations
- @DeveloperApi()
-
val
uid: String
- Definition Classes
- Reader2Image → Identifiable
- val userMessage: Param[String]
-
def
validateRequiredParameters(): Unit
- Attributes
- protected
-
final
def
wait(): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... )
-
final
def
wait(arg0: Long, arg1: Int): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... )
-
final
def
wait(arg0: Long): Unit
- Definition Classes
- AnyRef
- Annotations
- @throws( ... ) @native()
-
def
write: MLWriter
- Definition Classes
- ParamsAndFeaturesWritable → DefaultParamsWritable → MLWritable
-
val
xmlKeepTags: Param[Boolean]
- Definition Classes
- HasXmlReaderProperties