object ResourceDownloader
- Alphabetic
- By Inheritance
- ResourceDownloader
- AnyRef
- Any
- Hide All
- Show All
- Public
- All
Value Members
- def cacheFolder: String
- def clearCache(request: ResourceRequest): Unit
- def clearCache(name: String, language: Option[String] = None, folder: String = publicLoc): Unit
- var communityDownloader: ResourceDownloader
- def downloadModel[TModel <: PipelineStage](reader: DefaultParamsReadable[TModel], request: ResourceRequest): TModel
- def downloadModel[TModel <: PipelineStage](reader: DefaultParamsReadable[TModel], name: String, language: Option[String] = None, folder: String = publicLoc): TModel
-
def
downloadModelDirectly(model: String, folder: String = publicLoc, unzip: Boolean = true): Unit
Downloads a model from the default S3 bucket to the cache pretrained folder.
Downloads a model from the default S3 bucket to the cache pretrained folder.
- model
the name of the key in the S3 bucket or s3 URI
- folder
the folder of the model
- unzip
used to unzip the model, by default true
- def downloadPipeline(request: ResourceRequest): PipelineModel
- def downloadPipeline(name: String, language: Option[String] = None, folder: String = publicLoc): PipelineModel
-
def
downloadResource(request: ResourceRequest): String
Loads resource to path
Loads resource to path
- request
Request for resource
- returns
path of downloaded resource
-
def
downloadResource(name: String, language: Option[String] = None, folder: String = publicLoc): String
Loads resource to path
Loads resource to path
- name
Name of Resource
- language
Desired language of Resource
- folder
Subfolder in s3 where to search model (e.g. medicine)
- returns
path of downloaded resource
- val fileSystem: FileSystem
- def getDownloadSize(resourceRequest: ResourceRequest): String
- def getResourceDownloader(folder: String): ResourceDownloader
- lazy val libVersion: Version
- def listAvailableAnnotators(folder: String = publicLoc): List[String]
- def listPretrainedResources(folder: String, resourceType: ResourceType, lang: String, version: Version): List[String]
- def listPretrainedResources(folder: String, resourceType: ResourceType, version: Version): List[String]
- def listPretrainedResources(folder: String, resourceType: ResourceType, lang: String): List[String]
-
def
listPretrainedResources(folder: String, resourceType: ResourceType, annotator: Option[String] = None, lang: Option[String] = None, version: Option[Version] = None): List[String]
Lists pretrained resource from metadata.json, depending on the set filters.
Lists pretrained resource from metadata.json, depending on the set filters. The folder in the S3 location and the resourceType is necessary. The other filters are optional and will be ignored if not set.
- folder
Folder in the S3 location
- resourceType
Type of the Resource. Can Either
ResourceType.MODEL
,ResourceType.PIPELINE
orResourceType.NOT_DEFINED
- annotator
Name of the model class
- lang
Language of the model
- version
Version that the model should be compatible with
- returns
A list of the available resources
-
def
listPublicModels(): List[String]
List all pretrained models in public name_lang
-
def
listPublicPipelines(): List[String]
List all pretrained pipelines in public
-
def
listUnCategorizedResources(): List[String]
Returns models or pipelines in metadata json which has not been categorized yet.
Returns models or pipelines in metadata json which has not been categorized yet.
- returns
list of models or pipelines which are not categorized in metadata json
- var privateDownloader: ResourceDownloader
- var publicDownloader: ResourceDownloader
- val publicLoc: String
- def publicResourceString(annotator: Option[String] = None, lang: Option[String] = None, version: Option[String] = Some(Build.version), resourceType: ResourceType): String
-
def
resetResourceDownloader(): Unit
Reset the cache and recreate ResourceDownloader S3 credentials
- def s3Bucket: String
- def s3BucketCommunity: String
- def s3Path: String
- def showAvailableAnnotators(folder: String = publicLoc): Unit
-
def
showPublicModels(annotator: String, lang: String, version: String): Unit
Prints all pretrained models for a particular annotator, that are compatible with a version of Spark NLP.
Prints all pretrained models for a particular annotator, that are compatible with a version of Spark NLP.
- annotator
Name of the model class, for example "NerDLModel"
- lang
Language of the pretrained models to display, for example "en"
- version
Version of Spark NLP that the model should be compatible with, for example "3.2.3"
-
def
showPublicModels(annotator: String, lang: String): Unit
Prints all pretrained models for a particular annotator model, that are compatible with this version of Spark NLP.
Prints all pretrained models for a particular annotator model, that are compatible with this version of Spark NLP.
- annotator
Name of the annotator class
- lang
Language of the pretrained models to display
-
def
showPublicModels(annotator: String): Unit
Prints all pretrained models for a particular annotator model, that are compatible with this version of Spark NLP.
Prints all pretrained models for a particular annotator model, that are compatible with this version of Spark NLP.
- annotator
Name of the annotator class
-
def
showPublicModels(annotator: Option[String] = None, lang: Option[String] = None, version: Option[String] = Some(Build.version)): Unit
Prints all pretrained models for a particular annotator model, that are compatible with a version of Spark NLP.
Prints all pretrained models for a particular annotator model, that are compatible with a version of Spark NLP. If any of the optional arguments are not set, the filter is not considered.
- annotator
Name of the model class, for example "NerDLModel"
- lang
Language of the pretrained models to display, for example "en"
- version
Version of Spark NLP that the model should be compatible with, for example "3.2.3"
-
def
showPublicPipelines(lang: String, version: String): Unit
Prints all Pipelines available for a language and a version of Spark NLP.
Prints all Pipelines available for a language and a version of Spark NLP.
- lang
Language of the Pipeline
- version
Version of Spark NLP
-
def
showPublicPipelines(lang: String): Unit
Prints all Pipelines available for a language and this version of Spark NLP.
Prints all Pipelines available for a language and this version of Spark NLP.
- lang
Language of the Pipeline
-
def
showPublicPipelines(lang: Option[String] = None, version: Option[String] = Some(Build.version)): Unit
Prints all Pipelines available for a language and a version of Spark NLP.
Prints all Pipelines available for a language and a version of Spark NLP. By default shows all languages and uses the current version of Spark NLP.
- lang
Language of the Pipeline
- version
Version of Spark NLP
- def showString(list: List[String], resourceType: ResourceType): String
- def showUnCategorizedResources(lang: String, version: String): Unit
- def showUnCategorizedResources(lang: String): Unit
- lazy val sparkVersion: Version