args:
- auto: PREDEFINED
  defaultValue: 'yes'
  description: Clean html from text value?
  name: cleanHtml
  predefined:
  - 'yes'
  - 'no'
- auto: PREDEFINED
  defaultValue: 'yes'
  description: Remove line breaks?
  name: removeLineBreaks
  predefined:
  - 'yes'
  - 'no'
- defaultValue: utf-8
  description: Text encoding
  name: encoding
- description: If non-empty hash the words with this seed.
  name: hashWordWithSeed
- auto: PREDEFINED
  defaultValue: 'no'
  description: Remove non-english words
  name: removeNonEnglishWords
  predefined:
  - 'yes'
  - 'no'
- auto: PREDEFINED
  defaultValue: 'yes'
  description: Remove english stop words ("the", "a", "an", etc)
  name: removeStopWords
  predefined:
  - 'yes'
  - 'no'
- auto: PREDEFINED
  defaultValue: 'yes'
  description: Remove punctuation
  name: removePunctuation
  predefined:
  - 'yes'
  - 'no'
- auto: PREDEFINED
  defaultValue: 'yes'
  description: Replace emails with magic
  name: replaceEmails
  predefined:
  - 'yes'
  - 'no'
- auto: PREDEFINED
  defaultValue: 'yes'
  description: Replace urls with magic
  name: replaceUrls
  predefined:
  - 'yes'
  - 'no'
- auto: PREDEFINED
  defaultValue: 'yes'
  description: Replace numbers with magic
  name: replaceNumbers
  predefined:
  - 'yes'
  - 'no'
- auto: PREDEFINED
  defaultValue: 'no'
  description: Remove non-alphabetic words
  name: removeNonAlphaWords
  predefined:
  - 'yes'
  - 'no'
- auto: PREDEFINED
  defaultValue: 'yes'
  description: 'Use lemmatization. you can read more about lemma here: https://en.wikipedia.org/wiki/Lemmatisation'
  name: useLemmatization
  predefined:
  - 'yes'
  - 'no'
- description: The text value
  name: value
  required: true
- auto: PREDEFINED
  defaultValue: 'no'
  description: Is the input text value is json encoded?
  name: isValueJson
  predefined:
  - 'yes'
  - 'no'
- auto: PREDEFINED
  defaultValue: English
  description: The language of the input text. Default is "English". Can be "English", "German", "French", "Spanish", "Portuguese", "Italian", "Dutch", or "Other".
  name: language
  predefined:
  - English
  - German
  - French
  - Spanish
  - Portuguese
  - Italian
  - Dutch
  - Other
- auto: PREDEFINED
  defaultValue: tokenizer
  description: Tokenization method for text. Only required when the language argument is set to "Other". Can be "tokenizer", "byWords", or "byLetters".
  name: tokenizationMethod
  predefined:
  - tokenizer
  - byWords
  - byLetters
comment: Deprecated. Use DBotPreProcessTextData instead.
commonfields:
  id: WordTokenizerNLP
  version: -1
enabled: true
name: WordTokenizerNLP
outputs:
- contextPath: WordTokenizeOutput
  description: Output text
  type: Unknown
script: '-'
subtype: python2
tags:
- phishing
- ml
timeout: 60µs
type: python
dockerimage: demisto/dl:1.4
runas: DBotWeakRole
runonce: true
tests:
- No test
fromversion: 5.0.0
deprecated: true