args:
- description: The input file entry ID or the file content (as a string).
  name: input
  required: true
- defaultValue: '15'
  description: Sample text for which the total number words are less than or equal to this number will be ignored.
  name: removeShortTextThreshold
- defaultValue: '0.99'
  description: Remove emails with similarity greater than this threshold, range 0-1, where 1 is completly identical.
  name: dedupThreshold
- defaultValue: emailsubject|name,emailbody|emailbodyhtml
  description: A comma-separated list of incident field names with the text to process. You can also use "|" if you want to choose the first non-empty value from a list of fields.
  name: textFields
- auto: PREDEFINED
  defaultValue: json
  description: The input type.
  name: inputType
  predefined:
  - json
  - pickle
  - csv
  - json_string
  - pickle_string
  - csv_string
  - json_b64_string
  - pickle_b64_string
  - csv_b64_string
  - string
- auto: PREDEFINED
  defaultValue: nlp
  description: Text pre-processing type. The default is "json".
  name: preProcessType
  predefined:
  - nlp
  - none
- auto: PREDEFINED
  defaultValue: 'true'
  description: Whether to remove HTML tags. Default is "true".
  name: cleanHTML
  predefined:
  - 'true'
  - 'false'
- description: A comma-separate list of fields inside the JSON by which to filter.
  name: whitelistFields
- description: If non-empty, hash every word with this seed.
  name: hashSeed
- auto: PREDEFINED
  defaultValue: pickle
  description: The output file format.
  name: outputFormat
  predefined:
  - json
  - pickle
- auto: PREDEFINED
  defaultValue: 'false'
  description: Whether to add the original text fields to the output. Default is "false".
  name: outputOriginalTextFields
  predefined:
  - 'true'
  - 'false'
- auto: PREDEFINED
  defaultValue: Any
  description: The language of the input text. Default is "Any". Can be "Any", "English", "German", "French", "Spanish", "Portuguese", "Italian", "Dutch", or "Other". If "Any"  or "Other" is selected, the script preprocess the entire input, no matter what its acutual language is. If a specific language is selected, the script filters out any other language from the output text.
  name: language
  predefined:
  - Any
  - English
  - German
  - French
  - Spanish
  - Portuguese
  - Italian
  - Dutch
  - Other
- auto: PREDEFINED
  defaultValue: tokenizer
  description: Tokenization method for text. Only required when the language argument is set to "Other". Can be "tokenizer", "byWords", or "byLetters". Default is "tokenizer".
  name: tokenizationMethod
  predefined:
  - tokenizer
  - byWords
  - byLetters
comment: Pre-process text data for the machine learning text classifier.
commonfields:
  id: DBotPreProcessTextData
  version: -1
name: DBotPreProcessTextData
outputs:
- contextPath: DBotPreProcessTextData.Filename
  description: The output file name.
  type: String
- contextPath: DBotPreProcessTextData.TextField
  description: The original text field inside the file.
  type: String
- contextPath: DBotPreProcessTextData.TextFieldProcessed
  description: The processed text field inside the JSON file.
  type: String
- contextPath: DBotPreProcessTextData.FileFormat
  description: The output file format.
  type: String
script: '-'
subtype: python3
tags:
- ml
timeout: 120µs
type: python
dockerimage: demisto/ml:1.0.0.12042988
tests:
- Create Phishing Classifier V2 ML Test
fromversion: 5.0.0