CommonServerPython
Common code that will be merged into each server script when it runs.
- Type
- python
- Pack
- Base
Source
"""Common functions script
This script will be appended to each server script before being executed.
Please notice that to add custom common code, add it to the CommonServerUserPython script.
Note that adding code to CommonServerUserPython can override functions in CommonServerPython
"""
# If you change this section, make sure you update the line offset magic number
from __future__ import print_function
import base64
import binascii
import gc
import io as _io
import json
import logging
import os
import pickle as _pickle
import pickletools
import re
import socket
import sys
import time
import traceback
import types
import urllib
import gzip
import ssl
import signal
from random import randint
import xml.etree.cElementTree as ET
from collections import OrderedDict
from datetime import datetime, timedelta
from abc import abstractmethod
from distutils.version import LooseVersion
from threading import Lock
from functools import wraps
from inspect import currentframe
import demistomock as demisto
import warnings
def __line__():
cf = currentframe()
return cf.f_back.f_lineno # type: ignore[union-attr]
# The number is the line offset from the beginning of the file. If you added an import, update this number accordingly.
_MODULES_LINE_MAPPING = {
'CommonServerPython': {'start': __line__() - 50, 'end': float('inf')},
}
XSIAM_EVENT_CHUNK_SIZE = 2 ** 20 # 1 Mib
XSIAM_EVENT_CHUNK_SIZE_LIMIT = 9 * (10 ** 6) # 9 MB, note that the allowed max size for 1 entry is 5MB.
# So if you're using a "heavy" API it is recommended to use maximum of 4MB chunk size.
MAX_ALLOWED_ENTRY_SIZE = 5 * (10 ** 6) # 5 MB, this is the maximum allowed size of a single entry.
# So if you're using a "heavy" API it is recommended to use maximum of 9MB chunk size.
ASSETS = "assets"
EVENTS = "events"
DATA_TYPES = [EVENTS, ASSETS]
MASK = '<XX_REPLACED>'
SEND_PREFIX = "send: b'"
SAFE_SLEEP_START_TIME = datetime.now()
MAX_ERROR_MESSAGE_LENGTH = 50000
# Max number of characters of an external API response body kept when appended
# to a CortexExternalApiError message (avoids dumping huge payloads).
MAX_API_RESPONSE_BODY_LENGTH = 500
NUM_OF_WORKERS = 20
HAVE_SUPPORT_MULTITHREADING_CALLED_ONCE = False
JSON_SEPARATORS = (",", ":") # To get the most compact JSON representation, we should specify (',', ':') to eliminate whitespace.
DEFAULT_INSIGHT_CACHE_SIZE = 3072
FETCH_COMMANDS = ('fetch-incidents', 'fetch-credentials', 'fetch-indicators', 'fetch-assets')
LONG_RUNNING_COMMAND = 'long-running-execution'
def register_module_line(module_name, start_end, line, wrapper=0):
global _MODULES_LINE_MAPPING
default_module_info = {'start': 0, 'start_wrapper': 0, 'end': float('inf'), 'end_wrapper': float('inf')}
try:
if start_end not in ('start', 'end'):
raise ValueError('Invalid start_end argument. Acceptable values are: start, end.')
if not isinstance(line, int) or line < 0:
raise ValueError('Invalid line argument. Expected non-negative integer, '
'got {}({})'.format(type(line), line))
_MODULES_LINE_MAPPING.setdefault(module_name, default_module_info).update(
{start_end: line, '{}_wrapper'.format(start_end): line + wrapper}
)
except Exception as exc:
demisto.info(
'failed to register module line. '
'module: "{}" start_end: "{}" line: "{}".\nError: {}'.format(module_name, start_end, line, exc))
def _find_relevant_module(line):
global _MODULES_LINE_MAPPING
relevant_module = ''
for module, info in _MODULES_LINE_MAPPING.items():
if info['start'] <= line <= info['end']:
if not relevant_module:
relevant_module = module
elif info['start'] > _MODULES_LINE_MAPPING[relevant_module]['start']:
relevant_module = module
return relevant_module
def fix_traceback_line_numbers(trace_str):
def is_adjusted_block(start, end, adjusted_lines):
return any(
block_start < start < end < block_end
for block_start, block_end in adjusted_lines.items()
)
for number in re.findall(r'line (\d+)', trace_str):
line_num = int(number)
module = _find_relevant_module(line_num)
if module:
module_start_line = _MODULES_LINE_MAPPING.get(module, {'start': 0})['start']
actual_number = line_num - module_start_line
# in case of ApiModule injections, adjust the line numbers of the code after the injection
# should avoid adjust ApiModule twice in case of ApiModuleA -> import ApiModuleB
adjusted_lines = {} # type: ignore[var-annotated]
modules_info = list(_MODULES_LINE_MAPPING.values())
modules_info.sort(key=lambda obj: int(obj.get('start_wrapper', obj['start'])))
for module_info in modules_info:
block_start = module_info.get('start_wrapper', module_info['start'])
block_end = module_info.get('end_wrapper', module_info['end'])
if block_start > module_start_line and block_end < line_num \
and not is_adjusted_block(block_start, block_end, adjusted_lines):
actual_number -= block_end - block_start
adjusted_lines[block_start] = block_end
# a traceback line is of the form: File "<string>", line 8853, in func5
trace_str = trace_str.replace(
'File "<string>", line {},'.format(number),
'File "<{}>", line {},'.format(module, actual_number)
)
return trace_str
OS_LINUX = False
OS_MAC = False
OS_WINDOWS = False
if sys.platform.startswith('linux'):
OS_LINUX = True
elif sys.platform.startswith('darwin'):
OS_MAC = True
elif sys.platform.startswith('win32'):
OS_WINDOWS = True
class WarningsHandler(object):
# Wrapper to handle warnings. We use a class to cleanup after execution
@staticmethod
def handle_warning(message, category, filename, lineno, file=None, line=None):
try:
msg = warnings.formatwarning(message, category, filename, lineno, line)
demisto.info("python warning: " + msg)
except Exception:
# ignore the warning if it can't be handled for some reason
pass
def __init__(self):
self.org_handler = warnings.showwarning
warnings.showwarning = WarningsHandler.handle_warning
def __del__(self):
warnings.showwarning = self.org_handler
_warnings_handler = WarningsHandler()
# ignore warnings from logging as a result of not being setup
logging.raiseExceptions = False
# imports something that can be missed from docker image
try:
import requests
from requests.adapters import HTTPAdapter
from urllib3.util import Retry
from typing import Optional, Dict, List, Any, Union, Set, cast
from urllib3 import disable_warnings
disable_warnings()
import dateparser # type: ignore
from datetime import timezone # type: ignore
except Exception:
if sys.version_info[0] < 3:
# in python 2 an exception in the imports might still be raised even though it is caught.
# for more info see https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
sys.exc_clear()
CONTENT_RELEASE_VERSION = '0.0.0'
CONTENT_BRANCH_NAME = 'master'
IS_PY3 = sys.version_info[0] == 3
PY_VER_MINOR = sys.version_info[1]
STIX_PREFIX = "STIX "
# pylint: disable=undefined-variable
ZERO = timedelta(0)
HOUR = timedelta(hours=1)
# The max number of profiling related rows to print to the log on memory dump
PROFILING_DUMP_ROWS_LIMIT = 20
if IS_PY3:
STRING_TYPES = (str, bytes) # type: ignore
STRING_OBJ_TYPES = (str,)
import concurrent.futures
else:
STRING_TYPES = (str, unicode) # type: ignore # noqa: F821
STRING_OBJ_TYPES = STRING_TYPES # type: ignore
# pylint: enable=undefined-variable
# DEPRECATED - use EntryType enum instead
entryTypes = {
'note': 1,
'downloadAgent': 2,
'file': 3,
'error': 4,
'pinned': 5,
'userManagement': 6,
'image': 7,
'playgroundError': 8,
'entryInfoFile': 9,
'warning': 11,
'map': 15,
'debug': 16,
'widget': 17
}
ENDPOINT_STATUS_OPTIONS = [
'Online',
'Offline',
'Unknown'
]
ENDPOINT_ISISOLATED_OPTIONS = [
'Yes',
'No',
'Pending isolation',
'Pending unisolation'
]
class EntryType(object):
"""
Enum: contains all the entry types (e.g. NOTE, ERROR, WARNING, FILE, etc.)
:return: None
:rtype: ``None``
"""
NOTE = 1
DOWNLOAD_AGENT = 2
FILE = 3
ERROR = 4
PINNED = 5
USER_MANAGEMENT = 6
IMAGE = 7
PLAYGROUND_ERROR = 8
ENTRY_INFO_FILE = 9
VIDEO_FILE = 10
WARNING = 11
STATIC_VIDEO_FILE = 12
MAP_ENTRY_TYPE = 15
DEBUG_MODE_FILE = 16
WIDGET = 17
EXECUTION_METRICS = 19
class IncidentStatus(object):
"""
Enum: contains all the incidents status types (e.g. pending, active, done, archive)
:return: None
:rtype: ``None``
"""
PENDING = 0
ACTIVE = 1
DONE = 2
ARCHIVE = 3
class IncidentSeverity(object):
"""
Enum: contains all the incident severity types
:return: None
:rtype: ``None``
"""
UNKNOWN = 0
INFO = 0.5
LOW = 1
MEDIUM = 2
HIGH = 3
CRITICAL = 4
# DEPRECATED - use EntryFormat enum instead
formats = {
'html': 'html',
'table': 'table',
'json': 'json',
'text': 'text',
'dbotResponse': 'dbotCommandResponse',
'markdown': 'markdown'
}
class EntryFormat(object):
"""
Enum: contains all the entry formats (e.g. HTML, TABLE, JSON, etc.)
"""
HTML = 'html'
TABLE = 'table'
JSON = 'json'
TEXT = 'text'
DBOT_RESPONSE = 'dbotCommandResponse'
MARKDOWN = 'markdown'
@classmethod
def is_valid_type(cls, _type):
# type: (str) -> bool
return _type in (
EntryFormat.HTML,
EntryFormat.TABLE,
EntryFormat.JSON,
EntryFormat.TEXT,
EntryFormat.MARKDOWN,
EntryFormat.DBOT_RESPONSE
)
class FileAttachmentType(object):
"""
Enum: contains the file attachment types,
Used to add metadata to the description of the attachment
whether the file content is expected to be inline or attached as a file
:return:: The file attachment type
:rtype: ``str``
"""
ATTACHED = "attached_file"
class QuickActionPreview(object):
"""
A container class for storing quick action data previews.
This class is intended to be populated by commands like `!get-remote-data-preview`
and placed directly into the root context under `QuickActionPreview`.
:return: None
:rtype: ``None``.
"""
def __init__(self, id=None, title=None, description=None, status=None,
assignee=None, creation_date=None, severity=None):
"""
A container class for storing quick action data previews.
This class is intended to be populated by commands like `!get-remote-data-preview`
and placed directly into the root context under `QuickActionPreview`.
Attributes:
id (Optional[str]): The ID of the ticket.
title (Optional[str]): The title or summary of the ticket or action.
description (Optional[str]): A brief description or details about the action.
status (Optional[str]): Current status (e.g., Open, In Progress, Closed).
assignee (Optional[str]): The user or entity assigned to the action.
creation_date (Optional[str]): The date and time when the item was created.
severity (Optional[str]): Indicates the priority or severity level.
:return: None
:rtype: ``None``
"""
self.id = id
self.title = title
self.description = description
self.status = status
self.assignee = assignee
self.creation_date = creation_date
self.severity = severity
missing_fields = [field_name for field_name, value in self.__dict__.items() if value is None]
if missing_fields:
demisto.debug("Missing fields: {}".format(', '.join(missing_fields)))
def to_context(self):
"""
Converts the instance to a dictionary.
:return: Dictionary representation of the QuickActionPreview instance.
:rtype: dict
"""
return self.__dict__
class MirrorObject(object):
"""
A container class for storing ticket metadata used in mirroring integrations.
This class is intended to be populated by commands like `!jira-create-issue`
and placed directly into the root context under `MirrorObject`.
:return: None
:rtype: ``None``.
"""
def __init__(self, object_url=None, object_id=None, object_name=None):
"""
A container class for storing ticket metadata used in mirroring integrations.
This class is intended to be populated by commands like `!jira-create-issue`
and placed directly into the root context under `MirrorObject`.
Attributes:
object_url (Optional[str]): Direct URL to the created ticket for preview/use.
object_id (Optional[str]): Unique identifier of the created ticket.
:return: None
:rtype: ``None``.
"""
self.object_url = object_url
self.object_id = object_id
self.object_name = object_name
missing_fields_list = []
if self.object_url is None:
missing_fields_list.append('object_url')
if self.object_id is None:
missing_fields_list.append('object_id')
if self.object_name is None:
missing_fields_list.append('object_name')
if missing_fields_list:
debug_message = "MirrorObject: Initialized with missing mandatory fields: {}"
demisto.debug(debug_message.format(', '.join(missing_fields_list)))
def to_context(self):
"""
Converts the instance to a dictionary.
:return: Dictionary representation of the MirrorObject instance.
:rtype: dict
"""
return self.__dict__
brands = {
'xfe': 'xfe',
'vt': 'virustotal',
'wf': 'WildFire',
'cy': 'cylance',
'cs': 'crowdstrike-intel'
}
providers = {
'xfe': 'IBM X-Force Exchange',
'vt': 'VirusTotal',
'wf': 'WildFire',
'cy': 'Cylance',
'cs': 'CrowdStrike'
}
thresholds = {
'xfeScore': 4,
'vtPositives': 10,
'vtPositiveUrlsForIP': 30
}
class DBotScoreType(object):
"""
Enum: contains all the indicator types
DBotScoreType.IP
DBotScoreType.FILE
DBotScoreType.DOMAIN
DBotScoreType.URL
DBotScoreType.CVE
DBotScoreType.ACCOUNT
DBotScoreType.CRYPTOCURRENCY
DBotScoreType.EMAIL
DBotScoreType.ATTACKPATTERN
DBotScoreType.CUSTOM
:return: None
:rtype: ``None``
"""
IP = 'ip'
FILE = 'file'
DOMAIN = 'domain'
URL = 'url'
CVE = 'cve'
ACCOUNT = 'account'
CIDR = 'cidr',
DOMAINGLOB = 'domainglob'
CERTIFICATE = 'certificate'
CRYPTOCURRENCY = 'cryptocurrency'
EMAIL = 'email'
ATTACKPATTERN = 'attackpattern'
CUSTOM = 'custom'
def __init__(self):
# required to create __init__ for create_server_docs.py purpose
pass
@classmethod
def is_valid_type(cls, _type):
# type: (str) -> bool
return _type in (
DBotScoreType.IP,
DBotScoreType.FILE,
DBotScoreType.DOMAIN,
DBotScoreType.URL,
DBotScoreType.CVE,
DBotScoreType.ACCOUNT,
DBotScoreType.CIDR,
DBotScoreType.DOMAINGLOB,
DBotScoreType.CERTIFICATE,
DBotScoreType.CRYPTOCURRENCY,
DBotScoreType.EMAIL,
DBotScoreType.ATTACKPATTERN,
DBotScoreType.CUSTOM,
)
class DBotScoreReliability(object):
"""
Enum: Source reliability levels
Values are case sensitive
:return: None
:rtype: ``None``
"""
A_PLUS_PLUS = 'A++ - Reputation script'
A_PLUS = 'A+ - 3rd party enrichment'
A = 'A - Completely reliable'
B = 'B - Usually reliable'
C = 'C - Fairly reliable'
D = 'D - Not usually reliable'
E = 'E - Unreliable'
F = 'F - Reliability cannot be judged'
def __init__(self):
# required to create __init__ for create_server_docs.py purpose
pass
@staticmethod
def is_valid_type(_type):
# type: (str) -> bool
return _type in (
DBotScoreReliability.A_PLUS_PLUS,
DBotScoreReliability.A_PLUS,
DBotScoreReliability.A,
DBotScoreReliability.B,
DBotScoreReliability.C,
DBotScoreReliability.D,
DBotScoreReliability.E,
DBotScoreReliability.F,
)
@staticmethod
def get_dbot_score_reliability_from_str(reliability_str): # pragma: no cover
if reliability_str == DBotScoreReliability.A_PLUS_PLUS:
return DBotScoreReliability.A_PLUS_PLUS
if reliability_str == DBotScoreReliability.A_PLUS:
return DBotScoreReliability.A_PLUS
elif reliability_str == DBotScoreReliability.A:
return DBotScoreReliability.A
elif reliability_str == DBotScoreReliability.B:
return DBotScoreReliability.B
elif reliability_str == DBotScoreReliability.C:
return DBotScoreReliability.C
elif reliability_str == DBotScoreReliability.D:
return DBotScoreReliability.D
elif reliability_str == DBotScoreReliability.E:
return DBotScoreReliability.E
elif reliability_str == DBotScoreReliability.F:
return DBotScoreReliability.F
raise Exception("Please use supported reliability only.")
INDICATOR_TYPE_TO_CONTEXT_KEY = {
'ip': 'Address',
'email': 'Address',
'url': 'Data',
'domain': 'Name',
'cve': 'ID',
'md5': 'file',
'sha1': 'file',
'sha256': 'file',
'crc32': 'file',
'sha512': 'file',
'ctph': 'file',
'ssdeep': 'file'
}
class ErrorTypes(object):
"""
Enum: contains all the available error types
:return: None
:rtype: ``None``
"""
SUCCESS = 'Successful'
QUOTA_ERROR = 'QuotaError'
GENERAL_ERROR = 'GeneralError'
AUTH_ERROR = 'AuthError'
SERVICE_ERROR = 'ServiceError'
CONNECTION_ERROR = 'ConnectionError'
PROXY_ERROR = 'ProxyError'
SSL_ERROR = 'SSLError'
TIMEOUT_ERROR = 'TimeoutError'
RETRY_ERROR = "RetryError"
class CortexErrorCode(object):
"""Error codes for content standardized errors.
Provides a unified taxonomy covering argument validation, resource lookup,
API/network issues, data parsing, permissions, and execution failures.
Used by :class:`CortexError` and its subclasses so that LLM agents can
programmatically distinguish error categories and decide whether to retry,
fix an argument, or escalate.
:return: None
:rtype: ``None``
"""
# -- Argument Errors ----------------------------------------------
MISSING_ARGUMENT = "MISSING_ARGUMENT"
INVALID_ARGUMENT = "INVALID_ARGUMENT"
CONFLICTING_ARGUMENTS = "CONFLICTING_ARGUMENTS"
# -- Resource Errors ----------------------------------------------
RESOURCE_NOT_FOUND = "RESOURCE_NOT_FOUND"
# -- API / Network Errors -----------------------------------------
API_ERROR = "API_ERROR"
AUTH_ERROR = "AUTH_ERROR"
QUOTA_ERROR = "QUOTA_ERROR"
SERVICE_ERROR = "SERVICE_ERROR"
CONNECTION_ERROR = "CONNECTION_ERROR"
PROXY_ERROR = "PROXY_ERROR"
SSL_ERROR = "SSL_ERROR"
TIMEOUT_ERROR = "TIMEOUT_ERROR"
# -- Data Errors (failures parsing data returned by the API/service) --
API_RESPONSE_PARSE_ERROR = "API_RESPONSE_PARSE_ERROR"
# -- Permission Errors --------------------------------------------
PERMISSION_ERROR = "PERMISSION_ERROR"
# -- Execution Errors ---------------------------------------------
EXECUTION_ERROR = "EXECUTION_ERROR"
UNSUPPORTED_COMMAND = "UNSUPPORTED_COMMAND"
INTERNAL_ERROR = "INTERNAL_ERROR"
# Key used inside the ``ExtendedPayload`` dict of an error entry to carry
# the content error type. Defined as a module-level constant so the name
# can be changed in one place.
EXTENDED_PAYLOAD_ERROR_CODE_KEY = 'error_code'
# Value returned by ``demisto.caller()`` when the script/command is executed
# from an Agentix (LLM agent) flow. Used to decide whether ``return_error``
# should surface the automatic, machine-friendly error message instead of a
# human-authored one.
AGENTIX_CALLER = 'agentix'
def is_caller_agentix():
"""Check whether the current execution was triggered by an Agentix agent.
Uses ``demisto.caller()`` (when available) and compares it to
:data:`AGENTIX_CALLER`. Safe to call on older servers that do not expose
the ``caller`` method - in that case it returns ``False``.
:return: ``True`` if the caller is Agentix, ``False`` otherwise.
:rtype: ``bool``
"""
try:
if not hasattr(demisto, 'caller'):
return False
return demisto.caller() == AGENTIX_CALLER
except Exception as exc:
demisto.debug('is_caller_agentix failed to determine caller: {}'.format(exc))
return False
class FeedIndicatorType(object):
"""Type of Indicator (Reputations), used in TIM integrations"""
Account = "Account"
CVE = "CVE"
Domain = "Domain"
DomainGlob = "DomainGlob"
Email = "Email"
File = "File"
FQDN = "Domain"
Host = "Host"
IP = "IP"
CIDR = "CIDR"
IPv6 = "IPv6"
IPv6CIDR = "IPv6CIDR"
Registry = "Registry Key"
SSDeep = "ssdeep"
URL = "URL"
AS = "ASN"
MUTEX = "Mutex"
Malware = "Malware"
Identity = "Identity"
Location = "Location"
Software = "Software"
X509 = "X509 Certificate"
@staticmethod
def is_valid_type(_type):
return _type in (
FeedIndicatorType.Account,
FeedIndicatorType.CVE,
FeedIndicatorType.Domain,
FeedIndicatorType.DomainGlob,
FeedIndicatorType.Email,
FeedIndicatorType.File,
FeedIndicatorType.Host,
FeedIndicatorType.IP,
FeedIndicatorType.CIDR,
FeedIndicatorType.IPv6,
FeedIndicatorType.IPv6CIDR,
FeedIndicatorType.Registry,
FeedIndicatorType.SSDeep,
FeedIndicatorType.URL,
FeedIndicatorType.AS,
FeedIndicatorType.MUTEX,
FeedIndicatorType.Malware,
FeedIndicatorType.Identity,
FeedIndicatorType.Location,
FeedIndicatorType.Software,
FeedIndicatorType.X509,
)
@staticmethod
def list_all_supported_indicators():
indicator_types = []
for key, val in vars(FeedIndicatorType).items():
if not key.startswith('__') and type(val) == str:
indicator_types.append(val)
return indicator_types
@staticmethod
def ip_to_indicator_type(ip):
"""Returns the indicator type of the input IP.
:type ip: ``str``
:param ip: IP address to get it's indicator type.
:return:: Indicator type from FeedIndicatorType, or None if invalid IP address.
:rtype: ``str``
"""
if re.match(ipv4cidrRegex, ip):
return FeedIndicatorType.CIDR
elif re.match(ipv4Regex, ip):
return FeedIndicatorType.IP
elif re.match(ipv6cidrRegex, ip):
return FeedIndicatorType.IPv6CIDR
elif re.match(ipv6Regex, ip):
return FeedIndicatorType.IPv6
else:
return None
@staticmethod
def indicator_type_by_server_version(indicator_type):
"""Returns the indicator type of the input by the server version.
If the server version is 6.2 and greater, remove the STIX prefix of the type
:type indicator_type: ``str``
:param indicator_type: Type of an indicator.
:return:: Indicator type .
:rtype: ``str``
"""
if indicator_type.startswith(STIX_PREFIX):
return indicator_type[len(STIX_PREFIX):]
return indicator_type
# -------------------------------- Threat Intel Objects ----------------------------------- #
class ThreatIntel:
"""
XSOAR Threat Intel Objects
:return: None
:rtype: ``None``
"""
class ObjectsNames(object):
"""
Enum: Threat Intel Objects names.
:return: None
:rtype: ``None``
"""
CAMPAIGN = 'Campaign'
ATTACK_PATTERN = 'Attack Pattern'
REPORT = 'Report'
MALWARE = 'Malware'
COURSE_OF_ACTION = 'Course of Action'
INTRUSION_SET = 'Intrusion Set'
TOOL = 'Tool'
THREAT_ACTOR = 'Threat Actor'
INFRASTRUCTURE = 'Infrastructure'
TACTIC = 'Tactic'
class ObjectsScore(object):
"""
Enum: Threat Intel Objects Score.
:return: None
:rtype: ``None``
"""
CAMPAIGN = 3
ATTACK_PATTERN = 2
REPORT = 3
MALWARE = 3
COURSE_OF_ACTION = 0
INTRUSION_SET = 3
TOOL = 2
THREAT_ACTOR = 3
INFRASTRUCTURE = 2
TACTIC = 0
class KillChainPhases(object):
"""
Enum: Kill Chain Phases names.
:return: None
:rtype: ``None``
"""
BUILD_CAPABILITIES = "Build Capabilities"
PRIVILEGE_ESCALATION = "Privilege Escalation"
ADVERSARY_OPSEC = "Adversary Opsec"
CREDENTIAL_ACCESS = "Credential Access"
EXFILTRATION = "Exfiltration"
LATERAL_MOVEMENT = "Lateral Movement"
DEFENSE_EVASION = "Defense Evasion"
PERSISTENCE = "Persistence"
COLLECTION = "Collection"
IMPACT = "Impact"
INITIAL_ACCESS = "Initial Access"
DISCOVERY = "Discovery"
EXECUTION = "Execution"
INSTALLATION = "Installation"
DELIVERY = "Delivery"
WEAPONIZATION = "Weaponization"
ACT_ON_OBJECTIVES = "Actions on Objectives"
COMMAND_AND_CONTROL = "Command \u0026 Control"
def is_debug_mode():
"""Return if this script/command was passed debug-mode=true option
:return: true if debug-mode is enabled
:rtype: ``bool``
"""
# use `hasattr(demisto, 'is_debug')` to ensure compatibility with server version <= 4.5
return hasattr(demisto, 'is_debug') and demisto.is_debug
def get_schedule_metadata(context):
"""
Get the entry schedule metadata if available
:type context: ``dict``
:param context: Context in which the command was executed.
:return: Dict with metadata of scheduled entry
:rtype: ``dict``
"""
schedule_metadata = {}
parent_entry = context.get('ParentEntry', {})
if parent_entry:
schedule_metadata = assign_params(
is_polling=True if parent_entry.get('polling') else False,
polling_command=parent_entry.get('pollingCommand'),
polling_args=parent_entry.get('pollingArgs'),
times_ran=int(parent_entry.get('timesRan', 0)) + 1,
start_date=parent_entry.get('startDate'),
end_date=parent_entry.get('endingDate')
)
return schedule_metadata
def detect_file_indicator_type(indicator_value):
"""
Detect the type of the file indicator.
:type indicator_value: ``str``
:param indicator_value: The indicator whose type we want to check. (required)
:return: The type of the indicator.
:rtype: ``str``
"""
if ":" in indicator_value:
return 'ssdeep'
if re.match(sha256Regex, indicator_value):
return 'sha256'
if re.match(md5Regex, indicator_value):
return 'md5'
if re.match(sha1Regex, indicator_value):
return 'sha1'
if re.match(sha512Regex, indicator_value):
return 'sha512'
return None
def auto_detect_indicator_type(indicator_value):
"""
Infer the type of the indicator.
:type indicator_value: ``str``
:param indicator_value: The indicator whose type we want to check. (required)
:return: The type of the indicator.
:rtype: ``str``
"""
try:
import tldextract
except Exception:
raise Exception("Missing tldextract module, In order to use the auto detect function please use a docker"
" image with it installed such as: demisto/jmespath")
indicator_value = indicator_value.replace('[.]', '.').replace('[@]', '@') # Refang indicator prior to checking
if re.match(ipv4cidrRegex, indicator_value):
return FeedIndicatorType.CIDR
if re.match(ipv6cidrRegex, indicator_value):
return FeedIndicatorType.IPv6CIDR
if re.match(ipv4Regex, indicator_value):
return FeedIndicatorType.IP
if re.match(ipv6Regex, indicator_value):
return FeedIndicatorType.IPv6
if re.match(cveRegex, indicator_value):
return FeedIndicatorType.CVE
if re.match(md5Regex, indicator_value):
return FeedIndicatorType.File
if re.match(sha1Regex, indicator_value):
return FeedIndicatorType.File
if re.match(sha256Regex, indicator_value):
return FeedIndicatorType.File
if re.match(sha512Regex, indicator_value):
return FeedIndicatorType.File
if re.match(emailRegex, indicator_value):
return FeedIndicatorType.Email
if re.match(urlRegex, indicator_value):
return FeedIndicatorType.URL
try:
tldextract_version = tldextract.__version__
if LooseVersion(tldextract_version) < '3.0.0':
no_cache_extract = tldextract.TLDExtract(cache_file=False, suffix_list_urls=None)
else:
no_cache_extract = tldextract.TLDExtract(cache_dir=False, suffix_list_urls=None)
if no_cache_extract(indicator_value).suffix:
if '*' in indicator_value:
return FeedIndicatorType.DomainGlob
return FeedIndicatorType.Domain
except Exception:
demisto.debug('tldextract failed to detect indicator type. indicator value: {}'.format(indicator_value))
demisto.debug('Failed to detect indicator type. Indicator value: {}'.format(indicator_value))
return None
def add_http_prefix_if_missing(address=''):
"""
This function adds `http://` prefix to the proxy address in case it is missing.
:type address: ``string``
:param address: Proxy address.
:return: proxy address after the 'http://' prefix was added, if needed.
:rtype: ``string``
"""
PROXY_PREFIXES = ['http://', 'https://', 'socks5://', 'socks5h://', 'socks4://', 'socks4a://']
if not address:
return ''
for prefix in PROXY_PREFIXES:
if address.startswith(prefix):
return address
return 'http://' + address
def handle_proxy_for_long_running(proxy_param_name='proxy', checkbox_default_value=False, handle_insecure=True,
insecure_param_name=None):
"""
Handle logic for long running integration routing traffic through the system proxy.
Should usually be called at the beginning of the integration, depending on proxy checkbox state.
Long running integrations on hosted tenants XSOAR8 and XSIAM has a dedicated env. var.: CRTX_HTTP_PROXY.
Fallback call to handle_proxy in cases long running integration on engine or XSOAR6
:type proxy_param_name: ``string``
:param proxy_param_name: name of the "use system proxy" integration parameter
:type checkbox_default_value: ``bool``
:param checkbox_default_value: Default value of the proxy param checkbox
:type handle_insecure: ``bool``
:param handle_insecure: Whether to check the insecure param and unset env variables
:type insecure_param_name: ``string``
:param insecure_param_name: Name of insecure param. If None will search insecure and unsecure
:return: proxies dict for the 'proxies' parameter of 'requests' functions and use_ssl boolean
:rtype: ``Tuple[dict, boolean]``
"""
proxies = {}
crtx_http_proxy = os.environ.get('CRTX_HTTP_PROXY', None)
if crtx_http_proxy:
demisto.error('Setting proxies according to CRTX_HTTP_PROXY: {}'.format(crtx_http_proxy))
proxies = {
'http': crtx_http_proxy,
'https': crtx_http_proxy
}
handle_insecure = True
return proxies, handle_insecure
return handle_proxy(proxy_param_name, checkbox_default_value, handle_insecure, insecure_param_name), handle_insecure
def handle_proxy(proxy_param_name='proxy', checkbox_default_value=False, handle_insecure=True,
insecure_param_name=None):
"""
Handle logic for routing traffic through the system proxy.
Should usually be called at the beginning of the integration, depending on proxy checkbox state.
Additionally will unset env variables REQUESTS_CA_BUNDLE and CURL_CA_BUNDLE if handle_insecure is speficied (default).
This is needed as when these variables are set and a requests.Session object is used, requests will ignore the
Sesssion.verify setting. See: https://github.com/psf/requests/blob/master/requests/sessions.py#L703
:type proxy_param_name: ``string``
:param proxy_param_name: name of the "use system proxy" integration parameter
:type checkbox_default_value: ``bool``
:param checkbox_default_value: Default value of the proxy param checkbox
:type handle_insecure: ``bool``
:param handle_insecure: Whether to check the insecure param and unset env variables
:type insecure_param_name: ``string``
:param insecure_param_name: Name of insecure param. If None will search insecure and unsecure
:return: proxies dict for the 'proxies' parameter of 'requests' functions
:rtype: ``dict``
"""
proxies = {} # type: dict
if demisto.params().get(proxy_param_name, checkbox_default_value):
ensure_proxy_has_http_prefix()
proxies = {
'http': os.environ.get('HTTP_PROXY') or os.environ.get('http_proxy', ''),
'https': os.environ.get('HTTPS_PROXY') or os.environ.get('https_proxy', '')
}
else:
skip_proxy()
if handle_insecure:
if insecure_param_name is None:
param_names = ('insecure', 'unsecure')
else:
param_names = (insecure_param_name,) # type: ignore[assignment]
for p in param_names:
if demisto.params().get(p, False):
skip_cert_verification()
return proxies
def skip_proxy():
"""
The function deletes the proxy environment vars in order to http requests to skip routing through proxy
:return: None
:rtype: ``None``
"""
for k in ('HTTP_PROXY', 'HTTPS_PROXY', 'http_proxy', 'https_proxy'):
if k in os.environ:
del os.environ[k]
def ensure_proxy_has_http_prefix():
"""
The function checks if proxy environment vars are missing http/https prefixes, and adds http if so.
:return: None
:rtype: ``None``
"""
for k in ('HTTP_PROXY', 'HTTPS_PROXY', 'http_proxy', 'https_proxy'):
if k in os.environ:
proxy_env_var = os.getenv(k)
if proxy_env_var:
os.environ[k] = add_http_prefix_if_missing(os.environ[k])
def skip_cert_verification():
"""
The function deletes the self signed certificate env vars in order to http requests to skip certificate validation.
:return: None
:rtype: ``None``
"""
for k in ('REQUESTS_CA_BUNDLE', 'CURL_CA_BUNDLE'):
if k in os.environ:
del os.environ[k]
def urljoin(url, suffix=""):
"""
Will join url and its suffix
Example:
"https://google.com/", "/" => "https://google.com/"
"https://google.com", "/" => "https://google.com/"
"https://google.com", "api" => "https://google.com/api"
"https://google.com", "/api" => "https://google.com/api"
"https://google.com/", "api" => "https://google.com/api"
"https://google.com/", "/api" => "https://google.com/api"
:type url: ``string``
:param url: URL string (required)
:type suffix: ``string``
:param suffix: the second part of the url
:return: Full joined url
:rtype: ``string``
"""
if url[-1:] != "/":
url = url + "/"
if suffix.startswith("/"):
suffix = suffix[1:]
return url + suffix
return url + suffix
def positiveUrl(entry):
"""
Checks if the given entry from a URL reputation query is positive (known bad) (deprecated)
:type entry: ``dict``
:param entry: URL entry (required)
:return: True if bad, false otherwise
:rtype: ``bool``
"""
if entry['Type'] != entryTypes['error'] and entry['ContentsFormat'] == formats['json']:
if entry['Brand'] == brands['xfe']:
return demisto.get(entry, 'Contents.url.result.score') > thresholds['xfeScore']
if entry['Brand'] == brands['vt']:
return demisto.get(entry, 'Contents.positives') > thresholds['vtPositives']
if entry['Brand'] == brands['cs'] and demisto.get(entry, 'Contents'):
c = demisto.get(entry, 'Contents')[0]
return demisto.get(c, 'indicator') and demisto.get(c, 'malicious_confidence') in ['high', 'medium']
return False
def positiveFile(entry): # pragma: no cover
"""
Checks if the given entry from a file reputation query is positive (known bad) (deprecated)
:type entry: ``dict``
:param entry: File entry (required)
:return: True if bad, false otherwise
:rtype: ``bool``
"""
if entry['Type'] != entryTypes['error'] and entry['ContentsFormat'] == formats['json']:
if entry['Brand'] == brands['xfe'] and (demisto.get(entry, 'Contents.malware.family')
or demisto.gets(entry, 'Contents.malware.origins.external.family')):
return True
if entry['Brand'] == brands['vt']:
return demisto.get(entry, 'Contents.positives') > thresholds['vtPositives']
if entry['Brand'] == brands['wf']:
return demisto.get(entry, 'Contents.wildfire.file_info.malware') == 'yes'
if entry['Brand'] == brands['cy'] and demisto.get(entry, 'Contents'):
contents = demisto.get(entry, 'Contents')
k = contents.keys()
if k and len(k) > 0:
v = contents[k[0]]
if v and demisto.get(v, 'generalscore'):
return v['generalscore'] < -0.5
if entry['Brand'] == brands['cs'] and demisto.get(entry, 'Contents'):
c = demisto.get(entry, 'Contents')[0]
return demisto.get(c, 'indicator') and demisto.get(c, 'malicious_confidence') in ['high', 'medium']
return False
def vtCountPositives(entry):
"""
Counts the number of detected URLs in the entry
:type entry: ``dict``
:param entry: Demisto entry (required)
:return: The number of detected URLs
:rtype: ``int``
"""
positives = 0
if demisto.get(entry, 'Contents.detected_urls'):
for detected in demisto.get(entry, 'Contents.detected_urls'):
if demisto.get(detected, 'positives') > thresholds['vtPositives']:
positives += 1
return positives
def positiveIp(entry): # pragma: no cover
"""
Checks if the given entry from a file reputation query is positive (known bad) (deprecated)
:type entry: ``dict``
:param entry: IP entry (required)
:return: True if bad, false otherwise
:rtype: ``bool``
"""
if entry['Type'] != entryTypes['error'] and entry['ContentsFormat'] == formats['json']:
if entry['Brand'] == brands['xfe']:
return demisto.get(entry, 'Contents.reputation.score') > thresholds['xfeScore']
if entry['Brand'] == brands['vt'] and demisto.get(entry, 'Contents.detected_urls'):
return vtCountPositives(entry) > thresholds['vtPositiveUrlsForIP']
if entry['Brand'] == brands['cs'] and demisto.get(entry, 'Contents'):
c = demisto.get(entry, 'Contents')[0]
return demisto.get(c, 'indicator') and demisto.get(c, 'malicious_confidence') in ['high', 'medium']
return False
def formatEpochDate(t):
"""
Convert a time expressed in seconds since the epoch to a string representing local time
:type t: ``int``
:param t: Time represented in seconds (required)
:return: A string representing local time
:rtype: ``str``
"""
if t:
return time.ctime(t)
return ''
def shortCrowdStrike(entry): # pragma: no cover
"""
Display CrowdStrike Intel results in Markdown (deprecated)
:type entry: ``dict``
:param entry: CrowdStrike result entry (required)
:return: A Demisto entry containing the shortened CrowdStrike info
:rtype: ``dict``
"""
if entry['Type'] != entryTypes['error'] and entry['ContentsFormat'] == formats['json']:
if entry['Brand'] == brands['cs'] and demisto.get(entry, 'Contents'):
c = demisto.get(entry, 'Contents')[0]
csRes = '## CrowdStrike Falcon Intelligence'
csRes += '\n\n### Indicator - ' + demisto.gets(c, 'indicator')
labels = demisto.get(c, 'labels')
if labels:
csRes += '\n### Labels'
csRes += '\nName|Created|Last Valid'
csRes += '\n----|-------|----------'
for label in labels:
csRes += '\n' + demisto.gets(label, 'name') + '|' + \
formatEpochDate(demisto.get(label, 'created_on')) + '|' + \
formatEpochDate(demisto.get(label, 'last_valid_on'))
relations = demisto.get(c, 'relations')
if relations:
csRes += '\n### Relations'
csRes += '\nIndicator|Type|Created|Last Valid'
csRes += '\n---------|----|-------|----------'
for r in relations:
csRes += '\n' + demisto.gets(r, 'indicator') + '|' + demisto.gets(r, 'type') + '|' + \
formatEpochDate(demisto.get(label, 'created_date')) + '|' + \
formatEpochDate(demisto.get(label, 'last_valid_date'))
return {'ContentsFormat': formats['markdown'], 'Type': entryTypes['note'], 'Contents': csRes}
return entry
def shortUrl(entry): # pragma: no cover
"""
Formats a URL reputation entry into a short table (deprecated)
:type entry: ``dict``
:param entry: URL result entry (required)
:return: A Demisto entry containing the shortened URL info
:rtype: ``dict``
"""
if entry['Type'] != entryTypes['error'] and entry['ContentsFormat'] == formats['json']:
c = entry['Contents']
if entry['Brand'] == brands['xfe']:
return {'ContentsFormat': formats['table'], 'Type': entryTypes['note'], 'Contents': {
'Country': c['country'], 'MalwareCount': demisto.get(c, 'malware.count'),
'A': demisto.gets(c, 'resolution.A'), 'AAAA': demisto.gets(c, 'resolution.AAAA'),
'Score': demisto.get(c, 'url.result.score'), 'Categories': demisto.gets(c, 'url.result.cats'),
'URL': demisto.get(c, 'url.result.url'), 'Provider': providers['xfe'],
'ProviderLink': 'https://exchange.xforce.ibmcloud.com/url/' + demisto.get(c, 'url.result.url')}}
if entry['Brand'] == brands['vt']:
return {'ContentsFormat': formats['table'], 'Type': entryTypes['note'], 'Contents': {
'ScanDate': c['scan_date'], 'Positives': c['positives'], 'Total': c['total'],
'URL': c['url'], 'Provider': providers['vt'], 'ProviderLink': c['permalink']}}
if entry['Brand'] == brands['cs'] and demisto.get(entry, 'Contents'):
return shortCrowdStrike(entry)
return {'ContentsFormat': 'text', 'Type': 4, 'Contents': 'Unknown provider for result: ' + entry['Brand']}
def shortFile(entry): # pragma: no cover
"""
Formats a file reputation entry into a short table (deprecated)
:type entry: ``dict``
:param entry: File result entry (required)
:return: A Demisto entry containing the shortened file info
:rtype: ``dict``
"""
if entry['Type'] != entryTypes['error'] and entry['ContentsFormat'] == formats['json']:
c = entry['Contents']
if entry['Brand'] == brands['xfe']:
cm = c['malware']
return {'ContentsFormat': formats['table'], 'Type': entryTypes['note'], 'Contents': {
'Family': cm['family'], 'MIMEType': cm['mimetype'], 'MD5': cm['md5'][2:] if 'md5' in cm else '',
'CnCServers': demisto.get(cm, 'origins.CncServers.count'),
'DownloadServers': demisto.get(cm, 'origins.downloadServers.count'),
'Emails': demisto.get(cm, 'origins.emails.count'),
'ExternalFamily': demisto.gets(cm, 'origins.external.family'),
'ExternalCoverage': demisto.get(cm, 'origins.external.detectionCoverage'),
'Provider': providers['xfe'],
'ProviderLink': 'https://exchange.xforce.ibmcloud.com/malware/' + cm['md5'].replace('0x', '')}}
if entry['Brand'] == brands['vt']:
return {'ContentsFormat': formats['table'], 'Type': entryTypes['note'], 'Contents': {
'Resource': c['resource'], 'ScanDate': c['scan_date'], 'Positives': c['positives'],
'Total': c['total'], 'SHA1': c['sha1'], 'SHA256': c['sha256'], 'Provider': providers['vt'],
'ProviderLink': c['permalink']}}
if entry['Brand'] == brands['wf']:
c = demisto.get(entry, 'Contents.wildfire.file_info')
if c:
return {'Contents': {'Type': c['filetype'], 'Malware': c['malware'], 'MD5': c['md5'],
'SHA256': c['sha256'], 'Size': c['size'], 'Provider': providers['wf']},
'ContentsFormat': formats['table'], 'Type': entryTypes['note']}
if entry['Brand'] == brands['cy'] and demisto.get(entry, 'Contents'):
contents = demisto.get(entry, 'Contents')
k = contents.keys()
if k and len(k) > 0:
v = contents[k[0]]
if v and demisto.get(v, 'generalscore'):
return {'Contents': {'Status': v['status'], 'Code': v['statuscode'], 'Score': v['generalscore'],
'Classifiers': str(v['classifiers']), 'ConfirmCode': v['confirmcode'],
'Error': v['error'], 'Provider': providers['cy']},
'ContentsFormat': formats['table'], 'Type': entryTypes['note']}
if entry['Brand'] == brands['cs'] and demisto.get(entry, 'Contents'):
return shortCrowdStrike(entry)
return {'ContentsFormat': formats['text'], 'Type': entryTypes['error'],
'Contents': 'Unknown provider for result: ' + entry['Brand']}
def shortIp(entry):
"""
Formats an ip reputation entry into a short table (deprecated)
:type entry: ``dict``
:param entry: IP result entry (required)
:return: A Demisto entry containing the shortened IP info
:rtype: ``dict``
"""
if entry['Type'] != entryTypes['error'] and entry['ContentsFormat'] == formats['json']:
c = entry['Contents']
if entry['Brand'] == brands['xfe']:
cr = c['reputation']
return {'ContentsFormat': formats['table'], 'Type': entryTypes['note'], 'Contents': {
'IP': cr['ip'], 'Score': cr['score'], 'Geo': str(cr['geo']), 'Categories': str(cr['cats']),
'Provider': providers['xfe']}}
if entry['Brand'] == brands['vt']:
return {'ContentsFormat': formats['table'], 'Type': entryTypes['note'],
'Contents': {'Positive URLs': vtCountPositives(entry), 'Provider': providers['vt']}}
if entry['Brand'] == brands['cs'] and demisto.get(entry, 'Contents'):
return shortCrowdStrike(entry)
return {'ContentsFormat': formats['text'], 'Type': entryTypes['error'],
'Contents': 'Unknown provider for result: ' + entry['Brand']}
def shortDomain(entry):
"""
Formats a domain reputation entry into a short table (deprecated)
:type entry: ``dict``
:param entry: Domain result entry (required)
:return: A Demisto entry containing the shortened domain info
:rtype: ``dict``
"""
if entry['Type'] != entryTypes['error'] and entry['ContentsFormat'] == formats['json']:
if entry['Brand'] == brands['vt']:
return {'ContentsFormat': formats['table'], 'Type': entryTypes['note'],
'Contents': {'Positive URLs': vtCountPositives(entry), 'Provider': providers['vt']}}
return {'ContentsFormat': formats['text'], 'Type': entryTypes['error'],
'Contents': 'Unknown provider for result: ' + entry['Brand']}
def get_error(execute_command_result):
"""
execute_command_result must contain error entry - check the result first with is_error function
if there is no error entry in the result then it will raise an Exception
:type execute_command_result: ``dict`` or ``list``
:param execute_command_result: result of demisto.executeCommand()
:return: Error message extracted from the demisto.executeCommand() result
:rtype: ``string``
"""
if not is_error(execute_command_result):
raise ValueError("execute_command_result has no error entry. before using get_error use is_error")
if isinstance(execute_command_result, dict):
return execute_command_result['Contents']
error_messages = []
for entry in execute_command_result:
is_error_entry = type(entry) == dict and entry['Type'] == entryTypes['error']
if is_error_entry:
error_messages.append(entry['Contents'])
return '\n'.join(error_messages)
def is_error(execute_command_result):
"""
Check if the given execute_command_result has an error entry
:type execute_command_result: ``dict`` or ``list``
:param execute_command_result: Demisto entry (required) or result of demisto.executeCommand()
:return: True if the execute_command_result has an error entry, false otherwise
:rtype: ``bool``
"""
if execute_command_result is None:
return False
if isinstance(execute_command_result, list):
if len(execute_command_result) > 0:
for entry in execute_command_result:
if type(entry) == dict and entry['Type'] == entryTypes['error']:
return True
return type(execute_command_result) == dict and execute_command_result['Type'] == entryTypes['error']
isError = is_error
def get_error_code(execute_command_result):
"""Extract the standardized error code from an ``executeCommand`` result.
Reads the machine-readable error code that :func:`return_error` attaches to
an error entry's ``ExtendedPayload`` (under
:data:`EXTENDED_PAYLOAD_ERROR_CODE_KEY`). The value is a
:class:`CortexErrorCode` member (e.g. ``"AUTH_ERROR"``,
``"MISSING_ARGUMENT"``), letting a calling script/playbook classify the
failure of a sub-command without parsing the human-readable ``Contents``.
Accepts either a single entry (``dict``) or a list of entries (the typical
shape returned by ``demisto.executeCommand()``); for a list, the first
error entry that carries an error code is returned.
:type execute_command_result: ``dict`` or ``list``
:param execute_command_result: Result of ``demisto.executeCommand()`` - a
single entry dict or a list of entry dicts.
:rtype: ``str`` or ``None``
:return: The :class:`CortexErrorCode` value from ``ExtendedPayload``, or
``None`` if no error code is present.
"""
if isinstance(execute_command_result, dict):
execute_command_result = [execute_command_result]
if isinstance(execute_command_result, list):
for entry in execute_command_result:
if isinstance(entry, dict) and entry.get('Type') == entryTypes['error']:
extended = entry.get('ExtendedPayload') or {}
error_code = extended.get(EXTENDED_PAYLOAD_ERROR_CODE_KEY)
if error_code:
return error_code
return None
def _classify_error_message(message):
"""Classify an error message into a :class:`CortexErrorCode` value.
Uses text heuristics to detect common error patterns - including
server-generated errors (e.g. ``"Unsupported Command"``) that have
no ``ExtendedPayload``.
:type message: ``str``
:param message: The error message to classify.
:rtype: ``str`` or ``None``
:return: A :class:`CortexErrorCode` value, or *None* if unrecognised.
"""
if not message:
return None
msg = message.lower()
if 'unsupported command' in msg:
return CortexErrorCode.UNSUPPORTED_COMMAND
if 'no integration instance' in msg or 'verify you have proper integration' in msg:
return CortexErrorCode.EXECUTION_ERROR
if any(kw in msg for kw in ('unauthorized', 'authentication failed', 'invalid credentials')):
return CortexErrorCode.AUTH_ERROR
if any(kw in msg for kw in ('rate limit', 'quota exceeded', 'too many requests')):
return CortexErrorCode.QUOTA_ERROR
if any(kw in msg for kw in ('timed out', 'timeout', 'request timeout')):
return CortexErrorCode.TIMEOUT_ERROR
if any(kw in msg for kw in ('connection error', 'connection refused', 'unreachable', 'connect timeout')):
return CortexErrorCode.CONNECTION_ERROR
if any(kw in msg for kw in ('permission denied', 'forbidden', 'insufficient permissions')):
return CortexErrorCode.PERMISSION_ERROR
if any(kw in msg for kw in ('invalid argument', 'invalid value', 'invalid parameter', 'is not a valid')):
return CortexErrorCode.INVALID_ARGUMENT
if any(kw in msg for kw in ('missing argument', 'missing required argument', 'required argument')):
return CortexErrorCode.MISSING_ARGUMENT
if 'not found' in msg:
return CortexErrorCode.RESOURCE_NOT_FOUND
if any(kw in msg for kw in ('ssl', 'certificate verify')):
return CortexErrorCode.SSL_ERROR
if 'proxy' in msg:
return CortexErrorCode.PROXY_ERROR
if any(kw in msg for kw in ('failed to parse', 'json decode', 'jsondecodeerror', 'invalid json', 'unable to parse')):
return CortexErrorCode.API_RESPONSE_PARSE_ERROR
return None
def FormatADTimestamp(ts):
"""
Formats an Active Directory timestamp into human readable time representation
:type ts: ``int``
:param ts: The timestamp to be formatted (required)
:return: A string represeting the time
:rtype: ``str``
"""
return (datetime(year=1601, month=1, day=1) + timedelta(seconds=int(ts) / 10 ** 7)).ctime()
def PrettifyCompactedTimestamp(x):
"""
Formats a compacted timestamp string into human readable time representation
:type x: ``str``
:param x: The timestamp to be formatted (required)
:return: A string represeting the time
:rtype: ``str``
"""
return '%s-%s-%sT%s:%s:%s' % (x[:4], x[4:6], x[6:8], x[8:10], x[10:12], x[12:])
def NormalizeRegistryPath(strRegistryPath):
"""
Normalizes a registry path string
:type strRegistryPath: ``str``
:param strRegistryPath: The registry path (required)
:return: The normalized string
:rtype: ``str``
"""
dSub = {
'HKCR': 'HKEY_CLASSES_ROOT',
'HKCU': 'HKEY_CURRENT_USER',
'HKLM': 'HKEY_LOCAL_MACHINE',
'HKU': 'HKEY_USERS',
'HKCC': 'HKEY_CURRENT_CONFIG',
'HKPD': 'HKEY_PERFORMANCE_DATA'
}
for k in dSub:
if strRegistryPath[:len(k)] == k:
return dSub[k] + strRegistryPath[len(k):]
return strRegistryPath
def scoreToReputation(score):
"""
Converts score (in number format) to human readable reputation format
:type score: ``int``
:param score: The score to be formatted (required)
:return: The formatted score
:rtype: ``str``
"""
to_str = {
4: 'Critical',
3: 'Bad',
2: 'Suspicious',
1: 'Good',
0.5: 'Informational',
0: 'Unknown'
}
return to_str.get(score, 'None')
def b64_encode(text):
"""
Base64 encode a string. Wrapper function around base64.b64encode which will accept a string
In py3 will encode the string to binary using utf-8 encoding and return a string result decoded using utf-8
:param text: string to encode
:type text: str
:return: encoded string
:rtype: str
"""
if not text:
return ''
elif isinstance(text, bytes):
to_encode = text
else:
to_encode = text.encode('utf-8', 'ignore')
res = base64.b64encode(to_encode)
if IS_PY3:
res = res.decode('utf-8') # type: ignore
return res
def b64_decode(b64_str):
"""
Decode a str in a base 64 format to a picture.
Replaces the use of base64.b64decode function which doesn't add padding to the supplied str.
:param b64_str: string to decode
:type b64_str: str
:return: decoded binary
:rtype: bytes
"""
b64 = b64_str.encode('ascii')
b64 += b'=' * (-len(b64) % 4) # add padding
return binascii.a2b_base64(b64)
def encode_string_results(text):
"""
Encode string as utf-8, if any unicode character exists.
:param text: string to encode
:type text: str
:return: encoded string
:rtype: str
"""
if not isinstance(text, STRING_OBJ_TYPES):
return text
try:
return str(text)
except UnicodeEncodeError:
return text.encode("utf8", "replace")
def safe_load_json(json_object): # pragma: no cover
"""
Safely loads a JSON object from an argument. Allows the argument to accept either a JSON in string form,
or an entry ID corresponding to a JSON file.
:param json_object: Entry ID or JSON string.
:type json_object: str
:return: Dictionary object from a parsed JSON file or string.
:rtype: dict
"""
safe_json = None
if isinstance(json_object, dict) or isinstance(json_object, list):
return json_object
if (json_object.startswith('{') and json_object.endswith('}')) or (
json_object.startswith('[') and json_object.endswith(']')):
try:
safe_json = json.loads(json_object)
except ValueError as e:
return_error(
'Unable to parse JSON string. Please verify the JSON is valid. - ' + str(e))
else:
try:
path = demisto.getFilePath(json_object)
with open(path['path'], 'rb') as data:
try:
safe_json = json.load(data)
except Exception: # lgtm [py/catch-base-exception]
safe_json = json.loads(data.read())
except Exception as e:
return_error('Unable to parse JSON file. Please verify the JSON is valid or the Entry'
'ID is correct. - ' + str(e))
return safe_json
def datetime_to_string(datetime_obj):
"""
Converts a datetime object into a string. When used with `json.dumps()` for the `default` parameter,
e.g. `json.dumps(response, default=datetime_to_string)` datetime_to_string allows entire JSON objects
to be safely added to context without causing any datetime marshalling errors.
:param datetime_obj: Datetime object.
:type datetime_obj: datetime.datetime
:return: String representation of a datetime object.
:rtype: str
"""
if isinstance(datetime_obj, datetime): # type: ignore
return datetime_obj.__str__()
def remove_empty_elements(d):
"""
Recursively remove empty lists, empty dicts, or None elements from a dictionary.
:param d: Input dictionary.
:type d: dict
:return: Dictionary with all empty lists, and empty dictionaries removed.
:rtype: dict
"""
def empty(x):
return x is None or x == {} or x == []
if not isinstance(d, (dict, list)):
return d
elif isinstance(d, list):
return [v for v in (remove_empty_elements(v) for v in d) if not empty(v)]
else:
return {k: v for k, v in ((k, remove_empty_elements(v)) for k, v in d.items()) if not empty(v)}
class SmartGetDict(dict):
"""A dict that when called with get(key, default) will return the default passed
value, even if there is a value of "None" in the place of the key. Example with built-in dict:
```
>>> d = {}
>>> d['test'] = None
>>> d.get('test', 1)
>>> print(d.get('test', 1))
None
```
Example with SmartGetDict:
```
>>> d = SmartGetDict()
>>> d['test'] = None
>>> d.get('test', 1)
>>> print(d.get('test', 1))
1
```
:return: SmartGetDict
:rtype: ``SmartGetDict``
"""
def get(self, key, default=None):
res = dict.get(self, key)
if res is not None:
return res
return default
if (not os.getenv('COMMON_SERVER_NO_AUTO_PARAMS_REMOVE_NULLS')) and hasattr(demisto, 'params') and demisto.params():
demisto.callingContext['params'] = SmartGetDict(demisto.params())
def aws_table_to_markdown(response, table_header):
"""
Converts a raw response from AWS into a markdown formatted table. This function checks to see if
there is only one nested dict in the top level of the dictionary and will use the nested data.
:param response: Raw response from AWS
:type response: dict
:param table_header: The header string to use for the table.
:type table_header: str
:return: Markdown formatted table as a string.
:rtype: str
"""
if isinstance(response, dict):
if len(response) == 1:
if isinstance(response[list(response.keys())[0]], dict) or isinstance(
response[list(response.keys())[0]], list):
if isinstance(response[list(response.keys())[0]], list):
list_response = response[list(response.keys())[0]]
if not list_response:
human_readable = tableToMarkdown(table_header, list_response)
elif isinstance(list_response[0], str):
human_readable = tableToMarkdown(
table_header, response)
else:
human_readable = tableToMarkdown(
table_header, response[list(response.keys())[0]])
else:
human_readable = tableToMarkdown(
table_header, response[list(response.keys())[0]])
else:
human_readable = tableToMarkdown(table_header, response)
else:
human_readable = tableToMarkdown(table_header, response)
else:
human_readable = tableToMarkdown(table_header, response)
return human_readable
def stringEscape(st):
"""
Escape newline chars in the given string.
:type st: ``str``
:param st: The string to be modified (required).
:return: A modified string.
:rtype: ``str``
"""
return st.replace('\r', '\\r').replace('\n', '\\n').replace('\t', '\\t')
def stringUnEscape(st):
"""
Unescape newline chars in the given string.
:type st: ``str``
:param st: The string to be modified (required).
:return: A modified string.
:rtype: ``str``
"""
return st.replace('\\r', '\r').replace('\\n', '\n').replace('\\t', '\t')
def doubleBackslashes(st):
"""
Double any backslashes in the given string if it contains two backslashes.
:type st: ``str``
:param st: The string to be modified (required).
:return: A modified string with doubled backslashes.
:rtype: ``str``
"""
return st.replace('\\', '\\\\')
class IntegrationLogger(object):
"""
a logger for python integrations:
use `LOG(<message>)` to add a record to the logger (message can be any object with __str__)
use `LOG.print_log(verbose=True/False)` to display all records in War-Room (if verbose) and server log.
use `add_replace_strs` to add sensitive strings that should be replaced before going to the log.
:type message: ``str``
:param message: The message to be logged
:return: No data returned
:rtype: ``None``
"""
def __init__(self, debug_logging=False):
self.messages = [] # type: list
self.write_buf = [] # type: list
self.replace_strs = [] # type: list
self.curl = [] # type: list
self.buffering = True
self.debug_logging = debug_logging
# if for some reason you don't want to auto add credentials.password to replace strings
# set the os env COMMON_SERVER_NO_AUTO_REPLACE_STRS. Either in CommonServerUserPython, or docker env
if (not os.getenv('COMMON_SERVER_NO_AUTO_REPLACE_STRS') and hasattr(demisto, 'getParam')):
# add common params
sensitive_params = ('key', 'private', 'password', 'secret', 'token', 'credentials', 'service_account')
if demisto.params():
self._iter_sensistive_dict_obj(demisto.params(), sensitive_params)
def _iter_sensistive_dict_obj(self, dict_obj, sensitive_params):
for (k, v) in dict_obj.items():
if isinstance(v, dict): # credentials object case. recurse into the object
self._iter_sensistive_dict_obj(v, sensitive_params)
if v.get('identifier') and v.get('password'): # also add basic auth case
basic_auth = '{}:{}'.format(v.get('identifier'), v.get('password'))
self.add_replace_strs(b64_encode(basic_auth))
elif isinstance(v, STRING_OBJ_TYPES):
k_lower = k.lower()
for p in sensitive_params:
if p in k_lower:
self.add_replace_strs(v, b64_encode(v))
def encode(self, message):
try:
res = str(message)
except UnicodeEncodeError as exception:
# could not decode the message
# if message is an Exception, try encode the exception's message
if isinstance(message, Exception) and message.args and isinstance(message.args[0], STRING_OBJ_TYPES):
res = message.args[0].encode('utf-8', 'replace') # type: ignore
elif isinstance(message, STRING_OBJ_TYPES):
# try encode the message itself
res = message.encode('utf-8', 'replace') # type: ignore
else:
res = "Failed encoding message with error: {}".format(exception)
for s in self.replace_strs:
res = res.replace(s, MASK)
return res
def __call__(self, message):
text = self.encode(message)
if self.buffering:
self.messages.append(text)
if self.debug_logging:
demisto.debug(text)
else:
demisto.info(text)
return text
def add_replace_strs(self, *args):
'''
Add strings which will be replaced when logging.
Meant for avoiding passwords and so forth in the log.
'''
to_add = []
for a in args:
if a:
a = self.encode(a)
to_add.append(stringEscape(a))
to_add.append(stringUnEscape(a))
to_add.append(doubleBackslashes(a))
js = json.dumps(a)
if js.startswith('"'):
js = js[1:]
if js.endswith('"'):
js = js[:-1]
to_add.append(js)
if IS_PY3:
from urllib import parse as urllib_parse
to_add.append(urllib_parse.quote_plus(a)) # type: ignore[attr-defined]
else:
to_add.append(urllib.quote_plus(a)) # type: ignore[attr-defined]
self.replace_strs.extend(to_add)
def set_buffering(self, state):
"""
set whether the logger buffers messages or writes staight to the demisto log
:param state: True/False
:type state: boolean
"""
self.buffering = state
def print_log(self, verbose=False):
if self.write_buf:
self.messages.append("".join(self.write_buf))
if self.messages:
text = 'Full Integration Log:\n' + '\n'.join(self.messages)
if verbose:
demisto.log(text) # pylint: disable=E9012
if not self.debug_logging: # we don't print out if in debug_logging as already all message where printed
demisto.info(text)
self.messages = []
def build_curl(self, text):
"""
Parses the HTTP client "send" log messages and generates cURL queries out of them.
:type text: ``str``
:param text: The HTTP client log message.
:return: No data returned
:rtype: ``None``
"""
http_methods = ['GET', 'POST', 'PUT', 'DELETE', 'PATCH']
data = text.split(SEND_PREFIX)[1]
if data and data[0] in {'{', '<'}:
# it is the request url query params/post body - will always come after we already have the url and headers
# `<` is for xml body
self.curl[-1] += "-d '{}".format(data)
elif any(http_method in data for http_method in http_methods):
method = ''
url = ''
headers = []
headers_to_skip = ['Content-Length', 'User-Agent', 'Accept-Encoding', 'Connection']
request_parts = repr(data).split('\\\\r\\\\n') # splitting lines on repr since data is a bytes-string
for line, part in enumerate(request_parts):
if line == 0:
method, url, _ = part[1:].split() # ignoring " at first char
elif line != len(request_parts) - 1: # ignoring the last line which is empty
if part.startswith('Host:'):
_, host = part.split('Host: ')
url = 'https://{}{}'.format(host, url)
else:
if any(header_to_skip in part for header_to_skip in headers_to_skip):
continue
headers.append(part)
curl_headers = ''
for header in headers:
if header:
curl_headers += '-H "{}" '.format(header)
curl = 'curl -X {} {} {}'.format(method, url, curl_headers)
if demisto.params().get('proxy'):
proxy_address = os.environ.get('https_proxy')
if proxy_address:
curl += '--proxy {} '.format(proxy_address)
else:
curl += '--noproxy "*" '
if demisto.params().get('insecure'):
curl += '-k '
self.curl.append(curl)
def write(self, msg):
# same as __call__ but allows IntegrationLogger to act as a File like object.
msg = self.encode(msg)
has_newline = False
if '\n' in msg:
has_newline = True
# if new line is last char we trim it out
if msg[-1] == '\n':
msg = msg[:-1]
self.write_buf.append(msg)
if has_newline:
text = "".join(self.write_buf)
if self.buffering:
self.messages.append(text)
else:
if is_debug_mode():
if text.startswith(('send:', 'header:')):
try:
# ensures the logged data follows a standard convention
text = text.replace("send: b\"", "send: b'")
text = censor_request_logs(text)
except Exception as e: # should fail silently
demisto.debug('Failed censoring request logs - {}'.format(str(e)))
if text.startswith('send:'):
try:
self.build_curl(text)
except Exception as e: # should fail silently
demisto.debug('Failed generating curl - {}'.format(str(e)))
demisto.info(text)
self.write_buf = []
def print_override(self, *args, **kwargs):
# print function that can be used to override print usage of internal modules
# will print to the log if the print target is stdout/stderr
try:
import __builtin__ # type: ignore
except ImportError:
# Python 3
import builtins as __builtin__ # type: ignore
file_ = kwargs.get('file')
if (not file_) or file_ == sys.stdout or file_ == sys.stderr:
kwargs['file'] = self
__builtin__.print(*args, **kwargs)
"""
a logger for python integrations:
use `LOG(<message>)` to add a record to the logger (message can be any object with __str__)
use `LOG.print_log()` to display all records in War-Room and server log.
"""
LOG = IntegrationLogger(debug_logging=is_debug_mode())
def formatAllArgs(args, kwds):
"""
makes a nice string representation of all the arguments
:type args: ``list``
:param args: function arguments (required)
:type kwds: ``dict``
:param kwds: function keyword arguments (required)
:return: string representation of all the arguments
:rtype: ``string``
"""
formattedArgs = ','.join([repr(a) for a in args]) + ',' + str(kwds).replace(':', "=").replace(" ", "")[1:-1]
return formattedArgs
def logger(func):
"""
decorator function to log the function call using LOG
:type func: ``function``
:param func: function to call (required)
:return: returns the func return value.
:rtype: ``any``
"""
def func_wrapper(*args, **kwargs):
LOG('calling {}({})'.format(func.__name__, formatAllArgs(args, kwargs)))
ret_val = func(*args, **kwargs)
if is_debug_mode():
LOG('Return value [{}]: {}'.format(func.__name__, str(ret_val)))
return ret_val
return func_wrapper
def formatCell(data, is_pretty=True, json_transform=None):
"""
Convert a given object to md while decending multiple levels
:type data: ``str`` or ``list`` or ``dict``
:param data: The cell content (required)
:type is_pretty: ``bool``
:param is_pretty: Should cell content be prettified (default is True)
:type json_transform: ``JsonTransformer``
:param json_transform: The Json transform object to transform the data
:return: The formatted cell content as a string
:rtype: ``str``
"""
if json_transform is None:
json_transform = JsonTransformer(flatten=True)
return json_transform.json_to_str(data, is_pretty)
def flattenCell(data, is_pretty=True):
"""
Flattens a markdown table cell content into a single string
:type data: ``str`` or ``list``
:param data: The cell content (required)
:type is_pretty: ``bool``
:param is_pretty: Should cell content be pretified (default is True)
:return: A sting representation of the cell content
:rtype: ``str``
"""
indent = 4 if is_pretty else None
if isinstance(data, STRING_TYPES):
return data
elif isinstance(data, list):
string_list = []
for d in data:
try:
if IS_PY3 and isinstance(d, bytes):
string_list.append(d.decode('utf-8'))
else:
string_list.append(str(d))
except UnicodeEncodeError:
string_list.append(d.encode('utf-8'))
return ',\n'.join(string_list)
else:
return json.dumps(data, indent=indent, ensure_ascii=False, default=str)
def FormatIso8601(t):
"""
Convert a time expressed in seconds to ISO 8601 time format string
:type t: ``int``
:param t: Time expressed in seconds (required)
:return: An ISO 8601 time format string
:rtype: ``str``
"""
return t.strftime("%Y-%m-%dT%H:%M:%S")
def argToList(arg, separator=',', transform=None):
"""
Converts a string representation of args to a python list
:type arg: ``str`` or ``list``
:param arg: Args to be converted (required)
:type separator: ``str``
:param separator: A string separator to separate the strings, the default is a comma.
:type transform: ``callable``
:param transform: A function transformer to transfer the returned list arguments.
:return: A python list of args
:rtype: ``list``
"""
if not arg:
return []
result = []
if isinstance(arg, list):
result = arg
elif isinstance(arg, STRING_TYPES):
is_comma_separated = True
if arg[0] == '[' and arg[-1] == ']':
try:
result = json.loads(arg)
is_comma_separated = False
except Exception:
demisto.debug('Failed to load {} as JSON, trying to split'.format(arg)) # type: ignore[str-bytes-safe]
if is_comma_separated:
result = [s.strip() for s in arg.split(separator)]
else:
result = [arg]
if transform:
return [transform(s) for s in result]
return result
def remove_duplicates_from_list_arg(args, field):
"""
Removes duplicates from a list after calling argToList.
For example: args: {'ids': "1,2,1"}, field='ids'
The return output will be ["1", "2"]
:type args: ``dict``
:param args: Args to be converted (required)
:type field: ``str``
:param field: Field in args to be converted into list without duplicates (required)
:return: A python list of args without duplicates
:rtype: ``list``
"""
convert_to_list = argToList(args.get(field))
return list(set(convert_to_list))
def argToBoolean(value):
"""
Boolean-ish arguments that are passed through demisto.args() could be type bool or type string.
This command removes the guesswork and returns a value of type bool, regardless of the input value's type.
It will also return True for 'yes' and False for 'no'.
:param value: the value to evaluate
:type value: ``string|bool``
:return: a boolean representation of 'value'
:rtype: ``bool``
"""
if isinstance(value, bool):
return value
if isinstance(value, STRING_OBJ_TYPES):
if value.lower() in ('y', 'yes', 't', 'true', 'on', '1'):
return True
elif value.lower() in ('n', 'no', 'f', 'false', 'off', '0'):
return False
else:
raise ValueError('Argument does not contain a valid boolean-like value')
else:
raise ValueError('Argument is neither a string nor a boolean')
def arg_to_bool_or_none(value):
"""
Converts a value to a boolean or None.
:type value: ``Any``
:param value: The value to convert to boolean or None.
:return: Returns None if the input is None, otherwise returns the boolean representation of the value using the argToBoolean function.
:rtype: ``bool`` or ``None``
"""
if value is None:
return None
else:
return argToBoolean(value)
def appendContext(key, data, dedup=False):
"""
Append data to the investigation context.
Usable by scripts not integrations, since it uses setContext
:type key: ``str``
:param key: The context path (required)
:type data: ``any``
:param data: Data to be added to the context (required)
:type dedup: ``bool``
:param dedup: True if de-duplication is required. Default is False.
:return: No data returned
:rtype: ``None``
"""
if data is None:
return
existing = demisto.get(demisto.context(), key)
if existing:
if isinstance(existing, STRING_TYPES):
if isinstance(data, STRING_TYPES):
new_val = data + ',' + existing # type: ignore[operator]
else:
new_val = data + existing # will raise a self explanatory TypeError
elif isinstance(existing, dict):
if isinstance(data, dict):
new_val = [existing, data] # type: ignore[assignment]
elif isinstance(data, list) and all([isinstance(sub_data, dict) for sub_data in data]):
# For cases the context have only one value but we append a few values at once.
new_val = [existing] + data # type: ignore[assignment]
else:
new_val = data + existing # will raise a self explanatory TypeError
elif isinstance(existing, list):
if isinstance(data, list):
existing.extend(data)
else:
existing.append(data)
new_val = existing # type: ignore[assignment]
else:
new_val = [existing, data] # type: ignore[assignment]
if dedup and isinstance(new_val, list):
new_val = list(set(new_val))
demisto.setContext(key, new_val)
else:
demisto.setContext(key, data)
def url_to_clickable_markdown(data, url_keys):
"""
Turn the given urls fields in to clickable url, used for the markdown table.
:type data: ``[Union[str, List[Any], Dict[str, Any]]]``
:param data: a dictionary or a list containing data with some values that are urls
:type url_keys: ``List[str]``
:param url_keys: the keys of the url's wished to turn clickable
:return: markdown format for clickable url
:rtype: ``[Union[str, List[Any], Dict[str, Any]]]``
"""
if isinstance(data, list):
data = [url_to_clickable_markdown(item, url_keys) for item in data]
elif isinstance(data, dict):
data = {key: create_clickable_url(value) if key in url_keys else url_to_clickable_markdown(data[key], url_keys)
for key, value in data.items()}
return data
def create_clickable_url(url, text=None):
"""
Make the given url clickable when in markdown format by concatenating itself, with the proper brackets
:type url: ``Union[List[str], str]``
:param url: the url of interest or a list of urls
:type text: ``Union[List[str], str, None]``
:param text: the text of the url or a list of texts of urls.
:return: Markdown format for clickable url
:rtype: ``Union[List[str], str]``
"""
if not url:
return None
elif isinstance(url, list):
if isinstance(text, list):
assert len(url) == len(text), 'The URL list and the text list must be the same length.'
return ['[{}]({})'.format(text, item) for text, item in zip(text, url)]
return ['[{}]({})'.format(item, item) for item in url]
return '[{}]({})'.format(text or url, url)
class JsonTransformer:
"""
A class to transform a json to
:type flatten: ``bool``
:param flatten: Should we flatten the json using `flattenCell` (for BC)
:type keys: ``Set[str]``
:param keys: Set of keys to keep
:type is_nested: ``bool``
:param is_nested: If look for nested
:type func: ``Callable``
:param func: A function to parse the json
:return: None
:rtype: ``None``
"""
def __init__(self, flatten=False, keys=None, is_nested=False, func=None):
"""
Constructor for JsonTransformer
:type flatten: ``bool``
:param flatten: Should we flatten the json using `flattenCell` (for BC)
:type keys: ``Iterable[str]``
:param keys: an iterable of relevant keys list from the json. Notice we save it as a set in the class
:type is_nested: ``bool``
:param is_nested: Whether to search in nested keys or not
:type func: ``Callable``
:param func: A function to parse the json
"""
if keys is None:
keys = []
self.keys = set(keys)
self.is_nested = is_nested
self.func = func
self.flatten = flatten
def json_to_str(self, json_input, is_pretty=True):
if self.func:
return self.func(json_input)
if isinstance(json_input, STRING_TYPES):
return json_input
if self.flatten:
if not isinstance(json_input, dict):
return flattenCell(json_input, is_pretty)
return '\n'.join(
[u'{key}: {val}'.format(key=k, val=flattenCell(v, is_pretty)) for k, v in json_input.items()]) # for BC
str_lst = []
prev_path = [] # type: ignore
for path, key, val in self.json_to_path_generator(json_input):
str_path = ''
full_tabs = '\t' * len(path)
if path != prev_path: # need to construct tha `path` string only of it changed from the last one
common_prefix_index = len(os.path.commonprefix((prev_path, path))) # type: ignore
path_suffix = path[common_prefix_index:]
str_path_lst = []
for i, p in enumerate(path_suffix):
is_list = isinstance(p, int)
tabs = (common_prefix_index + i) * '\t'
path_value = p if not is_list else '-'
delim = ':\n' if not is_list else ''
str_path_lst.append('{tabs}**{path_value}**{delim}'.format(tabs=tabs, path_value=path_value, delim=delim))
str_path = ''.join(str_path_lst)
prev_path = path
if path and isinstance(path[-1], int):
# if it is a beginning of a list, there is only one tab left
full_tabs = '\t'
str_lst.append(
'{path}{tabs}***{key}***: {val}'.format(path=str_path, tabs=full_tabs, key=key, val=flattenCell(val, is_pretty)))
return '\n'.join(str_lst)
def json_to_path_generator(self, json_input, path=None):
"""
:type json_input: ``list`` or ``dict``
:param json_input: The json input to transform
:type path: ``List[str + int]``
:param path: The path of the key, value pair inside the json
:rtype ``Tuple[List[str + int], str, str]``
:return: A tuple. the second and third elements are key, values, and the first is their path in the json
"""
if path is None:
path = []
is_in_path = not self.keys or any(p for p in path if p in self.keys)
if isinstance(json_input, dict):
for k, v in json_input.items():
if is_in_path or k in self.keys: # found data to return
# recurse until finding a primitive value
if not isinstance(v, dict) and not isinstance(v, list):
yield path, k, v
else:
for res in self.json_to_path_generator(v, path + [k]): # this is yield from for python2 BC
yield res
elif self.is_nested:
# recurse all the json_input to find the relevant data
for res in self.json_to_path_generator(v, path + [k]): # this is yield from for python2 BC
yield res
if isinstance(json_input, list):
if not json_input or (not isinstance(json_input[0], list) and not isinstance(json_input[0], dict)):
# if the items of the lists are primitive, put the values in one line
yield path, 'values', ', '.join(str(x) for x in json_input)
else:
for i, item in enumerate(json_input):
for res in self.json_to_path_generator(item, path + [i]): # this is yield from for python2 BC
yield res
def tableToMarkdown(name, t, headers=None, headerTransform=None, removeNull=False, metadata=None, url_keys=None,
date_fields=None, json_transform_mapping=None, is_auto_json_transform=False, sort_headers=True):
"""
Converts a demisto table in JSON form to a Markdown table
:type name: ``str``
:param name: The name of the table (required)
:type t: ``dict`` or ``list``
:param t: The JSON table - List of dictionaries with the same keys or a single dictionary (required)
:type headers: ``list`` or ``string``
:param headers: A list of headers to be presented in the output table (by order). If string will be passed
then table will have single header. Default will include all available headers.
:type headerTransform: ``function``
:param headerTransform: A function that formats the original data headers (optional)
:type removeNull: ``bool``
:param removeNull: Remove empty columns from the table. Default is False
:type metadata: ``str``
:param metadata: Metadata about the table contents
:type url_keys: ``list``
:param url_keys: a list of keys in the given JSON table that should be turned in to clickable
:type date_fields: ``list``
:param date_fields: A list of date fields to format the value to human-readable output.
:type json_transform_mapping: ``Dict[str, JsonTransformer]``
:param json_transform_mapping: A mapping between a header key to corresponding JsonTransformer
:type is_auto_json_transform: ``bool``
:param is_auto_json_transform: Boolean to try to auto transform complex json
:type sort_headers: ``bool``
:param sort_headers: Sorts the table based on its headers only if the headers parameter is not specified
:return: A string representation of the markdown table
:rtype: ``str``
"""
# Turning the urls in the table to clickable
if url_keys:
t = url_to_clickable_markdown(t, url_keys)
mdResult = ''
if name:
mdResult = '### ' + name + '\n'
if metadata:
mdResult += metadata + '\n'
if not t or len(t) == 0:
mdResult += '**No entries.**\n'
return mdResult
if not headers and isinstance(t, dict) and len(t.keys()) == 1:
# in case of a single key, create a column table where each element is in a different row.
headers = list(t.keys())
# if the value of the single key is a non-empty list, unpack it for creating a column table.
single_value = list(t.values())[0]
if isinstance(single_value, list) and single_value:
t = single_value
if not isinstance(t, list):
t = [t]
if headers and isinstance(headers, STRING_TYPES):
headers = [headers]
if not isinstance(t[0], dict):
# the table contains only simple objects (strings, numbers)
# should be only one header
if headers and len(headers) > 0:
header = headers[0]
t = [{header: item} for item in t]
else:
raise Exception("Missing headers param for tableToMarkdown. Example: headers=['Some Header']")
# in case of headers was not provided (backward compatibility)
if not headers:
headers = list(t[0].keys())
if sort_headers or not IS_PY3:
headers.sort()
if removeNull:
headers_aux = headers[:]
for header in headers:
if all(obj.get(header) in ('', None, [], {}) for obj in t):
headers_aux.remove(header)
headers = headers_aux
if not json_transform_mapping:
json_transform_mapping = {header: JsonTransformer(flatten=not is_auto_json_transform) for header in
headers}
if t and len(headers) > 0:
newHeaders = []
if headerTransform is None: # noqa
def headerTransform(s): return stringEscapeMD(s, True, True) # noqa
for header in headers:
newHeaders.append(headerTransform(header))
mdResult += '|'
if len(newHeaders) == 1:
mdResult += newHeaders[0]
else:
mdResult += '|'.join(newHeaders)
mdResult += '|\n'
sep = '---'
mdResult += '|' + '|'.join([sep] * len(headers)) + '|\n'
for entry in t:
entry_copy = entry.copy()
if date_fields:
for field in date_fields:
try:
entry_copy[field] = datetime.fromtimestamp(int(entry_copy[field]) / 1000).strftime('%Y-%m-%d %H:%M:%S')
except Exception:
pass
vals = [stringEscapeMD((formatCell(entry_copy.get(h, ''), False,
json_transform_mapping.get(h)) if entry_copy.get(h) is not None else ''),
True, True) for h in headers]
# this pipe is optional
mdResult += '| '
try:
mdResult += ' | '.join(vals)
except UnicodeDecodeError:
vals = [str(v) for v in vals]
mdResult += ' | '.join(vals)
mdResult += ' |\n'
else:
mdResult += '**No entries.**\n'
return mdResult
tblToMd = tableToMarkdown
def createContextSingle(obj, id=None, keyTransform=None, removeNull=False): # pragma: no cover
"""Receives a dict with flattened key values, and converts them into nested dicts
:type obj: ``dict`` or ``list``
:param obj: The data to be added to the context (required)
:type id: ``str``
:param id: The ID of the context entry
:type keyTransform: ``function``
:param keyTransform: A formatting function for the markdown table headers
:type removeNull: ``bool``
:param removeNull: True if empty columns should be removed, false otherwise
:return: The converted context list
:rtype: ``list``
"""
res = {} # type: dict
if keyTransform is None:
def keyTransform(s): return s # noqa
keys = obj.keys()
for key in keys:
if removeNull and obj[key] in ('', None, [], {}):
continue
values = key.split('.')
current = res
for v in values[:-1]:
current.setdefault(v, {})
current = current[v]
current[keyTransform(values[-1])] = obj[key]
if id is not None:
res.setdefault('ID', id)
return res
def createContext(data, id=None, keyTransform=None, removeNull=False):
"""Receives a dict with flattened key values, and converts them into nested dicts
:type data: ``dict`` or ``list``
:param data: The data to be added to the context (required)
:type id: ``str``
:param id: The ID of the context entry
:type keyTransform: ``function``
:param keyTransform: A formatting function for the markdown table headers
:type removeNull: ``bool``
:param removeNull: True if empty columns should be removed, false otherwise
:return: The converted context list
:rtype: ``list``
"""
if isinstance(data, (list, tuple)):
return [createContextSingle(d, id, keyTransform, removeNull) for d in data]
else:
return createContextSingle(data, id, keyTransform, removeNull)
def sectionsToMarkdown(root):
"""
Converts a list of Demisto JSON tables to markdown string of tables
:type root: ``dict`` or ``list``
:param root: The JSON table - List of dictionaries with the same keys or a single dictionary (required)
:return: A string representation of the markdown table
:rtype: ``str``
"""
mdResult = ''
if isinstance(root, dict):
for section in root:
data = root[section]
if isinstance(data, dict):
data = [data]
data = [{k: formatCell(row[k]) for k in row} for row in data]
mdResult += tblToMd(section, data)
return mdResult
def fileResult(filename, data, file_type=None):
"""
Creates a file from the given data
:type filename: ``str``
:param filename: The name of the file to be created (required)
:type data: ``str`` or ``bytes``
:param data: The file data (required)
:type file_type: ``str``
:param file_type: one of the entryTypes file or entryInfoFile (optional)
:return: A Demisto war room entry
:rtype: ``dict``
"""
if file_type is None:
file_type = entryTypes['file']
temp = demisto.uniqueFile()
# pylint: disable=undefined-variable
if (IS_PY3 and isinstance(data, str)) or (not IS_PY3 and isinstance(data, unicode)): # type: ignore # noqa: F821
data = data.encode('utf-8')
# pylint: enable=undefined-variable
with open(demisto.investigation()['id'] + '_' + temp, 'wb') as f:
f.write(data)
# when there is ../ in the filename, xsoar thinks that path of the file is in the previous folder(s) and because of that
# xsoar returns empty files to war-rooms
if isinstance(filename, str):
replaced_filename = filename.replace("../", "")
if filename != replaced_filename:
filename = replaced_filename
demisto.debug(
"replaced {filename} with new file name {replaced_file_name}".format(
filename=filename, replaced_file_name=replaced_filename
)
)
return {'Contents': '', 'ContentsFormat': formats['text'], 'Type': file_type, 'File': filename, 'FileID': temp}
def hash_djb2(s, seed=5381):
"""
Hash string with djb2 hash function
:type s: ``str``
:param s: The input string to hash
:type seed: ``int``
:param seed: The seed for the hash function (default is 5381)
:return: The hashed value
:rtype: ``int``
"""
hash_name = seed
for x in s:
hash_name = ((hash_name << 5) + hash_name) + ord(x)
return hash_name & 0xFFFFFFFF
def file_result_existing_file(filename, saveFilename=None):
"""
Rename an existing file
:type filename: ``str``
:param filename: The name of the file to be modified (required)
:type saveFilename: ``str``
:param saveFilename: The new file name
:return: A Demisto war room entry
:rtype: ``dict``
"""
temp = demisto.uniqueFile()
os.rename(filename, demisto.investigation()['id'] + '_' + temp)
return {'Contents': '', 'ContentsFormat': formats['text'], 'Type': entryTypes['file'],
'File': saveFilename if saveFilename else filename, 'FileID': temp}
def flattenRow(rowDict):
"""
Flatten each element in the given rowDict
:type rowDict: ``dict``
:param rowDict: The dict to be flattened (required)
:return: A flattened dict
:rtype: ``dict``
"""
return {k: formatCell(rowDict[k]) for k in rowDict}
def flattenTable(tableDict):
"""
Flatten each row in the given tableDict
:type tableDict: ``dict``
:param tableDict: The table to be flattened (required)
:return: A flattened table
:rtype: ``dict``
"""
return [flattenRow(row) for row in tableDict]
MARKDOWN_CHARS = r"\`*_{}[]()#+-!|~"
def stringEscapeMD(st, minimal_escaping=False, escape_multiline=False):
"""
Escape any chars that might break a markdown string
:type st: ``str``
:param st: The string to be modified (required)
:type minimal_escaping: ``bool``
:param minimal_escaping: Whether replace all special characters or table format only (optional)
:type escape_multiline: ``bool``
:param escape_multiline: Whether convert line-ending characters (optional)
:return: A modified string
:rtype: ``str``
"""
if escape_multiline:
st = st.replace('\r\n', '<br>') # Windows
st = st.replace('\r', '<br>') # old Mac
st = st.replace('\n', '<br>') # Unix
if minimal_escaping:
for c in ('|', '`'):
st = st.replace(c, '\\' + c)
else:
st = "".join(["\\" + str(c) if c in MARKDOWN_CHARS else str(c) for c in st])
return st
def sanitize_html_output(value, allow_tags=None):
# type: (str, Optional[Set[str]]) -> str
"""Escape HTML for safe rendering in ContentsFormat:'html' outputs.
:type value: ``str``
:param value: Raw string that may contain attacker-controlled content.
:type allow_tags: ``Optional[Set[str]]``
:param allow_tags: Optional set of allowed HTML tag names
(e.g. ``{'b','i','a','br','p','table','tr','td','th'}``). If ``None``, escapes everything.
:return: HTML-safe string.
:rtype: ``str``
"""
try:
from html import escape as _html_escape # Python 3
except ImportError:
from cgi import escape as _cgi_escape # Python 2
def _html_escape(s, quote=True):
return _cgi_escape(s, quote=quote).replace("'", "'")
if allow_tags is None:
return _html_escape(str(value))
# For allowlist mode, use bleach if available, else strip all
try:
import bleach
return bleach.clean(str(value), tags=allow_tags, strip=True)
except ImportError:
return _html_escape(str(value))
def getFilePathSafe(entry_id):
# type: (str) -> dict
"""Wrapper around demisto.getFilePath() that basenames the 'name' field.
Prevents path traversal when callers use the returned name as a filesystem destination.
:type entry_id: ``str``
:param entry_id: The entry ID of the file.
:return: dict with ``'id'``, ``'path'``, and ``'name'`` keys, where ``'name'`` is basenamed.
:rtype: ``dict``
"""
result = demisto.getFilePath(entry_id)
if not result:
return result
if "name" in result:
result["name"] = os.path.basename(result["name"])
return result
def raiseTable(root, key):
newInternal = {}
if key in root and isinstance(root[key], dict):
for sub in root[key]:
if sub not in root:
root[sub] = root[key][sub]
else:
newInternal[sub] = root[key][sub]
if newInternal:
root[key] = newInternal
else:
del root[key]
def zoomField(item, fieldName):
if isinstance(item, dict) and fieldName in item:
return item[fieldName]
else:
return item
def isCommandAvailable(cmd):
"""
Check the list of available modules to see whether a command is currently available to be run.
:type cmd: ``str``
:param cmd: The command to check (required)
:return: True if command is available, False otherwise
:rtype: ``bool``
"""
modules = demisto.getAllSupportedCommands()
for m in modules:
if modules[m] and isinstance(modules[m], list):
for c in modules[m]:
if c['name'] == cmd:
return True
return False
def epochToTimestamp(epoch):
return datetime.utcfromtimestamp(epoch / 1000.0).strftime("%Y-%m-%d %H:%M:%S")
def formatTimeColumns(data, timeColumnNames):
for row in data:
for k in timeColumnNames:
row[k] = epochToTimestamp(row[k])
def strip_tag(tag):
split_array = tag.split('}')
if len(split_array) > 1:
strip_ns_tag = split_array[1]
tag = strip_ns_tag
return tag
def elem_to_internal(elem, strip_ns=1, strip=1):
"""Convert an Element into an internal dictionary (not JSON!)."""
d = OrderedDict() # type: dict
elem_tag = elem.tag
if strip_ns:
elem_tag = strip_tag(elem.tag)
for key, value in list(elem.attrib.items()):
d['@' + key] = value
# loop over subelements to merge them
for subelem in elem:
v = elem_to_internal(subelem, strip_ns=strip_ns, strip=strip)
tag = subelem.tag
if strip_ns:
tag = strip_tag(subelem.tag)
value = v[tag]
try:
# add to existing list for this tag
d[tag].append(value)
except AttributeError:
# turn existing entry into a list
d[tag] = [d[tag], value]
except KeyError:
# add a new non-list entry
d[tag] = value
text = elem.text
tail = elem.tail
if strip:
# ignore leading and trailing whitespace
if text:
text = text.strip()
if tail:
tail = tail.strip()
if tail:
d['#tail'] = tail
if d:
# use #text element if other attributes exist
if text:
d["#text"] = text
else:
# text is the value if no attributes
d = text or None # type: ignore
return {elem_tag: d}
def internal_to_elem(pfsh, factory=ET.Element):
"""Convert an internal dictionary (not JSON!) into an Element.
Whatever Element implementation we could import will be
used by default; if you want to use something else, pass the
Element class as the factory parameter.
"""
attribs = OrderedDict() # type: dict
text = None
tail = None
sublist = []
tag = list(pfsh.keys())
if len(tag) != 1:
raise ValueError("Illegal structure with multiple tags: %s" % tag)
tag = tag[0]
value = pfsh[tag]
if isinstance(value, dict):
for k, v in list(value.items()):
if k[:1] == "@":
attribs[k[1:]] = v
elif k == "#text":
text = v
elif k == "#tail":
tail = v
elif isinstance(v, list):
for v2 in v:
sublist.append(internal_to_elem({k: v2}, factory=factory))
else:
sublist.append(internal_to_elem({k: v}, factory=factory))
else:
text = value
e = factory(tag, attribs)
for sub in sublist:
e.append(sub)
e.text = text
e.tail = tail
return e
def elem2json(elem, options, strip_ns=1, strip=1):
"""Convert an ElementTree or Element into a JSON string."""
if hasattr(elem, 'getroot'):
elem = elem.getroot()
if 'pretty' in options:
return json.dumps(elem_to_internal(elem, strip_ns=strip_ns, strip=strip), indent=4, separators=(',', ': '))
else:
return json.dumps(elem_to_internal(elem, strip_ns=strip_ns, strip=strip))
def json2elem(json_data, factory=ET.Element):
"""Convert a JSON string into an Element.
Whatever Element implementation we could import will be used by
default; if you want to use something else, pass the Element class
as the factory parameter.
"""
return internal_to_elem(json.loads(json_data), factory)
def xml2json(xmlstring, options={}, strip_ns=1, strip=1):
"""
Convert an XML string into a JSON string.
:type xmlstring: ``str``
:param xmlstring: The string to be converted (required)
:return: The converted JSON
:rtype: ``dict`` or ``list``
"""
try:
import defusedxml.ElementTree as defused_ET
elem = defused_ET.fromstring(xmlstring)
except ImportError:
demisto.debug('defused_ET is not supported, using ET instead.')
elem = ET.fromstring(xmlstring)
return elem2json(elem, options, strip_ns=strip_ns, strip=strip)
def json2xml(json_data, factory=ET.Element):
"""Convert a JSON string into an XML string.
Whatever Element implementation we could import will be used by
default; if you want to use something else, pass the Element class
as the factory parameter.
"""
if not isinstance(json_data, dict):
json_data = json.loads(json_data)
elem = internal_to_elem(json_data, factory)
return ET.tostring(elem, encoding='utf-8')
def get_hash_type(hash_file):
"""
Checks the type of the given hash. Returns 'md5', 'sha1', 'sha256' or 'Unknown'.
:type hash_file: ``str``
:param hash_file: The hash to be checked (required)
:return: The hash type
:rtype: ``str``
"""
hash_len = len(hash_file)
if (hash_len == 32):
return 'md5'
elif (hash_len == 40):
return 'sha1'
elif (hash_len == 64):
return 'sha256'
elif (hash_len == 128):
return 'sha512'
else:
return 'Unknown'
def is_mac_address(mac):
"""
Test for valid mac address
:type mac: ``str``
:param mac: MAC address in the form of AA:BB:CC:00:11:22
:return: True/False
:rtype: ``bool``
"""
if re.search(r'([0-9A-F]{2}[:]){5}([0-9A-F]){2}', mac.upper()) is not None:
return True
else:
return False
def is_ipv6_valid(address):
"""
Checks if the given string represents a valid IPv6 address.
:type address: str
:param address: The string to check.
:return: True if the given string represents a valid IPv6 address.
:rtype: ``bool``
"""
try:
socket.inet_pton(socket.AF_INET6, address)
except socket.error: # not a valid address
return False
return True
def is_ip_address_internal(ip):
"""
Checks if an IP address is an internal (RFC 1918) IP, Available from python3.
:return: True if the given IP address is an internal.
:rtype: ``bool``
"""
if IS_PY3:
# pylint: disable=import-error
import ipaddress
# pylint: enable=import-error
try:
ip_obj = ipaddress.ip_address(ip)
return ip_obj.is_private
except ValueError:
return False
else:
return False
def is_ip_valid(s, accept_v6_ips=False):
"""
Checks if the given string represents a valid IP address.
By default, will only return 'True' for IPv4 addresses.
:type s: ``str``
:param s: The string to be checked (required)
:type accept_v6_ips: ``bool``
:param accept_v6_ips: A boolean determining whether the
function should accept IPv6 addresses
:return: True if the given string represents a valid IP address, False otherwise
:rtype: ``bool``
"""
a = s.split('.')
if accept_v6_ips and is_ipv6_valid(s):
return True
elif len(a) != 4:
return False
else:
for x in a:
if not x.isdigit():
return False
i = int(x)
if i < 0 or i > 255:
return False
return True
def get_integration_name():
"""
Getting calling integration's name
:return: Calling integration's name
:rtype: ``str``
"""
return demisto.callingContext.get('context', {}).get('IntegrationBrand', '')
def get_integration_instance_name():
"""
Getting calling integration instance name
:return: Calling integration instance name
:rtype: ``str``
"""
return demisto.callingContext.get('context', {}).get('IntegrationInstance', '')
def get_script_name():
"""
Getting calling script name
:return: Calling script name
:rtype: ``str``
"""
return demisto.callingContext.get('context', {}).get('ScriptName', '')
class Common(object):
class Indicator(object):
"""
interface class
"""
@abstractmethod
def to_context(self):
pass
@abstractmethod
def to_minimum_context(self):
pass
@staticmethod
def create_context_table(data):
"""
Gets a list of items of a specific class (such as CommunityNotes, Publications etc) and returns a context
list.
"""
table = []
for item in data:
table.append(item.to_context())
return table
class DBotScore(object):
"""
DBotScore class
:type indicator: ``str``
:param indicator: indicator value, ip, hash, domain, url, etc
:type indicator_type: ``DBotScoreType``
:param indicator_type: use DBotScoreType class
:type integration_name: ``str``
:param integration_name: For integrations - The class will automatically determine the integration name.
For scripts - The class will use the given integration name.
:type score: ``DBotScore``
:param score: DBotScore.NONE, DBotScore.GOOD, DBotScore.SUSPICIOUS, DBotScore.BAD
:type malicious_description: ``str``
:param malicious_description: if the indicator is malicious and have explanation for it then set it to this field
:type reliability: ``DBotScoreReliability``
:param reliability: use DBotScoreReliability class
:type message: ``str``
:param message: Used to message on the api response, for example: When return api response is "Not found".
:return: None
:rtype: ``None``
"""
NONE = 0
GOOD = 1
SUSPICIOUS = 2
BAD = 3
CONTEXT_PATH = 'DBotScore(val.Indicator && val.Indicator == obj.Indicator && val.Vendor == obj.Vendor ' \
'&& val.Type == obj.Type)'
CONTEXT_PATH_PRIOR_V5_5 = 'DBotScore'
def __init__(self, indicator, indicator_type, integration_name='', score=None, malicious_description=None,
reliability=None, message=None):
if not DBotScoreType.is_valid_type(indicator_type):
raise TypeError('indicator_type must be of type DBotScoreType enum')
if not Common.DBotScore.is_valid_score(score):
raise TypeError('indicator `score` must be of type DBotScore enum')
if reliability and not DBotScoreReliability.is_valid_type(reliability):
raise TypeError('reliability must be of type DBotScoreReliability enum')
self.indicator = indicator
self.indicator_type = indicator_type
# For integrations - The class will automatically determine the integration name.
if demisto.callingContext.get('integration'):
context_integration_name = get_integration_name()
self.integration_name = context_integration_name if context_integration_name else integration_name
else:
self.integration_name = integration_name
self.score = score
self.malicious_description = malicious_description
self.reliability = reliability
self.message = message
@staticmethod
def is_valid_score(score):
return score in (
Common.DBotScore.NONE,
Common.DBotScore.GOOD,
Common.DBotScore.SUSPICIOUS,
Common.DBotScore.BAD
)
@staticmethod
def get_context_path():
return Common.DBotScore.CONTEXT_PATH
def to_context(self):
dbot_context = {
'Indicator': self.indicator,
'Type': self.indicator_type,
'Vendor': self.integration_name,
'Score': self.score
}
if self.reliability:
dbot_context['Reliability'] = self.reliability
if self.message:
dbot_context['Message'] = self.message
ret_value = {
Common.DBotScore.get_context_path(): dbot_context
}
return ret_value
def to_minimum_context(self):
return self.to_context()
def to_readable(self):
dbot_score_to_text = {0: 'Unknown',
1: 'Good',
2: 'Suspicious',
3: 'Bad'}
return dbot_score_to_text.get(self.score, 'Undefined')
class CustomIndicator(Indicator):
def __init__(self, indicator_type, value, dbot_score, data, context_prefix, relationships=None):
"""
:type indicator_type: ``Str``
:param indicator_type: The name of the indicator type.
:type value: ``Any``
:param value: The value of the indicator.
:type dbot_score: ``DBotScore``
:param dbot_score: If custom indicator has a score then create and set a DBotScore object.
:type data: ``Dict(Str,Any)``
:param data: A dictionary containing all the param names and their values.
:type context_prefix: ``Str``
:param context_prefix: Will be used as the context path prefix.
:type relationships: ``list of EntityRelationship``
:param relationships: List of relationships of the indicator.
:return: None
:rtype: ``None``
"""
if hasattr(DBotScoreType, indicator_type.upper()):
raise ValueError('Creating a custom indicator type with an existing type name is not allowed')
if not value:
raise ValueError('value is mandatory for creating the indicator')
if not context_prefix:
raise ValueError('context_prefix is mandatory for creating the indicator')
self.CONTEXT_PATH = '{context_prefix}(val.value && val.value == obj.value)'. \
format(context_prefix=context_prefix)
self.value = value
self.relationships = relationships
if not isinstance(dbot_score, Common.DBotScore):
raise ValueError('dbot_score must be of type DBotScore')
self.dbot_score = dbot_score
self.indicator_type = indicator_type
self.data = data
INDICATOR_TYPE_TO_CONTEXT_KEY[indicator_type.lower()] = indicator_type.capitalize()
for key in self.data:
setattr(self, key, data[key])
def to_context(self):
custom_context = {
'value': self.value
}
custom_context.update(self.data)
ret_value = {
self.CONTEXT_PATH: custom_context
} # type: Dict[str, Any]
if self.dbot_score:
ret_value.update(self.dbot_score.to_context())
ret_value[Common.DBotScore.get_context_path()]['Type'] = self.indicator_type
if self.relationships:
relationships_context = [relationship.to_context() for relationship in self.relationships if
relationship.to_context()]
ret_value['Relationships'] = relationships_context
return ret_value
def to_minimum_context(self):
custom_context = {
'value': self.value
}
ret_value = {
self.CONTEXT_PATH: custom_context
}
if self.dbot_score:
ret_value.update(self.dbot_score.to_context())
ret_value[Common.DBotScore.get_context_path()]['Type'] = self.indicator_type
return ret_value
class IP(Indicator):
"""
IP indicator class - https://xsoar.pan.dev/docs/integrations/context-standards-mandatory#ip
:type ip: ``str``
:param ip: IP address
:type asn: ``str``
:param asn: The autonomous system name for the IP address, for example: "AS8948".
:type as_owner: ``str``
:param as_owner: The autonomous system owner of the IP.
:type region: ``str``
:param region: The region in which the IP is located.
:type port: ``str``
:param port: Ports that are associated with the IP.
:type internal: ``bool``
:param internal: Whether or not the IP is internal or external.
:type updated_date: ``date``
:param updated_date: The date that the IP was last updated.
:type registrar_abuse_name: ``str``
:param registrar_abuse_name: The name of the contact for reporting abuse.
:type registrar_abuse_address: ``str``
:param registrar_abuse_address: The address of the contact for reporting abuse.
:type registrar_abuse_country: ``str``
:param registrar_abuse_country: The country of the contact for reporting abuse.
:type registrar_abuse_network: ``str``
:param registrar_abuse_network: The network of the contact for reporting abuse.
:type registrar_abuse_phone: ``str``
:param registrar_abuse_phone: The phone number of the contact for reporting abuse.
:type registrar_abuse_email: ``str``
:param registrar_abuse_email: The email address of the contact for reporting abuse.
:type campaign: ``str``
:param campaign: The campaign associated with the IP.
:type traffic_light_protocol: ``str``
:param traffic_light_protocol: The Traffic Light Protocol (TLP) color that is suitable for the IP.
:type community_notes: ``CommunityNotes``
:param community_notes: Notes on the IP that were given by the community.
:type publications: ``Publications``
:param publications: Publications on the ip that was published.
:type threat_types: ``ThreatTypes``
:param threat_types: Threat types that are associated with the file.
:type hostname: ``str``
:param hostname: The hostname that is mapped to this IP address.
:type geo_latitude: ``str``
:param geo_latitude: The geolocation where the IP address is located, in the format: latitude
:type geo_longitude: ``str``
:param geo_longitude: The geolocation where the IP address is located, in the format: longitude.
:type geo_country: ``str``
:param geo_country: The country in which the IP address is located.
:type geo_description: ``str``
:param geo_description: Additional information about the location.
:type detection_engines: ``int``
:param detection_engines: The total number of engines that checked the indicator.
:type positive_engines: ``int``
:param positive_engines: The number of engines that positively detected the indicator as malicious.
:type organization_name: ``str``
:param organization_name: The organization of the IP
:type organization_type: ``str``
:param organization_type:The organization type of the IP
:type tags: ``str``
:param tags: Tags of the IP.
:type malware_family: ``str``
:param malware_family: The malware family associated with the IP.
:type feed_related_indicators: ``FeedRelatedIndicators``
:param feed_related_indicators: List of indicators that are associated with the IP.
:type relationships: ``list of EntityRelationship``
:param relationships: List of relationships of the indicator.
:type blocked: ``boolean``
:param blocked: Is the indicator blocked.
:type description: ``str``
:param description: A general description of the indicator
:type stix_id: ``str``
:param stix_id: The STIX id representing the indicator
:type whois_records: ``WhoisRecord``
:param whois_records: List of whois records
:type dbot_score: ``DBotScore``
:param dbot_score: If IP has a score then create and set a DBotScore object.
:type organization_prevalence: ``int``
:param organization_prevalence: The number of times the indicator is detected in the organization.
:type global_prevalence: ``int``
:param global_prevalence: The number of times the indicator is detected across all organizations.
:type organization_first_seen: ``str``
:param organization_first_seen: ISO 8601 date time string; when the indicator was first seen in the organization.
:type organization_last_seen: ``str``
:param organization_last_seen: ISO 8601 date time string; the last time a specific organization encountered an indicator.
:type first_seen_by_source: ``str``
:param first_seen_by_source: ISO 8601 date time string; when the indicator was first seen by the source vendor.
:type last_seen_by_source: ``str``
:param last_seen_by_source: ISO 8601 date time string; when the indicator was last seen by the source vendor.
:return: None
:rtype: ``None``
"""
CONTEXT_PATH = 'IP(val.Address && val.Address == obj.Address)'
def __init__(self, ip, dbot_score, asn=None, as_owner=None, region=None, port=None, internal=None,
updated_date=None, registrar_abuse_name=None, registrar_abuse_address=None,
registrar_abuse_country=None, registrar_abuse_network=None, registrar_abuse_phone=None,
registrar_abuse_email=None, campaign=None, traffic_light_protocol=None,
community_notes=None, publications=None, threat_types=None,
hostname=None, geo_latitude=None, geo_longitude=None,
geo_country=None, geo_description=None, detection_engines=None, positive_engines=None,
organization_name=None, organization_type=None, feed_related_indicators=None, tags=None,
malware_family=None, relationships=None, blocked=None, description=None, stix_id=None,
whois_records=None, organization_prevalence=None,
global_prevalence=None, organization_first_seen=None, organization_last_seen=None,
first_seen_by_source=None, last_seen_by_source=None, ip_type="IP"):
# Main value of the indicator
self.ip = ip
self.ip_type = ip_type
# Core custom fields - IP
self.blocked = blocked
self.community_notes = community_notes
self.description = description
self.tags = tags
self.geo_country = geo_country
self.geo_latitude = geo_latitude
self.geo_longitude = geo_longitude
self.internal = internal
self.stix_id = stix_id
self.traffic_light_protocol = traffic_light_protocol
self.whois_records = whois_records
# Other custom fields
self.asn = asn
self.as_owner = as_owner
self.region = region
self.port = port
self.updated_date = updated_date
self.registrar_abuse_name = registrar_abuse_name
self.registrar_abuse_address = registrar_abuse_address
self.registrar_abuse_country = registrar_abuse_country
self.registrar_abuse_network = registrar_abuse_network
self.registrar_abuse_phone = registrar_abuse_phone
self.registrar_abuse_email = registrar_abuse_email
self.campaign = campaign
self.publications = publications
self.threat_types = threat_types
self.hostname = hostname
self.geo_description = geo_description
self.detection_engines = detection_engines
self.positive_engines = positive_engines
self.organization_name = organization_name
self.organization_type = organization_type
self.feed_related_indicators = feed_related_indicators
self.malware_family = malware_family
self.relationships = relationships
self.organization_prevalence = organization_prevalence
self.global_prevalence = global_prevalence
self.organization_first_seen = organization_first_seen
self.organization_last_seen = organization_last_seen
self.first_seen_by_source = first_seen_by_source
self.last_seen_by_source = last_seen_by_source
if not isinstance(dbot_score, Common.DBotScore):
raise ValueError('dbot_score must be of type DBotScore')
self.dbot_score = dbot_score
def to_context(self):
ip_context = {
'Address': self.ip
}
if self.blocked:
ip_context['Blocked'] = self.blocked
if self.asn:
ip_context['ASN'] = self.asn
if self.as_owner:
ip_context['ASOwner'] = self.as_owner
if self.region:
ip_context['Region'] = self.region
if self.port:
ip_context['Port'] = self.port
if self.internal:
ip_context['Internal'] = self.internal
if self.stix_id:
ip_context['STIXID'] = self.stix_id
if self.updated_date:
ip_context['UpdatedDate'] = self.updated_date
if self.registrar_abuse_name or self.registrar_abuse_address or self.registrar_abuse_country or \
self.registrar_abuse_network or self.registrar_abuse_phone or self.registrar_abuse_email:
ip_context['Registrar'] = {'Abuse': {}}
if self.registrar_abuse_name:
ip_context['Registrar']['Abuse']['Name'] = self.registrar_abuse_name
if self.registrar_abuse_address:
ip_context['Registrar']['Abuse']['Address'] = self.registrar_abuse_address
if self.registrar_abuse_country:
ip_context['Registrar']['Abuse']['Country'] = self.registrar_abuse_country
if self.registrar_abuse_network:
ip_context['Registrar']['Abuse']['Network'] = self.registrar_abuse_network
if self.registrar_abuse_phone:
ip_context['Registrar']['Abuse']['Phone'] = self.registrar_abuse_phone
if self.registrar_abuse_email:
ip_context['Registrar']['Abuse']['Email'] = self.registrar_abuse_email
if self.campaign:
ip_context['Campaign'] = self.campaign
if self.description:
ip_context['Description'] = self.description
if self.traffic_light_protocol:
ip_context['TrafficLightProtocol'] = self.traffic_light_protocol
if self.community_notes:
ip_context['CommunityNotes'] = self.create_context_table(self.community_notes)
if self.publications:
ip_context['Publications'] = self.create_context_table(self.publications)
if self.threat_types:
ip_context['ThreatTypes'] = self.create_context_table(self.threat_types)
if self.whois_records:
ip_context['WhoisRecords'] = self.create_context_table(self.whois_records)
if self.hostname:
ip_context['Hostname'] = self.hostname
if self.geo_latitude or self.geo_country or self.geo_description:
ip_context['Geo'] = {}
if self.geo_latitude and self.geo_longitude:
ip_context['Geo']['Location'] = '{}:{}'.format(self.geo_latitude, self.geo_longitude)
if self.geo_country:
ip_context['Geo']['Country'] = self.geo_country
if self.geo_description:
ip_context['Geo']['Description'] = self.geo_description
if self.organization_name or self.organization_type:
ip_context['Organization'] = {}
if self.organization_name:
ip_context['Organization']['Name'] = self.organization_name
if self.organization_type:
ip_context['Organization']['Type'] = self.organization_type
if self.detection_engines is not None:
ip_context['DetectionEngines'] = self.detection_engines
if self.positive_engines is not None:
ip_context['PositiveDetections'] = self.positive_engines
if self.feed_related_indicators:
ip_context['FeedRelatedIndicators'] = self.create_context_table(self.feed_related_indicators)
if self.tags:
ip_context['Tags'] = self.tags
if self.malware_family:
ip_context['MalwareFamily'] = self.malware_family
if self.organization_prevalence is not None: # checking for `is not None` to allow `0`-value
ip_context['OrganizationPrevalence'] = self.organization_prevalence
if self.global_prevalence is not None: # checking for `is not None` to allow `0`-value
ip_context['GlobalPrevalence'] = self.global_prevalence
if self.organization_first_seen:
ip_context['OrganizationFirstSeen'] = self.organization_first_seen
if self.organization_last_seen:
ip_context['OrganizationLastSeen'] = self.organization_last_seen
if self.first_seen_by_source:
ip_context['FirstSeenBySource'] = self.first_seen_by_source
if self.last_seen_by_source:
ip_context['LastSeenBySource'] = self.last_seen_by_source
if self.dbot_score and self.dbot_score.score == Common.DBotScore.BAD:
ip_context['Malicious'] = {
'Vendor': self.dbot_score.integration_name,
'Description': self.dbot_score.malicious_description
}
if self.relationships:
relationships_context = [relationship.to_context() for relationship in self.relationships if
relationship.to_context()]
ip_context['Relationships'] = relationships_context
if self.ip_type == "IP":
context_path = Common.IP.CONTEXT_PATH
elif self.ip_type == "IPv6":
context_path = Common.IP.CONTEXT_PATH.replace("IP", "IPv6")
ret_value = {
context_path: ip_context
}
if self.dbot_score:
ret_value.update(self.dbot_score.to_context())
return ret_value
def to_minimum_context(self):
ip_context = {
'Address': self.ip
}
if self.ip_type == "IP":
context_path = Common.IP.CONTEXT_PATH
elif self.ip_type == "IPv6":
context_path = Common.IP.CONTEXT_PATH.replace("IP", "IPv6")
ret_value = {
context_path: ip_context
}
if self.dbot_score:
ret_value.update(self.dbot_score.to_context())
return ret_value
class FileSignature(object):
"""
FileSignature class
:type authentihash: ``str``
:param authentihash: The authentication hash.
:type copyright: ``str``
:param copyright: Copyright information.
:type description: ``str``
:param description: A description of the signature.
:type file_version: ``str``
:param file_version: The file version.
:type internal_name: ``str``
:param internal_name: The internal name of the file.
:type original_name: ``str``
:param original_name: The original name of the file.
:return: None
:rtype: ``None``
"""
def __init__(self, authentihash, copyright, description, file_version, internal_name, original_name):
self.authentihash = authentihash
self.copyright = copyright
self.description = description
self.file_version = file_version
self.internal_name = internal_name
self.original_name = original_name
def to_context(self):
return {
'Authentihash': self.authentihash,
'Copyright': self.copyright,
'Description': self.description,
'FileVersion': self.file_version,
'InternalName': self.internal_name,
'OriginalName': self.original_name,
}
def to_minimum_context(self):
return self.to_context()
class FeedRelatedIndicators(object):
"""
FeedRelatedIndicators class
Implements Subject Indicators that are associated with Another indicator
:type value: ``str``
:param value: Indicators that are associated with the indicator.
:type indicator_type: ``str``
:param indicator_type: The type of the indicators that are associated with the indicator.
:type description: ``str``
:param description: The description of the indicators that are associated with the indicator.
:return: None
:rtype: ``None``
"""
def __init__(self, value=None, indicator_type=None, description=None):
self.value = value
self.indicator_type = indicator_type
self.description = description
def to_context(self):
return {
'value': self.value,
'type': self.indicator_type,
'description': self.description
}
def to_minimum_context(self):
return self.to_context()
class Rank:
"""
Single row in a rank grid field
:type rank: float / int
:param rank: A numerical rank value
:type source: ``str``
:param source: The name of the source from which the rank was taken
"""
def __init__(self, rank=None, source=None):
self.rank = rank
self.source = source
def to_context(self):
return {
'source': self.source,
'rank': self.rank
}
def to_minimum_context(self):
return self.to_context()
class ExternalReference(object):
"""
ExternalReference class
Class to represent a single instance of an external reference for an indicator type.
:type source_name: ``str``
:param source_name: The name of the source referenced.
:type source_id: ``str``
:param source_id: The ID of the object in the external source.
:return: None
:rtype: ``None``
"""
def __init__(self, source_name=None, source_id=None):
self.source_name = source_name
self.source_id = source_id
def to_context(self):
return {
'sourcename': self.source_name,
'sourceid': self.source_id,
}
def to_minimum_context(self):
return self.to_context()
class Certificates(object):
"""
Certificates class
Class to represent a single instance of a certificate for an indicator type.
:type issued_to: ``str``
:param issued_to: Who was the certificated issued to
:type issued_by: ``str``
:param issued_by: Who issued the certificate
:type valid_from: ``str``
:param valid_from: Since when is the certificate valid
:type valid_to: ``str``
:param valid_to: Till when is the certificate valid
:return: None
:rtype: ``None``
"""
def __init__(self, issued_to=None, issued_by=None, valid_from=None, valid_to=None):
self.issued_to = issued_to
self.issued_by = issued_by
self.valid_from = valid_from
self.valid_to = valid_to
def to_context(self):
return {
'issuedto': self.issued_to,
'issuedby': self.issued_by,
'validfrom': self.valid_from,
'validto': self.valid_to
}
def to_minimum_context(self):
return self.to_context()
class Hash(object):
"""
Hash class
Class to represent a single instance of a hash for an indicator type.
:type hash_type: ``str``
:param hash_type: The type of the hash.
:type hash_value: ``str``
:param hash_value: The value of the hash.
:return: None
:rtype: ``None``
"""
def __init__(self, hash_type=None, hash_value=None):
self.hash_type = hash_type
self.hash_value = hash_value
def to_context(self):
return {
'type': self.hash_type,
'value': self.hash_value,
}
def to_minimum_context(self):
return self.to_context()
class CommunityNotes(object):
"""
CommunityNotes class
Implements Subject Community Notes of a indicator
:type note: ``str``
:param note: Notes on the indicator that were given by the community.
:type timestamp: ``Timestamp``
:param timestamp: The time in which the note was published.
:return: None
:rtype: ``None``
"""
def __init__(self, note=None, timestamp=None):
self.note = note
self.timestamp = timestamp
def to_context(self):
return {
'note': self.note,
'timestamp': self.timestamp,
}
def to_minimum_context(self):
return self.to_context()
class Publications(object):
"""
Publications class
Implements Subject Publications of a indicator
:type source: ``str``
:param source: The source in which the article was published.
:type title: ``str``
:param title: The name of the article.
:type link: ``str``
:param link: A link to the original article.
:type timestamp: ``Timestamp``
:param timestamp: The time in which the article was published.
:return: None
:rtype: ``None``
"""
def __init__(self, source=None, title=None, link=None, timestamp=None):
self.source = source
self.title = title
self.link = link
self.timestamp = timestamp
def to_context(self):
return {
'source': self.source,
'title': self.title,
'link': self.link,
'timestamp': self.timestamp,
}
def to_minimum_context(self):
return self.to_context()
class Behaviors(object):
"""
Behaviors class
Implements Subject Behaviors of a indicator
:type details: ``str``
:param details:
:type action: ``str``
:param action:
:return: None
:rtype: ``None``
"""
def __init__(self, details=None, action=None):
self.details = details
self.action = action
def to_context(self):
return {
'details': self.details,
'title': self.action,
}
def to_minimum_context(self):
return self.to_context()
class ThreatTypes(object):
"""
ThreatTypes class
Implements Subject ThreatTypes of a indicator
:type threat_category: ``str``
:param threat_category: The threat category associated to this indicator by the source vendor. For example,
Phishing, Control, TOR, etc.
:type threat_category_confidence: ``str``
:param threat_category_confidence: Threat Category Confidence is the confidence level provided by the vendor
for the threat type category
For example a confidence of 90 for threat type category "malware" means that the vendor rates that this
is 90% confidence of being a malware.
:return: None
:rtype: ``None``
"""
def __init__(self, threat_category=None, threat_category_confidence=None):
self.threat_category = threat_category
self.threat_category_confidence = threat_category_confidence
def to_context(self):
return {
'threatcategory': self.threat_category,
'threatcategoryconfidence': self.threat_category_confidence,
}
def to_minimum_context(self):
return self.to_context()
class WhoisRecord(object):
"""
WhoisRecord class
Class to represent a single instance of a record for an indicator type.
:type whois_record_type: ``str``
:param whois_record_type: The type of the whois record.
:type whois_record_value: ``str``
:param whois_record_value: The value of the whois record.
:type whois_record_date: ``Timestamp``
:param whois_record_date: when was the whois record fetched
:return: None
:rtype: ``None``
"""
def __init__(self, whois_record_type=None, whois_record_value=None, whois_record_date=None):
self.whois_record_type = whois_record_type
self.whois_record_value = whois_record_value
self.whois_record_date = whois_record_date
def to_context(self):
return {
'key': self.whois_record_type,
'value': self.whois_record_value,
'date': self.whois_record_date
}
def to_minimum_context(self):
return self.to_context()
class DNSRecord(object):
"""
DNSRecord class
Class to represent a single instance of a record for an indicator type.
:type dns_record_type: ``str``
:param dns_record_type: The type of the DNS record.
:type dns_record_data: ``str``
:param dns_record_data: The value of the whois record.
:type dns_ttl: ``str``
:param dns_ttl: The record TTL
:return: None
:rtype: ``None``
"""
def __init__(self, dns_record_type=None, dns_ttl=None, dns_record_data=None):
self.dns_record_type = dns_record_type
self.dns_ttl = dns_ttl
self.dns_record_data = dns_record_data
def to_context(self):
return {
'type': self.dns_record_type,
'ttl': self.dns_ttl,
'data': self.dns_record_data
}
def to_minimum_context(self):
return self.to_context()
class CPE:
"""
Represents one Common Platform Enumeration (CPE) object, see https://nvlpubs.nist.gov/nistpubs/legacy/ir/nistir7695.pdf
:type cpe: ``str``
:param cpe: a single CPE string
:return: None
:rtype: ``None``
"""
def __init__(self, cpe=None):
self.cpe = cpe
def to_context(self):
return {
'CPE': self.cpe,
}
def to_minimum_context(self):
return self.to_context()
class File(Indicator):
"""
File indicator class - https://xsoar.pan.dev/docs/integrations/context-standards-mandatory#file
:type name: ``str``
:param name: The full file name (including file extension).
:type entry_id: ``str``
:param entry_id: The ID for locating the file in the War Room.
:type size: ``int``
:param size: The size of the file in bytes.
:type md5: ``str``
:param md5: The MD5 hash of the file.
:type sha1: ``str``
:param sha1: The SHA1 hash of the file.
:type sha256: ``str``
:param sha256: The SHA256 hash of the file.
:type sha512: ``str``
:param sha512: The SHA512 hash of the file.
:type ssdeep: ``str``
:param ssdeep: The ssdeep hash of the file (same as displayed in file entries).
:type extension: ``str``
:param extension: The file extension, for example: "xls".
:type file_type: ``str``
:param file_type: The file type, as determined by libmagic (same as displayed in file entries).
:type hostname: ``str``
:param hostname: The name of the host where the file was found. Should match Path.
:type path: ``str``
:param path: The path where the file is located.
:type company: ``str``
:param company: The name of the company that released a binary.
:type product_name: ``str``
:param product_name: The name of the product to which this file belongs.
:type digital_signature__publisher: ``str``
:param digital_signature__publisher: The publisher of the digital signature for the file.
:type signature: ``FileSignature``
:param signature: File signature class
:type actor: ``str``
:param actor: The actor reference.
:type tags: ``str``
:param tags: Tags of the file.
:type feed_related_indicators: ``FeedRelatedIndicators``
:param feed_related_indicators: List of indicators that are associated with the file.
:type malware_family: ``str``
:param malware_family: The malware family associated with the File.
:type campaign: ``str``
:param campaign:
:type traffic_light_protocol: ``str``
:param traffic_light_protocol:
:type community_notes: ``CommunityNotes``
:param community_notes: Notes on the file that were given by the community.
:type publications: ``Publications``
:param publications: Publications on the file that was published.
:type threat_types: ``ThreatTypes``
:param threat_types: Threat types that are associated with the file.
:type imphash: ``str``
:param imphash: The Imphash hash of the file.
:type quarantined: ``bool``
:param quarantined: Is the file quarantined or not.
:type organization: ``str``
:param organization: The organization of the file.
:type associated_file_names: ``str``
:param associated_file_names: File names that are known as associated to the file.
:type behaviors: ``Behaviors``
:param behaviors: list of behaviors associated with the file.
:type relationships: ``list of EntityRelationship``
:param relationships: List of relationships of the indicator.
:type dbot_score: ``DBotScore``
:param dbot_score: If file has a score then create and set a DBotScore object
:type creation_date: ``str``
:param creation_date: The date the file was created.
:type description: ``str``
:param description: File description.
:type hashes: ``Hash``
:param hashes: List of hashes associated with the file.
:type stix_id: ``str``
:param stix_id: File assigned STIX ID.
:type organization_prevalence: ``int``
:param organization_prevalence: The number of times the indicator is detected in the organization.
:type global_prevalence: ``int``
:param global_prevalence: The number of times the indicator is detected across all organizations.
:type organization_first_seen: ``str``
:param organization_first_seen: ISO 8601 date time string; when the indicator was first seen in the organization.
:type organization_last_seen: ``str``
:param organization_last_seen: ISO 8601 date time string; the last time a specific organization encountered an indicator.
:type first_seen_by_source: ``str``
:param first_seen_by_source: ISO 8601 date time string; when the indicator was first seen by the source vendor.
:type last_seen_by_source: ``str``
:param last_seen_by_source: ISO 8601 date time string; when the indicator was last seen by the source vendor.
:rtype: ``None``
:return: None
"""
CONTEXT_PATH = 'File(val.MD5 && val.MD5 == obj.MD5 || val.SHA1 && val.SHA1 == obj.SHA1 || ' \
'val.SHA256 && val.SHA256 == obj.SHA256 || val.SHA512 && val.SHA512 == obj.SHA512 || ' \
'val.CRC32 && val.CRC32 == obj.CRC32 || val.CTPH && val.CTPH == obj.CTPH || ' \
'val.SSDeep && val.SSDeep == obj.SSDeep)'
def __init__(self, dbot_score, name=None, entry_id=None, size=None, md5=None, sha1=None, sha256=None,
sha512=None, ssdeep=None, extension=None, file_type=None, hostname=None, path=None, company=None,
product_name=None, digital_signature__publisher=None, signature=None, actor=None, tags=None,
feed_related_indicators=None, malware_family=None, imphash=None, quarantined=None, campaign=None,
associated_file_names=None, traffic_light_protocol=None, organization=None, community_notes=None,
publications=None, threat_types=None, behaviors=None, relationships=None,
creation_date=None, description=None, hashes=None, stix_id=None, organization_prevalence=None,
global_prevalence=None, organization_first_seen=None, organization_last_seen=None,
first_seen_by_source=None, last_seen_by_source=None):
# Main value of a file (Hashes)
self.md5 = md5
self.imphash = imphash
self.sha1 = sha1
self.sha256 = sha256
self.sha512 = sha512
self.ssdeep = ssdeep
self.hashes = hashes
# Core custom fields for File type
self.associated_file_names = associated_file_names
self.community_notes = community_notes
self.creation_date = creation_date
self.description = description
self.extension = extension
self.file_type = file_type
self.path = path
self.quarantined = quarantined
self.size = size
self.stix_id = stix_id
self.tags = tags
self.traffic_light_protocol = traffic_light_protocol
# Other custom fields
self.name = name
self.entry_id = entry_id
self.hostname = hostname
self.company = company
self.product_name = product_name
self.digital_signature__publisher = digital_signature__publisher
self.signature = signature
self.actor = actor
self.organization = organization
self.feed_related_indicators = feed_related_indicators
self.malware_family = malware_family
self.campaign = campaign
self.publications = publications
self.threat_types = threat_types
self.behaviors = behaviors
self.organization_prevalence = organization_prevalence
self.global_prevalence = global_prevalence
self.organization_first_seen = organization_first_seen
self.organization_last_seen = organization_last_seen
self.first_seen_by_source = first_seen_by_source
self.last_seen_by_source = last_seen_by_source
# XSOAR Fields
self.relationships = relationships
self.dbot_score = dbot_score
def to_context(self):
file_context = {'Hashes': []} # type: dict
if self.name:
file_context['Name'] = self.name
if self.hashes:
file_context['Hashes'] = self.create_context_table(self.hashes)
if self.entry_id:
file_context['EntryID'] = self.entry_id
if self.size:
file_context['Size'] = self.size
if self.md5:
file_context['MD5'] = self.md5
file_context['Hashes'].append({'type': 'MD5',
'value': self.md5})
if self.sha1:
file_context['SHA1'] = self.sha1
file_context['Hashes'].append({'type': 'SHA1',
'value': self.sha1})
if self.sha256:
file_context['SHA256'] = self.sha256
file_context['Hashes'].append({'type': 'SHA256',
'value': self.sha256})
if self.sha512:
file_context['SHA512'] = self.sha512
file_context['Hashes'].append({'type': 'SHA512',
'value': self.sha512})
if self.ssdeep:
file_context['SSDeep'] = self.ssdeep
file_context['Hashes'].append({'type': 'SSDeep',
'value': self.ssdeep})
if self.extension:
file_context['Extension'] = self.extension
if self.file_type:
file_context['Type'] = self.file_type
if self.hostname:
file_context['Hostname'] = self.hostname
if self.path:
file_context['Path'] = self.path
if self.company:
file_context['Company'] = self.company
if self.product_name:
file_context['ProductName'] = self.product_name
if self.digital_signature__publisher:
file_context['DigitalSignature'] = {
'Published': self.digital_signature__publisher
}
if self.signature:
file_context['Signature'] = self.signature.to_context()
if self.actor:
file_context['Actor'] = self.actor
if self.tags:
file_context['Tags'] = self.tags
if self.feed_related_indicators:
file_context['FeedRelatedIndicators'] = self.create_context_table(self.feed_related_indicators)
if self.malware_family:
file_context['MalwareFamily'] = self.malware_family
if self.campaign:
file_context['Campaign'] = self.campaign
if self.traffic_light_protocol:
file_context['TrafficLightProtocol'] = self.traffic_light_protocol
if self.community_notes:
file_context['CommunityNotes'] = self.create_context_table(self.community_notes)
if self.publications:
file_context['Publications'] = self.create_context_table(self.publications)
if self.threat_types:
file_context['ThreatTypes'] = self.create_context_table(self.threat_types)
if self.imphash:
file_context['Imphash'] = self.imphash
file_context['Hashes'].append({'type': 'Imphash',
'value': self.imphash})
if self.quarantined:
file_context['Quarantined'] = self.quarantined
if self.organization:
file_context['Organization'] = self.organization
if self.associated_file_names:
file_context['AssociatedFileNames'] = self.associated_file_names
if self.behaviors:
file_context['Behavior'] = self.create_context_table(self.behaviors)
if self.organization_prevalence is not None: # checking for `is not None` to allow `0`-value
file_context['OrganizationPrevalence'] = self.organization_prevalence
if self.global_prevalence is not None: # checking for `is not None` to allow `0`-value
file_context['GlobalPrevalence'] = self.global_prevalence
if self.organization_first_seen:
file_context['OrganizationFirstSeen'] = self.organization_first_seen
if self.organization_last_seen:
file_context['OrganizationLastSeen'] = self.organization_last_seen
if self.first_seen_by_source:
file_context['FirstSeenBySource'] = self.first_seen_by_source
if self.last_seen_by_source:
file_context['LastSeenBySource'] = self.last_seen_by_source
if self.dbot_score and self.dbot_score.score == Common.DBotScore.BAD:
file_context['Malicious'] = {
'Vendor': self.dbot_score.integration_name,
'Description': self.dbot_score.malicious_description
}
if self.relationships:
relationships_context = [relationship.to_context() for relationship in self.relationships if
relationship.to_context()]
file_context['Relationships'] = relationships_context
ret_value = {
Common.File.CONTEXT_PATH: file_context
}
if self.dbot_score:
ret_value.update(self.dbot_score.to_context())
return ret_value
def to_minimum_context(self):
file_context = {}
if self.md5:
file_context['MD5'] = self.md5
if self.sha1:
file_context['SHA1'] = self.sha1
if self.sha256:
file_context['SHA256'] = self.sha256
if self.sha512:
file_context['SHA512'] = self.sha512
ret_value = {
Common.File.CONTEXT_PATH: file_context
}
if self.dbot_score:
ret_value.update(self.dbot_score.to_context())
return ret_value
class CVE(Indicator):
"""
CVE indicator class - https://xsoar.pan.dev/docs/integrations/context-standards-mandatory#cve
:type id: ``str``
:param id: The ID of the CVE, for example: "CVE-2015-1653".
:type cvss: ``str``
:param cvss: The CVSS of the CVE, for example: "10.0".
:type published: ``str``
:param published: The timestamp of when the CVE was published.
:type modified: ``str``
:param modified: The timestamp of when the CVE was last modified.
:type description: ``str``
:param description: A description of the CVE.
:type relationships: ``list of EntityRelationship``
:param relationships: List of relationships of the indicator.
:type stix_id: ``str``
:param stix_id: CVE sitx id.
:type cvss_version: ``str``
:param cvss_version: The CVE CVSS version used.
:type cvss_score: ``str``
:param cvss_score: The CVE CVSS Score.
:type cvss_vector: ``str``
:param cvss_vector: CVE full cvss vector.
:type cvss_table: ``str``
:param cvss_table: CVE CVSS Table used to fill the different parts of the vector.
:type community_notes: ``str``
:param community_notes: Community notes about the CVE.
:type tags: ``str``
:param tags: Tags attached to the CVE.
:type traffic_light_protocol: ``str``
:param traffic_light_protocol: The CVE tlp color.
:type publications: ``str``
:param publications: Unique system-assigned ID of the vulnerability evaluation logic
:type dbot_score: ``DBotScore``
:param dbot_score: If file has a score then create and set a DBotScore object
:type vulnerable_products: ``CPE``
:param vulnerable_products: A list of CPE objects
:type vulnerable_configurations: ``CPE``
:param vulnerable_configurations: A list of CPE objects
:return: None
:rtype: ``None``
"""
CONTEXT_PATH = 'CVE(val.ID && val.ID == obj.ID)'
def __init__(self, id, cvss, published, modified, description, relationships=None, stix_id=None,
cvss_version=None, cvss_score=None, cvss_vector=None, cvss_table=None, community_notes=None,
tags=None, traffic_light_protocol=None, dbot_score=None, publications=None,
vulnerable_products=None, vulnerable_configurations=None):
# Main indicator value
self.id = id
# Core custom fields
self.community_notes = community_notes
self.cvss = cvss
self.cvss_version = cvss_version
self.cvss_score = cvss_score
self.cvss_vector = cvss_vector
self.cvss_table = cvss_table
self.description = description
self.modified = modified
self.published = published
self.stix_id = stix_id
self.tags = tags
self.traffic_light_protocol = traffic_light_protocol
self.publications = publications
# XSOAR Fields
self.relationships = relationships
self.dbot_score = dbot_score if dbot_score else Common.DBotScore(indicator=id,
indicator_type=DBotScoreType.CVE,
integration_name=None,
score=Common.DBotScore.NONE)
# Core custom fields for CVE type
self.vulnerable_products = vulnerable_products
self.vulnerable_configurations = vulnerable_configurations
def to_context(self):
cve_context = {
'ID': self.id,
'CVSS': {},
}
if self.cvss:
cve_context['CVSS']['Score'] = self.cvss
elif self.cvss_score:
cve_context['CVSS']['Score'] = self.cvss_score
if self.cvss_version:
cve_context['CVSS']['Version'] = self.cvss_version
if self.cvss_vector:
cve_context['CVSS']['Vector'] = self.cvss_vector
if self.cvss_table:
cve_context['CVSS']['Table'] = self.cvss_table
if self.published:
cve_context['Published'] = self.published
if self.modified:
cve_context['Modified'] = self.modified
if self.description:
cve_context['Description'] = self.description
if self.stix_id:
cve_context['STIXID'] = self.stix_id
if self.relationships:
relationships_context = [relationship.to_context() for relationship in self.relationships if
relationship.to_context()]
cve_context['Relationships'] = relationships_context
if self.community_notes:
cve_context['CommunityNotes'] = self.create_context_table(self.community_notes)
if self.tags:
cve_context['Tags'] = self.tags
if self.traffic_light_protocol:
cve_context['TrafficLightProtocol'] = self.traffic_light_protocol
ret_value = {
Common.CVE.CONTEXT_PATH: cve_context
}
if self.dbot_score:
ret_value.update(self.dbot_score.to_context())
if self.publications:
cve_context['Publications'] = self.create_context_table(self.publications)
if self.vulnerable_products:
cve_context['VulnerableProducts'] = self.create_context_table(self.vulnerable_products)
if self.vulnerable_configurations:
cve_context['VulnerableConfigurations'] = self.create_context_table(self.vulnerable_configurations)
return ret_value
def to_minimum_context(self):
cve_context = {
'ID': self.id
}
ret_value = {
Common.CVE.CONTEXT_PATH: cve_context
}
if self.dbot_score:
ret_value.update(self.dbot_score.to_context())
return ret_value
class EMAIL(Indicator):
"""
EMAIL indicator class
:type address ``str``
:param address: The email's address.
:type domain: ``str``
:param domain: The domain of the Email.
:type blocked: ``bool``
:param blocked: Whether the email address is blocked.
:type relationships: ``list of EntityRelationship``
:param relationships: List of relationships of the indicator.
:type description: ``str``
:param description: Description of the email address.
:type internal: ``bool``
:param internal: Is the email an internal address.
:type stix_id: ``str``
:param stix_id: The email assigned STIX ID.
:type tags: ``str``
:param tags: tags relevant to the email.
:type traffic_light_protocol: ``str``
:param traffic_light_protocol: The email address tlp color.
:return: None
:rtype: ``None``
"""
CONTEXT_PATH = 'Account(val.Email.Address && val.Email.Address == obj.Email.Address)'
def __init__(self, address, dbot_score, domain=None, blocked=None, relationships=None, description=None,
internal=None, stix_id=None, tags=None, traffic_light_protocol=None):
# type (str, str, bool) -> None
# Main indicator value
self.address = address
# Core custom fields
self.blocked = blocked
self.description = description
self.internal = internal
self.stix_id = stix_id
self.tags = tags
self.traffic_light_protocol = traffic_light_protocol
# Deprecated
self.domain = domain
# XSOAR Fields
self.dbot_score = dbot_score
self.relationships = relationships
def to_context(self):
email_context = {
'Email': {'Address': self.address}
}
if self.blocked:
email_context['Blocked'] = self.blocked
if self.domain:
email_context['Domain'] = self.domain
if self.description:
email_context['Description'] = self.description
if self.internal:
email_context['Internal'] = self.internal
if self.stix_id:
email_context['STIXID'] = self.stix_id
if self.tags:
email_context['Tags'] = self.tags
if self.traffic_light_protocol:
email_context['TrafficLightProtocol'] = self.traffic_light_protocol
if self.relationships:
relationships_context = [relationship.to_context() for relationship in self.relationships if
relationship.to_context()]
email_context['Relationships'] = relationships_context
ret_value = {
Common.EMAIL.CONTEXT_PATH: email_context
}
if self.dbot_score:
ret_value.update(self.dbot_score.to_context())
return ret_value
def to_minimum_context(self):
email_context = {
'Email': {'Address': self.address}
}
ret_value = {
Common.EMAIL.CONTEXT_PATH: email_context
}
if self.dbot_score:
ret_value.update(self.dbot_score.to_context())
return ret_value
class URL(Indicator):
"""
URL indicator - https://xsoar.pan.dev/docs/integrations/context-standards-mandatory#url
:type url: ``str``
:param url: The URL
:type detection_engines: ``int``
:param detection_engines: The total number of engines that checked the indicator.
:type positive_detections: ``int``
:param positive_detections: The number of engines that positively detected the indicator as malicious.
:type category: ``str``
:param category: The category associated with the indicator.
:type feed_related_indicators: ``FeedRelatedIndicators``
:param feed_related_indicators: List of indicators that are associated with the URL.
:type malware_family: ``str``
:param malware_family: The malware family associated with the URL.
:type tags: ``str``
:param tags: Tags of the URL.
:type port: ``str``
:param port: Ports that are associated with the URL.
:type internal: ``bool``
:param internal: Whether or not the URL is internal or external.
:type campaign: ``str``
:param campaign: The campaign associated with the URL.
:type traffic_light_protocol: ``str``
:param traffic_light_protocol: The Traffic Light Protocol (TLP) color that is suitable for the URL.
:type threat_types: ``ThreatTypes``
:param threat_types: Threat types that are associated with the file.
:type asn: ``str``
:param asn: The autonomous system name for the URL, for example: 'AS8948'.
:type as_owner: ``str``
:param as_owner: The autonomous system owner of the URL.
:type geo_country: ``str``
:param geo_country: The country in which the URL is located.
:type organization: ``str``
:param organization: The organization of the URL.
:type community_notes: ``CommunityNotes``
:param community_notes: List of notes on the URL that were given by the community.
:type publications: ``Publications``
:param publications: List of publications on the URL that was published.
:type relationships: ``list of EntityRelationship``
:param relationships: List of relationships of the indicator.
:type dbot_score: ``DBotScore``
:param dbot_score: If URL has reputation then create DBotScore object
:type blocked: ``bool``
:param blocked: Is the URL blocked.
:type certificates: ``Certificates``
:param certificates: A list of certificates associated with the url.
:type description: ``str``
:param description: A description of the URL.
:type stix_id: ``str``
:param stix_id: The URL STIX ID.
:type organization_prevalence: ``int``
:param organization_prevalence: The number of times the indicator is detected in the organization.
:type global_prevalence: ``int``
:param global_prevalence: The number of times the indicator is detected across all organizations.
:type organization_first_seen: ``str``
:param organization_first_seen: ISO 8601 date time string; when the indicator was first seen in the organization.
:type organization_last_seen: ``str``
:param organization_last_seen: ISO 8601 date time string; the last time a specific organization encountered an indicator.
:type first_seen_by_source: ``str``
:param first_seen_by_source: ISO 8601 date time string; when the indicator was first seen by the source vendor.
:type last_seen_by_source: ``str``
:param last_seen_by_source: ISO 8601 date time string; when the indicator was last seen by the source vendor.
:return: None
:rtype: ``None``
"""
CONTEXT_PATH = 'URL(val.Data && val.Data == obj.Data)'
def __init__(self, url, dbot_score, detection_engines=None, positive_detections=None, category=None,
feed_related_indicators=None, tags=None, malware_family=None, port=None, internal=None,
campaign=None, traffic_light_protocol=None, threat_types=None, asn=None, as_owner=None,
geo_country=None, organization=None, community_notes=None, publications=None, relationships=None,
blocked=None, certificates=None, description=None, stix_id=None, organization_prevalence=None,
global_prevalence=None, organization_first_seen=None, organization_last_seen=None,
first_seen_by_source=None, last_seen_by_source=None):
# Main indicator value
self.url = url
# Core custom fields
self.blocked = blocked
self.certificates = certificates
self.community_notes = community_notes
self.description = description
self.internal = internal
self.stix_id = stix_id
self.tags = tags
self.traffic_light_protocol = traffic_light_protocol
# Additional custom fields
self.detection_engines = detection_engines
self.positive_detections = positive_detections
self.category = category
self.feed_related_indicators = feed_related_indicators
self.malware_family = malware_family
self.port = port
self.campaign = campaign
self.threat_types = threat_types
self.asn = asn
self.as_owner = as_owner
self.geo_country = geo_country
self.organization = organization
self.publications = publications
self.organization_prevalence = organization_prevalence
self.global_prevalence = global_prevalence
self.organization_first_seen = organization_first_seen
self.organization_last_seen = organization_last_seen
self.first_seen_by_source = first_seen_by_source
self.last_seen_by_source = last_seen_by_source
# XSOAR Fields
self.relationships = relationships
self.dbot_score = dbot_score
def to_context(self):
url_context = {
'Data': self.url
}
if self.blocked:
url_context['Blocked'] = self.blocked
if self.certificates:
url_context['Certificates'] = self.create_context_table(self.certificates)
if self.description:
url_context['Description'] = self.description
if self.stix_id:
url_context['STIXID'] = self.stix_id
if self.detection_engines is not None:
url_context['DetectionEngines'] = self.detection_engines
if self.positive_detections is not None:
url_context['PositiveDetections'] = self.positive_detections
if self.category:
url_context['Category'] = self.category
if self.feed_related_indicators:
url_context['FeedRelatedIndicators'] = self.create_context_table(self.feed_related_indicators)
if self.tags:
url_context['Tags'] = self.tags
if self.malware_family:
url_context['MalwareFamily'] = self.malware_family
if self.port:
url_context['Port'] = self.port
if self.internal:
url_context['Internal'] = self.internal
if self.campaign:
url_context['Campaign'] = self.campaign
if self.traffic_light_protocol:
url_context['TrafficLightProtocol'] = self.traffic_light_protocol
if self.threat_types:
url_context['ThreatTypes'] = self.create_context_table(self.threat_types)
if self.asn:
url_context['ASN'] = self.asn
if self.as_owner:
url_context['ASOwner'] = self.as_owner
if self.geo_country:
url_context['Geo'] = {'Country': self.geo_country}
if self.organization:
url_context['Organization'] = self.organization
if self.community_notes:
url_context['CommunityNotes'] = self.create_context_table(self.community_notes)
if self.publications:
url_context['Publications'] = self.create_context_table(self.publications)
if self.organization_prevalence is not None: # checking for `is not None` to allow `0`-value
url_context['OrganizationPrevalence'] = self.organization_prevalence
if self.global_prevalence is not None: # checking for `is not None` to allow `0`-value
url_context['GlobalPrevalence'] = self.global_prevalence
if self.organization_first_seen:
url_context['OrganizationFirstSeen'] = self.organization_first_seen
if self.organization_last_seen:
url_context['OrganizationLastSeen'] = self.organization_last_seen
if self.first_seen_by_source:
url_context['FirstSeenBySource'] = self.first_seen_by_source
if self.last_seen_by_source:
url_context['LastSeenBySource'] = self.last_seen_by_source
if self.dbot_score and self.dbot_score.score == Common.DBotScore.BAD:
url_context['Malicious'] = {
'Vendor': self.dbot_score.integration_name,
'Description': self.dbot_score.malicious_description
}
if self.relationships:
relationships_context = [relationship.to_context() for relationship in self.relationships if
relationship.to_context()]
url_context['Relationships'] = relationships_context
ret_value = {
Common.URL.CONTEXT_PATH: url_context
}
if self.dbot_score:
ret_value.update(self.dbot_score.to_context())
return ret_value
def to_minimum_context(self):
url_context = {
'Data': self.url
}
ret_value = {
Common.URL.CONTEXT_PATH: url_context
}
if self.dbot_score:
ret_value.update(self.dbot_score.to_context())
return ret_value
class Domain(Indicator):
""" ignore docstring
Domain indicator - https://xsoar.pan.dev/docs/integrations/context-standards-mandatory#domain
:type whois_records: ``WhoisRecord``
:param whois_records: List of whois records
:type description: ``str``
:param description: A description of the Domain.
:type stix_id: ``str``
:param stix_id: The domain STIX ID.
:type blocked: ``bool``
:param blocked: Is the domain blocked.
:type certificates: ``Certificates``
:param certificates: The certificates belonging to the domain.
:type dns_records: ``DNSRecord``
:param dns_records: A list of DNS records for the domain.
:type organization_prevalence: ``int``
:param organization_prevalence: The number of times the indicator is detected in the organization.
:type global_prevalence: ``int``
:param global_prevalence: The number of times the indicator is detected across all organizations.
:type organization_first_seen: ``str``
:param organization_first_seen: ISO 8601 date time string; when the indicator was first seen in the organization.
:type organization_last_seen: ``str``
:param organization_last_seen: ISO 8601 date time string; the last time a specific organization encountered an indicator.
:type first_seen_by_source: ``str``
:param first_seen_by_source: ISO 8601 date time string; when the indicator was first seen by the source vendor.
:type last_seen_by_source: ``str``
:param last_seen_by_source: ISO 8601 date time string; when the indicator was last seen by the source vendor.
"""
CONTEXT_PATH = 'Domain(val.Name && val.Name == obj.Name)'
def __init__(self, domain, dbot_score, dns=None, detection_engines=None, positive_detections=None,
organization=None, sub_domains=None, creation_date=None, updated_date=None, expiration_date=None,
domain_status=None, name_servers=None, feed_related_indicators=None, malware_family=None,
registrar_name=None, registrar_abuse_email=None, registrar_abuse_phone=None,
registrant_name=None, registrant_email=None, registrant_phone=None, registrant_country=None,
admin_name=None, admin_email=None, admin_phone=None, admin_country=None, tags=None,
domain_idn_name=None, port=None,
internal=None, category=None, campaign=None, traffic_light_protocol=None, threat_types=None,
community_notes=None, publications=None, geo_location=None, geo_country=None, geo_description=None,
tech_country=None, tech_name=None, tech_email=None, tech_organization=None, billing=None,
whois_records=None, relationships=None, description=None, stix_id=None, blocked=None,
certificates=None, dns_records=None, rank=None, organization_prevalence=None,
global_prevalence=None, organization_first_seen=None, organization_last_seen=None,
first_seen_by_source=None, last_seen_by_source=None):
# Main indicator value
self.domain = domain
# Core custom fields
self.blocked = blocked
self.certificates = certificates
self.community_notes = community_notes
self.creation_date = creation_date
self.description = description
self.dns_records = dns_records
self.expiration_date = expiration_date
self.internal = internal
self.stix_id = stix_id
self.tags = tags
self.traffic_light_protocol = traffic_light_protocol
self.whois_records = whois_records
# Additional custom fields
self.dns = dns
self.detection_engines = detection_engines
self.positive_detections = positive_detections
self.organization = organization
self.sub_domains = sub_domains
self.updated_date = updated_date
self.rank = rank
# Whois related records - Registrar
self.registrar_name = registrar_name
self.registrar_abuse_email = registrar_abuse_email
self.registrar_abuse_phone = registrar_abuse_phone
self.registrant_name = registrant_name
self.registrant_email = registrant_email
self.registrant_phone = registrant_phone
self.registrant_country = registrant_country
self.admin_name = admin_name
self.admin_email = admin_email
self.admin_phone = admin_phone
self.admin_country = admin_country
self.tech_country = tech_country
self.tech_name = tech_name
self.tech_organization = tech_organization
self.tech_email = tech_email
self.billing = billing
# Additional custom records (non core)
self.domain_status = domain_status
self.name_servers = name_servers
self.feed_related_indicators = feed_related_indicators
self.malware_family = malware_family
self.domain_idn_name = domain_idn_name
self.port = port
self.category = category
self.campaign = campaign
self.threat_types = threat_types
self.publications = publications
self.geo_location = geo_location
self.geo_country = geo_country
self.geo_description = geo_description
self.organization_prevalence = organization_prevalence
self.global_prevalence = global_prevalence
self.organization_first_seen = organization_first_seen
self.organization_last_seen = organization_last_seen
self.first_seen_by_source = first_seen_by_source
self.last_seen_by_source = last_seen_by_source
# XSOAR Fields
self.relationships = relationships
self.dbot_score = dbot_score
def to_context(self):
domain_context = {
'Name': self.domain
}
whois_context = {}
if self.dns:
domain_context['DNS'] = self.dns
if self.detection_engines is not None:
domain_context['DetectionEngines'] = self.detection_engines
if self.positive_detections is not None:
domain_context['PositiveDetections'] = self.positive_detections
if self.registrar_name or self.registrar_abuse_email or self.registrar_abuse_phone:
domain_context['Registrar'] = {
'Name': self.registrar_name,
'AbuseEmail': self.registrar_abuse_email,
'AbusePhone': self.registrar_abuse_phone
}
whois_context['Registrar'] = domain_context['Registrar']
if self.registrant_name or self.registrant_phone or self.registrant_email or self.registrant_country:
domain_context['Registrant'] = {
'Name': self.registrant_name,
'Email': self.registrant_email,
'Phone': self.registrant_phone,
'Country': self.registrant_country
}
whois_context['Registrant'] = domain_context['Registrant']
if self.admin_name or self.admin_email or self.admin_phone or self.admin_country:
domain_context['Admin'] = {
'Name': self.admin_name,
'Email': self.admin_email,
'Phone': self.admin_phone,
'Country': self.admin_country
}
whois_context['Admin'] = domain_context['Admin']
if self.organization:
domain_context['Organization'] = self.organization
if self.sub_domains:
domain_context['Subdomains'] = self.sub_domains
if self.domain_status:
domain_context['DomainStatus'] = self.domain_status
whois_context['DomainStatus'] = domain_context['DomainStatus']
if self.creation_date:
domain_context['CreationDate'] = self.creation_date
whois_context['CreationDate'] = domain_context['CreationDate']
if self.updated_date:
domain_context['UpdatedDate'] = self.updated_date
whois_context['UpdatedDate'] = domain_context['UpdatedDate']
if self.expiration_date:
domain_context['ExpirationDate'] = self.expiration_date
whois_context['ExpirationDate'] = domain_context['ExpirationDate']
if self.name_servers:
domain_context['NameServers'] = self.name_servers
whois_context['NameServers'] = domain_context['NameServers']
if self.tags:
domain_context['Tags'] = self.tags
if self.feed_related_indicators:
domain_context['FeedRelatedIndicators'] = self.create_context_table(self.feed_related_indicators)
if self.whois_records:
domain_context['WhoisRecords'] = self.create_context_table(self.whois_records)
if self.malware_family:
domain_context['MalwareFamily'] = self.malware_family
if self.organization_prevalence is not None: # checking for `is not None` to allow `0`-value
domain_context['OrganizationPrevalence'] = self.organization_prevalence
if self.global_prevalence is not None: # checking for `is not None` to allow `0`-value
domain_context['GlobalPrevalence'] = self.global_prevalence
if self.organization_first_seen:
domain_context['OrganizationFirstSeen'] = self.organization_first_seen
if self.organization_last_seen:
domain_context['OrganizationLastSeen'] = self.organization_last_seen
if self.first_seen_by_source:
domain_context['FirstSeenBySource'] = self.first_seen_by_source
if self.last_seen_by_source:
domain_context['LastSeenBySource'] = self.last_seen_by_source
if self.dbot_score and self.dbot_score.score == Common.DBotScore.BAD:
domain_context['Malicious'] = {
'Vendor': self.dbot_score.integration_name,
'Description': self.dbot_score.malicious_description
}
if self.domain_idn_name:
domain_context['DomainIDNName'] = self.domain_idn_name
if self.port:
domain_context['Port'] = self.port
if self.internal:
domain_context['Internal'] = self.internal
if self.category:
domain_context['Category'] = self.category
if self.campaign:
domain_context['Campaign'] = self.campaign
if self.traffic_light_protocol:
domain_context['TrafficLightProtocol'] = self.traffic_light_protocol
if self.threat_types:
domain_context['ThreatTypes'] = self.create_context_table(self.threat_types)
if self.community_notes:
domain_context['CommunityNotes'] = self.create_context_table(self.community_notes)
if self.publications:
domain_context['Publications'] = self.create_context_table(self.publications)
if self.geo_location or self.geo_country or self.geo_description:
domain_context['Geo'] = {}
if self.geo_location:
domain_context['Geo']['Location'] = self.geo_location
if self.geo_country:
domain_context['Geo']['Country'] = self.geo_country
if self.geo_description:
domain_context['Geo']['Description'] = self.geo_description
if self.tech_country or self.tech_name or self.tech_organization or self.tech_email:
domain_context['Tech'] = {}
if self.tech_country:
domain_context['Tech']['Country'] = self.tech_country
if self.tech_name:
domain_context['Tech']['Name'] = self.tech_name
if self.tech_organization:
domain_context['Tech']['Organization'] = self.tech_organization
if self.tech_email:
domain_context['Tech']['Email'] = self.tech_email
if self.billing:
domain_context['Billing'] = self.billing
if whois_context:
domain_context['WHOIS'] = whois_context
if self.relationships:
relationships_context = [relationship.to_context() for relationship in self.relationships if
relationship.to_context()]
domain_context['Relationships'] = relationships_context
ret_value = {
Common.Domain.CONTEXT_PATH: domain_context
}
if self.dbot_score:
ret_value.update(self.dbot_score.to_context())
if self.dns_records:
domain_context['DNSRecords'] = self.create_context_table(self.dns_records)
if self.stix_id:
domain_context['STIXID'] = self.stix_id
if self.description:
domain_context['Description'] = self.description
if self.stix_id:
domain_context['Blocked'] = self.blocked
if self.certificates:
domain_context['Certificates'] = self.create_context_table(self.certificates)
if self.rank:
domain_context['Rank'] = self.create_context_table(self.rank)
return ret_value
def to_minimum_context(self):
domain_context = {
'Name': self.domain
}
ret_value = {
Common.Domain.CONTEXT_PATH: domain_context
}
if self.dbot_score:
ret_value.update(self.dbot_score.to_context())
return ret_value
class Endpoint(Indicator):
""" ignore docstring
Endpoint indicator - https://xsoar.pan.dev/docs/integrations/context-standards-mandatory#endpoint
"""
# Compare by both ID and Vendor if both exist, otherwise just by ID.
CONTEXT_PATH = 'Endpoint(val.ID && val.ID == obj.ID && val.Vendor == obj.Vendor)'
def __init__(self, id, hostname=None, ip_address=None, domain=None, mac_address=None,
os=None, os_version=None, dhcp_server=None, bios_version=None, model=None,
memory=None, processors=None, processor=None, relationships=None, vendor=None, status=None,
is_isolated=None):
self.id = id
self.hostname = hostname
self.ip_address = ip_address
self.domain = domain
self.mac_address = mac_address
self.os = os
self.os_version = os_version
self.dhcp_server = dhcp_server
self.bios_version = bios_version
self.model = model
self.memory = memory
self.processors = processors
self.processor = processor
self.vendor = vendor
self.status = status
self.is_isolated = is_isolated
self.relationships = relationships
def to_context(self):
endpoint_context = {
'ID': self.id
}
if self.hostname:
endpoint_context['Hostname'] = self.hostname
if self.ip_address:
endpoint_context['IPAddress'] = self.ip_address
if self.domain:
endpoint_context['Domain'] = self.domain
if self.mac_address:
endpoint_context['MACAddress'] = self.mac_address
if self.os:
endpoint_context['OS'] = self.os
if self.os_version:
endpoint_context['OSVersion'] = self.os_version
if self.dhcp_server:
endpoint_context['DHCPServer'] = self.dhcp_server
if self.bios_version:
endpoint_context['BIOSVersion'] = self.bios_version
if self.model:
endpoint_context['Model'] = self.model
if self.memory:
endpoint_context['Memory'] = self.memory
if self.processors:
endpoint_context['Processors'] = self.processors
if self.processor:
endpoint_context['Processor'] = self.processor
if self.relationships:
relationships_context = [relationship.to_context() for relationship in self.relationships if
relationship.to_context()]
endpoint_context['Relationships'] = relationships_context
if self.vendor:
endpoint_context['Vendor'] = self.vendor
if self.status:
if self.status not in ENDPOINT_STATUS_OPTIONS:
raise ValueError('Status does not have a valid value such as: Online or Offline')
endpoint_context['Status'] = self.status
if self.is_isolated:
if self.is_isolated not in ENDPOINT_ISISOLATED_OPTIONS:
raise ValueError('Is Isolated does not have a valid value such as: Yes, No, Pending'
' isolation or Pending unisolation')
endpoint_context['IsIsolated'] = self.is_isolated
ret_value = {
Common.Endpoint.CONTEXT_PATH: endpoint_context
}
return ret_value
def to_minimum_context(self):
endpoint_context = {
'ID': self.id
}
ret_value = {
Common.Endpoint.CONTEXT_PATH: endpoint_context
}
return ret_value
class Account(Indicator):
"""
Account indicator - https://xsoar.pan.dev/docs/integrations/context-standards-recommended#account
:type blocked: ``boolean``
:param blocked: Is the indicator blocked.
:type community_notes: ``CommunityNotes``
:param community_notes: Notes on the Account that were given by the community.
:type dbot_score: ``DBotScore``
:param dbot_score: If account has reputation then create DBotScore object
:type creation_date: ``str``
:param creation_date: The date the account was created
:type description: ``str``
:param description: Description of the account
:type stix_id: ``str``
:param stix_id: STIX ID for the account
:type tags: ``str``
:param tags: List of tags related to the account.
:type traffic_light_protocol: ``str``
:param traffic_light_protocol: The account indicator tlp color
:type user_id: ``str``
:param user_id: The account associated user id.
:return: None
:rtype: ``None``
"""
CONTEXT_PATH = 'Account(val.id && val.id == obj.id)'
def __init__(self, id=None, type=None, username=None, display_name=None, groups=None,
domain=None, email_address=None, telephone_number=None, office=None, job_title=None,
department=None, country=None, state=None, city=None, street=None, is_enabled=None,
dbot_score=None, relationships=None, blocked=None, community_notes=None, creation_date=None,
description=None, stix_id=None, tags=None, traffic_light_protocol=None, user_id=None,
manager_email=None, manager_display_name=None, risk_level=None, **kwargs):
self.id = id
self.type = type
self.blocked = blocked
self.community_notes = community_notes
self.creation_date = creation_date
self.description = description
self.stix_id = stix_id
self.username = username
self.display_name = display_name
self.groups = groups
self.domain = domain
self.email_address = email_address
self.telephone_number = telephone_number
self.office = office
self.job_title = job_title
self.department = department
self.country = country
self.state = state
self.city = city
self.street = street
self.is_enabled = is_enabled
self.tags = tags
self.traffic_light_protocol = traffic_light_protocol
self.user_id = user_id
self.relationships = relationships
self.manager_email_address = manager_email
self.manager_display_name = manager_display_name
self.risk_level = risk_level
self.kwargs = kwargs
if dbot_score and not isinstance(dbot_score, Common.DBotScore):
raise ValueError('dbot_score must be of type DBotScore')
self.dbot_score = dbot_score
def to_context(self):
account_context = {}
if self.id:
account_context['ID'] = self.id
if self.type:
account_context['Type'] = self.type
if self.blocked:
account_context['Blocked'] = self.blocked
if self.creation_date:
account_context['CreationDate'] = self.creation_date
irrelevent = ['CONTEXT_PATH', 'to_context', 'dbot_score', 'id', 'create_context_table', 'kwargs', 'to_minimum_context']
details = [detail for detail in dir(self) if not detail.startswith('__') and detail not in irrelevent]
for detail in details:
if self.__getattribute__(detail) is not None:
if detail == 'email_address':
account_context['Email'] = {
'Address': self.email_address
}
elif detail in ('manager_email_address', 'manager_display_name'):
if 'Manager' not in account_context:
account_context['Manager'] = {}
if detail == 'manager_email_address':
account_context['Manager']['Email'] = self.manager_email_address
elif detail == 'manager_display_name':
account_context['Manager']['DisplayName'] = self.manager_display_name
else:
Detail = camelize_string(detail, '_')
account_context[Detail] = self.__getattribute__(detail)
if self.dbot_score and self.dbot_score.score == Common.DBotScore.BAD:
account_context['Malicious'] = {
'Vendor': self.dbot_score.integration_name,
'Description': self.dbot_score.malicious_description
}
if self.relationships:
relationships_context = [relationship.to_context() for relationship in self.relationships if
relationship.to_context()]
account_context['Relationships'] = relationships_context
if self.community_notes:
account_context['CommunityNotes'] = self.create_context_table(self.community_notes)
if self.kwargs:
for key, value in self.kwargs.items():
if key not in account_context:
account_context[key] = value
else:
demisto.debug(
'Skipping the addition of the "{key}" key to the account context as it already exists.'.format(
key=key)
)
ret_value = {
Common.Account.CONTEXT_PATH: account_context
}
if self.dbot_score:
ret_value.update(self.dbot_score.to_context())
return ret_value
def to_minimum_context(self):
account_context = {}
if self.id:
account_context['ID'] = self.id
ret_value = {
Common.Account.CONTEXT_PATH: account_context
}
if self.dbot_score:
ret_value.update(self.dbot_score.to_context())
return ret_value
class Cryptocurrency(Indicator):
"""
Cryptocurrency indicator - https://xsoar.pan.dev/docs/integrations/context-standards-mandatory#cryptocurrency
:type address: ``str``
:param address: The Cryptocurrency address
:type address_type: ``str``
:param address_type: The Cryptocurrency type - e.g. `bitcoin`.
:type dbot_score: ``DBotScore``
:param dbot_score: If the address has reputation then create DBotScore object.
:return: None
:rtype: ``None``
"""
CONTEXT_PATH = 'Cryptocurrency(val.Address && val.Address == obj.Address)'
def __init__(self, address, address_type, dbot_score):
self.address = address
self.address_type = address_type
self.dbot_score = dbot_score
def to_context(self):
crypto_context = {
'Address': self.address,
'AddressType': self.address_type
}
if self.dbot_score and self.dbot_score.score == Common.DBotScore.BAD:
crypto_context['Malicious'] = {
'Vendor': self.dbot_score.integration_name,
'Description': self.dbot_score.malicious_description
}
ret_value = {
Common.Cryptocurrency.CONTEXT_PATH: crypto_context
}
if self.dbot_score:
ret_value.update(self.dbot_score.to_context())
return ret_value
def to_minimum_context(self):
crypto_context = {
'Address': self.address,
'AddressType': self.address_type
}
ret_value = {
Common.Cryptocurrency.CONTEXT_PATH: crypto_context
}
if self.dbot_score:
ret_value.update(self.dbot_score.to_context())
return ret_value
class AttackPattern(Indicator):
"""
Attack Pattern indicator
:type stix_id: ``str``
:param stix_id: The Attack Pattern STIX ID
:type kill_chain_phases: ``str``
:param kill_chain_phases: The Attack Pattern kill chain phases.
:type first_seen_by_source: ``str``
:param first_seen_by_source: The Attack Pattern first seen by source
:type description: ``str``
:param description: The Attack Pattern description
:type operating_system_refs: ``str``
:param operating_system_refs: The operating system refs of the Attack Pattern.
:type publications: ``str``
:param publications: The Attack Pattern publications
:type mitre_id: ``str``
:param mitre_id: The Attack Pattern kill mitre id.
:type tags: ``str``
:param tags: The Attack Pattern kill tags.
:type dbot_score: ``DBotScore``
:param dbot_score: If the address has reputation then create DBotScore object.
:type traffic_light_protocol: ``str``
:param traffic_light_protocol: The Traffic Light Protocol (TLP) color that is suitable for the AP.
:type community_notes: ``CommunityNotes``
:param community_notes: A list of community notes for the AP.
:type external_references: ``ExternalReference``
:param external_references: A list of id's and description of the AP via external refs.
:type value: ``str``
:param value: The Attack Pattern value (name) - example: "Plist File Modification"
:return: None
:rtype: ``None``
"""
CONTEXT_PATH = 'AttackPattern(val.value && val.value == obj.value)'
def __init__(self, stix_id, kill_chain_phases=None, first_seen_by_source=None, description=None,
operating_system_refs=None, publications=None, mitre_id=None, tags=None,
traffic_light_protocol=None, dbot_score=None, community_notes=None, external_references=None, value=None):
self.community_notes = community_notes
self.description = description
self.external_references = external_references
self.first_seen_by_source = first_seen_by_source
self.kill_chain_phases = kill_chain_phases
self.mitre_id = mitre_id
self.operating_system_refs = operating_system_refs
self.publications = publications
self.stix_id = stix_id
self.tags = tags
self.traffic_light_protocol = traffic_light_protocol
self.value = value
self.dbot_score = dbot_score
def to_context(self):
attack_pattern_context = {
'STIXID': self.stix_id,
"KillChainPhases": self.kill_chain_phases,
"FirstSeenBySource": self.first_seen_by_source,
'OperatingSystemRefs': self.operating_system_refs,
"Publications": self.publications,
"MITREID": self.mitre_id,
"Value": self.value,
"Tags": self.tags,
"Description": self.description
}
if self.external_references:
attack_pattern_context['ExternalReferences'] = self.create_context_table(self.external_references)
if self.traffic_light_protocol:
attack_pattern_context['TrafficLightProtocol'] = self.traffic_light_protocol
if self.dbot_score and self.dbot_score.score == Common.DBotScore.BAD:
attack_pattern_context['Malicious'] = {
'Vendor': self.dbot_score.integration_name,
'Description': self.dbot_score.malicious_description
}
ret_value = {
Common.AttackPattern.CONTEXT_PATH: attack_pattern_context
}
if self.dbot_score:
ret_value.update(self.dbot_score.to_context())
return ret_value
def to_minimum_context(self):
attack_pattern_context = {
'STIXID': self.stix_id,
'Value': self.value
}
ret_value = {
Common.AttackPattern.CONTEXT_PATH: attack_pattern_context
}
if self.dbot_score:
ret_value.update(self.dbot_score.to_context())
return ret_value
class CertificatePublicKey(object):
"""
CertificatePublicKey class
Defines an X509 PublicKey used in Common.Certificate
:type algorithm: ``str``
:param algorithm: The encryption algorithm: DSA, RSA, EC or UNKNOWN (Common.CertificatePublicKey.Algorithm enum)
:type length: ``int``
:param length: The length of the public key
:type publickey: ``Optional[str]``
:param publickey: publickey
:type p: ``Optional[str]``
:param p: P parameter used in DSA algorithm
:type q: ``Optional[str]``
:param q: Q parameter used in DSA algorithm
:type g: ``Optional[str]``
:param g: G parameter used in DSA algorithm
:type modulus: ``Optional[str]``
:param modulus: modulus parameter used in RSA algorithm
:type modulus: ``Optional[int]``
:param modulus: exponent parameter used in RSA algorithm
:type x: ``Optional[str]``
:param x: X parameter used in EC algorithm
:type y: ``Optional[str]``
:param y: Y parameter used in EC algorithm
:type curve: ``Optional[str]``
:param curve: curve parameter used in EC algorithm
:return: None
:rtype: ``None``
"""
class Algorithm(object):
"""
Algorithm class to enumerate available algorithms
:return: None
:rtype: ``None``
"""
DSA = "DSA"
RSA = "RSA"
EC = "EC"
UNKNOWN = "Unknown Algorithm"
@staticmethod
def is_valid_type(_type):
return _type in (
Common.CertificatePublicKey.Algorithm.DSA,
Common.CertificatePublicKey.Algorithm.RSA,
Common.CertificatePublicKey.Algorithm.EC,
Common.CertificatePublicKey.Algorithm.UNKNOWN
)
def __init__(
self,
algorithm, # type: str
length, # type: int
publickey=None, # type: str
p=None, # type: str
q=None, # type: str
g=None, # type: str
modulus=None, # type: str
exponent=None, # type: int
x=None, # type: str
y=None, # type: str
curve=None # type: str
):
if not Common.CertificatePublicKey.Algorithm.is_valid_type(algorithm):
raise TypeError('algorithm must be of type Common.CertificatePublicKey.Algorithm enum')
self.algorithm = algorithm
self.length = length
self.publickey = publickey
self.p = p
self.q = q
self.g = g
self.modulus = modulus
self.exponent = exponent
self.x = x
self.y = y
self.curve = curve
def to_context(self):
publickey_context = {
'Algorithm': self.algorithm,
'Length': self.length
}
if self.publickey:
publickey_context['PublicKey'] = self.publickey
if self.algorithm == Common.CertificatePublicKey.Algorithm.DSA:
if self.p:
publickey_context['P'] = self.p
if self.q:
publickey_context['Q'] = self.q
if self.g:
publickey_context['G'] = self.g
elif self.algorithm == Common.CertificatePublicKey.Algorithm.RSA:
if self.modulus:
publickey_context['Modulus'] = self.modulus
if self.exponent:
publickey_context['Exponent'] = self.exponent
elif self.algorithm == Common.CertificatePublicKey.Algorithm.EC:
if self.x:
publickey_context['X'] = self.x
if self.y:
publickey_context['Y'] = self.y
if self.curve:
publickey_context['Curve'] = self.curve
elif self.algorithm == Common.CertificatePublicKey.Algorithm.UNKNOWN:
pass
return publickey_context
def to_minimum_context(self):
return self.to_context()
class GeneralName(object):
"""
GeneralName class
Implements GeneralName interface from rfc5280
Enumerates the available General Name Types
:type gn_type: ``str``
:param gn_type: General Name Type
:type gn_value: ``str``
:param gn_value: General Name Value
:return: None
:rtype: ``None``
"""
OTHERNAME = 'otherName'
RFC822NAME = 'rfc822Name'
DNSNAME = 'dNSName'
DIRECTORYNAME = 'directoryName'
UNIFORMRESOURCEIDENTIFIER = 'uniformResourceIdentifier'
IPADDRESS = 'iPAddress'
REGISTEREDID = 'registeredID'
@staticmethod
def is_valid_type(_type):
return _type in (
Common.GeneralName.OTHERNAME,
Common.GeneralName.RFC822NAME,
Common.GeneralName.DNSNAME,
Common.GeneralName.DIRECTORYNAME,
Common.GeneralName.UNIFORMRESOURCEIDENTIFIER,
Common.GeneralName.IPADDRESS,
Common.GeneralName.REGISTEREDID
)
def __init__(
self,
gn_value, # type: str
gn_type # type: str
):
if not Common.GeneralName.is_valid_type(gn_type):
raise TypeError(
'gn_type must be of type Common.GeneralName enum'
)
self.gn_type = gn_type
self.gn_value = gn_value
def to_context(self):
return {
'Type': self.gn_type,
'Value': self.gn_value
}
def to_minimum_context(self):
return self.to_context()
def get_value(self):
return self.gn_value
class CertificateExtension(object):
"""
CertificateExtension class
Defines an X509 Certificate Extensions used in Common.Certificate
:type extension_type: ``str``
:param extension_type: The type of Extension (from Common.CertificateExtension.ExtensionType enum, or "Other)
:type critical: ``bool``
:param critical: Whether the extension is marked as critical
:type extension_name: ``Optional[str]``
:param extension_name: Name of the extension
:type oid: ``Optional[str]``
:param oid: OID of the extension
:type subject_alternative_names: ``Optional[List[Common.CertificateExtension.SubjectAlternativeName]]``
:param subject_alternative_names: Subject Alternative Names
:type authority_key_identifier: ``Optional[Common.CertificateExtension.AuthorityKeyIdentifier]``
:param authority_key_identifier: Authority Key Identifier
:type digest: ``Optional[str]``
:param digest: digest for Subject Key Identifier extension
:type digital_signature: ``Optional[bool]``
:param digital_signature: Digital Signature usage for Key Usage extension
:type content_commitment: ``Optional[bool]``
:param content_commitment: Content Commitment usage for Key Usage extension
:type key_encipherment: ``Optional[bool]``
:param key_encipherment: Key Encipherment usage for Key Usage extension
:type data_encipherment: ``Optional[bool]``
:param data_encipherment: Data Encipherment usage for Key Usage extension
:type key_agreement: ``Optional[bool]``
:param key_agreement: Key Agreement usage for Key Usage extension
:type key_cert_sign: ``Optional[bool]``
:param key_cert_sign: Key Cert Sign usage for Key Usage extension
:type usages: ``Optional[List[str]]``
:param usages: Usages for Extended Key Usage extension
:type distribution_points: ``Optional[List[Common.CertificateExtension.DistributionPoint]]``
:param distribution_points: Distribution Points
:type certificate_policies: ``Optional[List[Common.CertificateExtension.CertificatePolicy]]``
:param certificate_policies: Certificate Policies
:type authority_information_access: ``Optional[List[Common.CertificateExtension.AuthorityInformationAccess]]``
:param authority_information_access: Authority Information Access
:type basic_constraints: ``Optional[Common.CertificateExtension.BasicConstraints]``
:param basic_constraints: Basic Constraints
:type signed_certificate_timestamps: ``Optional[List[Common.CertificateExtension.SignedCertificateTimestamp]]``
:param signed_certificate_timestamps: (PreCertificate)Signed Certificate Timestamps
:type value: ``Optional[Union[str, List[Any], Dict[str, Any]]]``
:param value: Raw value of the Extension (used for "Other" type)
:return: None
:rtype: ``None``
"""
class SubjectAlternativeName(object):
"""
SubjectAlternativeName class
Implements Subject Alternative Name extension interface
:type gn: ``Optional[Common.GeneralName]``
:param gn: General Name Type provided as Common.GeneralName
:type gn_type: ``Optional[str]``
:param gn_type: General Name Type provided as string
:type gn_value: ``Optional[str]``
:param gn_value: General Name Value provided as string
:return: None
:rtype: ``None``
"""
def __init__(
self,
gn=None, # type: Optional[Common.GeneralName]
gn_type=None, # type: Optional[str]
gn_value=None # type: Optional[str]
):
if gn:
self.gn = gn
elif gn_type and gn_value:
self.gn = Common.GeneralName(
gn_value=gn_value,
gn_type=gn_type
)
else:
raise ValueError('either GeneralName or gn_type/gn_value required to inizialize SubjectAlternativeName')
def to_context(self):
return self.gn.to_context()
def to_minimum_context(self):
return self.to_context()
def get_value(self):
return self.gn.get_value()
class AuthorityKeyIdentifier(object):
"""
AuthorityKeyIdentifier class
Implements Authority Key Identifier extension interface
:type issuer: ``Optional[List[Common.GeneralName]]``
:param issuer: Issuer list
:type serial_number: ``Optional[str]``
:param serial_number: Serial Number
:type key_identifier: ``Optional[str]``
:param key_identifier: Key Identifier
:return: None
:rtype: ``None``
"""
def __init__(
self,
issuer=None, # type: Optional[List[Common.GeneralName]]
serial_number=None, # type: Optional[str]
key_identifier=None # type: Optional[str]
):
self.issuer = issuer
self.serial_number = serial_number
self.key_identifier = key_identifier
def to_context(self):
authority_key_identifier_context = {} # type: Dict[str, Any]
if self.issuer:
authority_key_identifier_context['Issuer'] = self.issuer,
if self.serial_number:
authority_key_identifier_context["SerialNumber"] = self.serial_number
if self.key_identifier:
authority_key_identifier_context["KeyIdentifier"] = self.key_identifier
return authority_key_identifier_context
def to_minimum_context(self):
return self.to_context()
class DistributionPoint(object):
"""
DistributionPoint class
Implements Distribution Point extension interface
:type full_name: ``Optional[List[Common.GeneralName]]``
:param full_name: Full Name list
:type relative_name: ``Optional[str]``
:param relative_name: Relative Name
:type crl_issuer: ``Optional[List[Common.GeneralName]]``
:param crl_issuer: CRL Issuer
:type reasons: ``Optional[List[str]]``
:param reasons: Reason list
:return: None
:rtype: ``None``
"""
def __init__(
self,
full_name=None, # type: Optional[List[Common.GeneralName]]
relative_name=None, # type: Optional[str]
crl_issuer=None, # type: Optional[List[Common.GeneralName]]
reasons=None # type: Optional[List[str]]
):
self.full_name = full_name
self.relative_name = relative_name
self.crl_issuer = crl_issuer
self.reasons = reasons
def to_context(self):
distribution_point_context = {} # type: Dict[str, Union[List, str]]
if self.full_name:
distribution_point_context["FullName"] = [fn.to_context() for fn in self.full_name]
if self.relative_name:
distribution_point_context["RelativeName"] = self.relative_name
if self.crl_issuer:
distribution_point_context["CRLIssuer"] = [ci.to_context() for ci in self.crl_issuer]
if self.reasons:
distribution_point_context["Reasons"] = self.reasons
return distribution_point_context
def to_minimum_context(self):
return self.to_context()
class CertificatePolicy(object):
"""
CertificatePolicy class
Implements Certificate Policy extension interface
:type policy_identifier: ``str``
:param policy_identifier: Policy Identifier
:type policy_qualifiers: ``Optional[List[str]]``
:param policy_qualifiers: Policy Qualifier list
:return: None
:rtype: ``None``
"""
def __init__(
self,
policy_identifier, # type: str
policy_qualifiers=None # type: Optional[List[str]]
):
self.policy_identifier = policy_identifier
self.policy_qualifiers = policy_qualifiers
def to_context(self):
certificate_policies_context = {
"PolicyIdentifier": self.policy_identifier
} # type: Dict[str, Union[List, str]]
if self.policy_qualifiers:
certificate_policies_context["PolicyQualifiers"] = self.policy_qualifiers
return certificate_policies_context
def to_minimum_context(self):
return self.to_context()
class AuthorityInformationAccess(object):
"""
AuthorityInformationAccess class
Implements Authority Information Access extension interface
:type access_method: ``str``
:param access_method: Access Method
:type access_location: ``Common.GeneralName``
:param access_location: Access Location
:return: None
:rtype: ``None``
"""
def __init__(
self,
access_method, # type: str
access_location # type: Common.GeneralName
):
self.access_method = access_method
self.access_location = access_location
def to_context(self):
return {
"AccessMethod": self.access_method,
"AccessLocation": self.access_location.to_context()
}
def to_minimum_context(self):
return self.to_context()
class BasicConstraints(object):
"""
BasicConstraints class
Implements Basic Constraints extension interface
:type ca: ``bool``
:param ca: Certificate Authority
:type path_length: ``int``
:param path_length: Path Length
:return: None
:rtype: ``None``
"""
def __init__(
self,
ca, # type: bool
path_length=None # type: int
):
self.ca = ca
self.path_length = path_length
def to_context(self):
basic_constraints_context = {
"CA": self.ca
} # type: Dict[str, Union[str, int]]
if self.path_length:
basic_constraints_context["PathLength"] = self.path_length
return basic_constraints_context
def to_minimum_context(self):
return self.to_context()
class SignedCertificateTimestamp(object):
"""
SignedCertificateTimestamp class
Implementsinterface for "SignedCertificateTimestamp" extensions
:type entry_type: ``str``
:param entry_type: Entry Type (from Common.CertificateExtension.SignedCertificateTimestamp.EntryType enum)
:type version: ``str``
:param version: Version
:type log_id: ``str``
:param log_id: Log ID
:type timestamp: ``str``
:param timestamp: Timestamp (ISO8601 string representation in UTC)
:return: None
:rtype: ``None``
"""
class EntryType(object):
"""
EntryType class
Enumerates Entry Types for SignedCertificateTimestamp class
:return: None
:rtype: ``None``
"""
PRECERTIFICATE = "PreCertificate"
X509CERTIFICATE = "X509Certificate"
@staticmethod
def is_valid_type(_type):
return _type in (
Common.CertificateExtension.SignedCertificateTimestamp.EntryType.PRECERTIFICATE,
Common.CertificateExtension.SignedCertificateTimestamp.EntryType.X509CERTIFICATE
)
def __init__(
self,
entry_type, # type: str
version, # type: int
log_id, # type: str
timestamp # type: str
):
if not Common.CertificateExtension.SignedCertificateTimestamp.EntryType.is_valid_type(entry_type):
raise TypeError(
'entry_type must be of type Common.CertificateExtension.SignedCertificateTimestamp.EntryType enum'
)
self.entry_type = entry_type
self.version = version
self.log_id = log_id
self.timestamp = timestamp
def to_context(self):
timestamps_context = {} # type: Dict[str, Any]
timestamps_context['Version'] = self.version
timestamps_context["LogId"] = self.log_id
timestamps_context["Timestamp"] = self.timestamp
timestamps_context["EntryType"] = self.entry_type
return timestamps_context
def to_minimum_context(self):
return self.to_context()
class ExtensionType(object):
"""
ExtensionType class
Enumerates Extension Types for Common.CertificatExtension class
:return: None
:rtype: ``None``
"""
SUBJECTALTERNATIVENAME = "SubjectAlternativeName"
AUTHORITYKEYIDENTIFIER = "AuthorityKeyIdentifier"
SUBJECTKEYIDENTIFIER = "SubjectKeyIdentifier"
KEYUSAGE = "KeyUsage"
EXTENDEDKEYUSAGE = "ExtendedKeyUsage"
CRLDISTRIBUTIONPOINTS = "CRLDistributionPoints"
CERTIFICATEPOLICIES = "CertificatePolicies"
AUTHORITYINFORMATIONACCESS = "AuthorityInformationAccess"
BASICCONSTRAINTS = "BasicConstraints"
SIGNEDCERTIFICATETIMESTAMPS = "SignedCertificateTimestamps"
PRESIGNEDCERTIFICATETIMESTAMPS = "PreCertSignedCertificateTimestamps"
OTHER = "Other"
@staticmethod
def is_valid_type(_type):
return _type in (
Common.CertificateExtension.ExtensionType.SUBJECTALTERNATIVENAME,
Common.CertificateExtension.ExtensionType.AUTHORITYKEYIDENTIFIER,
Common.CertificateExtension.ExtensionType.SUBJECTKEYIDENTIFIER,
Common.CertificateExtension.ExtensionType.KEYUSAGE,
Common.CertificateExtension.ExtensionType.EXTENDEDKEYUSAGE,
Common.CertificateExtension.ExtensionType.CRLDISTRIBUTIONPOINTS,
Common.CertificateExtension.ExtensionType.CERTIFICATEPOLICIES,
Common.CertificateExtension.ExtensionType.AUTHORITYINFORMATIONACCESS,
Common.CertificateExtension.ExtensionType.BASICCONSTRAINTS,
Common.CertificateExtension.ExtensionType.SIGNEDCERTIFICATETIMESTAMPS,
Common.CertificateExtension.ExtensionType.PRESIGNEDCERTIFICATETIMESTAMPS,
Common.CertificateExtension.ExtensionType.OTHER # for extensions that are not handled explicitly
)
def __init__(
self,
extension_type, # type: str
critical, # type: bool
oid=None, # type: Optional[str]
extension_name=None, # type: Optional[str]
subject_alternative_names=None, # type: Optional[List[Common.CertificateExtension.SubjectAlternativeName]]
authority_key_identifier=None, # type: Optional[Common.CertificateExtension.AuthorityKeyIdentifier]
digest=None, # type: str
digital_signature=None, # type: Optional[bool]
content_commitment=None, # type: Optional[bool]
key_encipherment=None, # type: Optional[bool]
data_encipherment=None, # type: Optional[bool]
key_agreement=None, # type: Optional[bool]
key_cert_sign=None, # type: Optional[bool]
crl_sign=None, # type: Optional[bool]
usages=None, # type: Optional[List[str]]
distribution_points=None, # type: Optional[List[Common.CertificateExtension.DistributionPoint]]
certificate_policies=None, # type: Optional[List[Common.CertificateExtension.CertificatePolicy]]
authority_information_access=None, # type: Optional[List[Common.CertificateExtension.AuthorityInformationAccess]]
basic_constraints=None, # type: Optional[Common.CertificateExtension.BasicConstraints]
signed_certificate_timestamps=None, # type: Optional[List[Common.CertificateExtension.SignedCertificateTimestamp]]
value=None # type: Optional[Union[str, List[Any], Dict[str, Any]]]
):
if not Common.CertificateExtension.ExtensionType.is_valid_type(extension_type):
raise TypeError('algorithm must be of type Common.CertificateExtension.ExtensionType enum')
self.extension_type = extension_type
self.critical = critical
if self.extension_type == Common.CertificateExtension.ExtensionType.SUBJECTALTERNATIVENAME:
self.subject_alternative_names = subject_alternative_names
self.oid = "2.5.29.17"
self.extension_name = "subjectAltName"
elif self.extension_type == Common.CertificateExtension.ExtensionType.SUBJECTKEYIDENTIFIER:
if not digest:
raise ValueError('digest is mandatory for SubjectKeyIdentifier extension')
self.digest = digest
self.oid = "2.5.29.14"
self.extension_name = "subjectKeyIdentifier"
elif self.extension_type == Common.CertificateExtension.ExtensionType.KEYUSAGE:
self.digital_signature = digital_signature
self.content_commitment = content_commitment
self.key_encipherment = key_encipherment
self.data_encipherment = data_encipherment
self.key_agreement = key_agreement
self.key_cert_sign = key_cert_sign
self.crl_sign = crl_sign
self.oid = "2.5.29.15"
self.extension_name = "keyUsage"
elif self.extension_type == Common.CertificateExtension.ExtensionType.EXTENDEDKEYUSAGE:
if not usages:
raise ValueError('usages is mandatory for ExtendedKeyUsage extension')
self.usages = usages
self.oid = "2.5.29.37"
self.extension_name = "extendedKeyUsage"
elif self.extension_type == Common.CertificateExtension.ExtensionType.AUTHORITYKEYIDENTIFIER:
self.authority_key_identifier = authority_key_identifier
self.oid = "2.5.29.35"
self.extension_name = "authorityKeyIdentifier"
elif self.extension_type == Common.CertificateExtension.ExtensionType.CRLDISTRIBUTIONPOINTS:
self.distribution_points = distribution_points
self.oid = "2.5.29.31"
self.extension_name = "cRLDistributionPoints"
elif self.extension_type == Common.CertificateExtension.ExtensionType.CERTIFICATEPOLICIES:
self.certificate_policies = certificate_policies
self.oid = "2.5.29.32"
self.extension_name = "certificatePolicies"
elif self.extension_type == Common.CertificateExtension.ExtensionType.AUTHORITYINFORMATIONACCESS:
self.authority_information_access = authority_information_access
self.oid = "1.3.6.1.5.5.7.1.1"
self.extension_name = "authorityInfoAccess"
elif self.extension_type == Common.CertificateExtension.ExtensionType.BASICCONSTRAINTS:
self.basic_constraints = basic_constraints
self.oid = "2.5.29.19"
self.extension_name = "basicConstraints"
elif self.extension_type == Common.CertificateExtension.ExtensionType.PRESIGNEDCERTIFICATETIMESTAMPS:
self.signed_certificate_timestamps = signed_certificate_timestamps
self.oid = "1.3.6.1.4.1.11129.2.4.2"
self.extension_name = "signedCertificateTimestampList"
elif self.extension_type == Common.CertificateExtension.ExtensionType.SIGNEDCERTIFICATETIMESTAMPS:
self.signed_certificate_timestamps = signed_certificate_timestamps
self.oid = "1.3.6.1.4.1.11129.2.4.5"
self.extension_name = "signedCertificateTimestampList"
elif self.extension_type == Common.CertificateExtension.ExtensionType.OTHER:
self.value = value
# override oid, extension_name if provided as inputs
if oid:
self.oid = oid
if extension_name:
self.extension_name = extension_name
def to_context(self):
extension_context = {
"OID": self.oid,
"Name": self.extension_name,
"Critical": self.critical
} # type: Dict[str, Any]
if (
self.extension_type == Common.CertificateExtension.ExtensionType.SUBJECTALTERNATIVENAME
and self.subject_alternative_names is not None
):
extension_context["Value"] = [san.to_context() for san in self.subject_alternative_names]
elif (
self.extension_type == Common.CertificateExtension.ExtensionType.AUTHORITYKEYIDENTIFIER
and self.authority_key_identifier is not None
):
extension_context["Value"] = self.authority_key_identifier.to_context()
elif (
self.extension_type == Common.CertificateExtension.ExtensionType.SUBJECTKEYIDENTIFIER
and self.digest is not None
):
extension_context["Value"] = {
"Digest": self.digest
}
elif self.extension_type == Common.CertificateExtension.ExtensionType.KEYUSAGE:
key_usage = {} # type: Dict[str, bool]
if self.digital_signature:
key_usage["DigitalSignature"] = self.digital_signature
if self.content_commitment:
key_usage["ContentCommitment"] = self.content_commitment
if self.key_encipherment:
key_usage["KeyEncipherment"] = self.key_encipherment
if self.data_encipherment:
key_usage["DataEncipherment"] = self.data_encipherment
if self.key_agreement:
key_usage["KeyAgreement"] = self.key_agreement
if self.key_cert_sign:
key_usage["KeyCertSign"] = self.key_cert_sign
if self.crl_sign:
key_usage["CrlSign"] = self.crl_sign
if key_usage:
extension_context["Value"] = key_usage
elif (
self.extension_type == Common.CertificateExtension.ExtensionType.EXTENDEDKEYUSAGE
and self.usages is not None
):
extension_context["Value"] = {
"Usages": [u for u in self.usages]
}
elif (
self.extension_type == Common.CertificateExtension.ExtensionType.CRLDISTRIBUTIONPOINTS
and self.distribution_points is not None
):
extension_context["Value"] = [dp.to_context() for dp in self.distribution_points]
elif (
self.extension_type == Common.CertificateExtension.ExtensionType.CERTIFICATEPOLICIES
and self.certificate_policies is not None
):
extension_context["Value"] = [cp.to_context() for cp in self.certificate_policies]
elif (
self.extension_type == Common.CertificateExtension.ExtensionType.AUTHORITYINFORMATIONACCESS
and self.authority_information_access is not None
):
extension_context["Value"] = [aia.to_context() for aia in self.authority_information_access]
elif (
self.extension_type == Common.CertificateExtension.ExtensionType.BASICCONSTRAINTS
and self.basic_constraints is not None
):
extension_context["Value"] = self.basic_constraints.to_context()
elif (
self.extension_type in [
Common.CertificateExtension.ExtensionType.SIGNEDCERTIFICATETIMESTAMPS,
Common.CertificateExtension.ExtensionType.PRESIGNEDCERTIFICATETIMESTAMPS
]
and self.signed_certificate_timestamps is not None
):
extension_context["Value"] = [sct.to_context() for sct in self.signed_certificate_timestamps]
elif (
self.extension_type == Common.CertificateExtension.ExtensionType.OTHER
and self.value is not None
):
extension_context["Value"] = self.value
return extension_context
def to_minimum_context(self):
return self.to_context()
class Certificate(Indicator):
"""
Implements the X509 Certificate interface
Certificate indicator - https://xsoar.pan.dev/docs/integrations/context-standards-mandatory#certificate
:type subject_dn: ``str``
:param subject_dn: Subject Distinguished Name
:type dbot_score: ``DBotScore``
:param dbot_score: If Certificate has a score then create and set a DBotScore object.
:type name: ``Optional[Union[str, List[str]]]``
:param name: Name (if not provided output is calculated from SubjectDN and SAN)
:type issuer_dn: ``Optional[str]``
:param issuer_dn: Issuer Distinguished Name
:type serial_number: ``Optional[str]``
:param serial_number: Serial Number
:type validity_not_after: ``Optional[str]``
:param validity_not_after: Certificate Expiration Timestamp (ISO8601 string representation)
:type validity_not_before: ``Optional[str]``
:param validity_not_before: Initial Certificate Validity Timestamp (ISO8601 string representation)
:type sha512: ``Optional[str]``
:param sha512: The SHA-512 hash of the certificate in binary encoded format (DER)
:type sha256: ``Optional[str]``
:param sha256: The SHA-256 hash of the certificate in binary encoded format (DER)
:type sha1: ``Optional[str]``
:param sha1: The SHA-1 hash of the certificate in binary encoded format (DER)
:type md5: ``Optional[str]``
:param md5: The MD5 hash of the certificate in binary encoded format (DER)
:type publickey: ``Optional[Common.CertificatePublicKey]``
:param publickey: Certificate Public Key
:type spki_sha256: ``Optional[str]``
:param sha1: The SHA-256 hash of the SPKI
:type signature_algorithm: ``Optional[str]``
:param signature_algorithm: Signature Algorithm
:type signature: ``Optional[str]``
:param signature: Certificate Signature
:type subject_alternative_name: \
``Optional[List[Union[str,Dict[str, str],Common.CertificateExtension.SubjectAlternativeName]]]``
:param subject_alternative_name: Subject Alternative Name list
:type extensions: ``Optional[List[Common.CertificateExtension]]`
:param extensions: Certificate Extension List
:type pem: ``Optional[str]``
:param pem: PEM encoded certificate
:return: None
:rtype: ``None``
"""
CONTEXT_PATH = 'Certificate(val.MD5 && val.MD5 == obj.MD5 || val.SHA1 && val.SHA1 == obj.SHA1 || ' \
'val.SHA256 && val.SHA256 == obj.SHA256 || val.SHA512 && val.SHA512 == obj.SHA512)'
def __init__(
self,
subject_dn, # type: str
dbot_score=None, # type: Optional[Common.DBotScore]
name=None, # type: Optional[Union[str, List[str]]]
issuer_dn=None, # type: Optional[str]
serial_number=None, # type: Optional[str]
validity_not_after=None, # type: Optional[str]
validity_not_before=None, # type: Optional[str]
sha512=None, # type: Optional[str]
sha256=None, # type: Optional[str]
sha1=None, # type: Optional[str]
md5=None, # type: Optional[str]
publickey=None, # type: Optional[Common.CertificatePublicKey]
spki_sha256=None, # type: Optional[str]
signature_algorithm=None, # type: Optional[str]
signature=None, # type: Optional[str]
subject_alternative_name=None, \
# type: Optional[List[Union[str,Dict[str, str],Common.CertificateExtension.SubjectAlternativeName]]]
extensions=None, # type: Optional[List[Common.CertificateExtension]]
pem=None # type: Optional[str]
):
self.subject_dn = subject_dn
self.dbot_score = dbot_score
self.name = None
if name:
if isinstance(name, str):
self.name = [name]
elif isinstance(name, list):
self.name = name
else:
raise TypeError('certificate name must be of type str or List[str]')
self.issuer_dn = issuer_dn
self.serial_number = serial_number
self.validity_not_after = validity_not_after
self.validity_not_before = validity_not_before
self.sha512 = sha512
self.sha256 = sha256
self.sha1 = sha1
self.md5 = md5
if publickey and not isinstance(publickey, Common.CertificatePublicKey):
raise TypeError('publickey must be of type Common.CertificatePublicKey')
self.publickey = publickey
self.spki_sha256 = spki_sha256
self.signature_algorithm = signature_algorithm
self.signature = signature
# if subject_alternative_name is set and is a list
# make sure it is a list of strings, dicts of strings or SAN Extensions
if (
subject_alternative_name
and isinstance(subject_alternative_name, list)
and not all(
isinstance(san, str)
or isinstance(san, dict)
or isinstance(san, Common.CertificateExtension.SubjectAlternativeName)
for san in subject_alternative_name)
):
raise TypeError(
'subject_alternative_name must be list of str or Common.CertificateExtension.SubjectAlternativeName'
)
self.subject_alternative_name = subject_alternative_name
if (
extensions
and not isinstance(extensions, list)
and any(isinstance(e, Common.CertificateExtension) for e in extensions) # type: ignore
):
raise TypeError('extensions must be of type List[Common.CertificateExtension]')
self.extensions = extensions
self.pem = pem
if not isinstance(dbot_score, Common.DBotScore):
raise ValueError('dbot_score must be of type DBotScore')
def to_context(self):
certificate_context = {
"SubjectDN": self.subject_dn
} # type: Dict[str, Any]
san_list = [] # type: List[Dict[str, str]]
if self.subject_alternative_name:
for san in self.subject_alternative_name:
if isinstance(san, str):
san_list.append({
'Value': san
})
elif isinstance(san, dict):
san_list.append(san)
elif (isinstance(san, Common.CertificateExtension.SubjectAlternativeName)):
san_list.append(san.to_context())
elif self.extensions: # autogenerate it from extensions
for ext in self.extensions:
if (
ext.extension_type == Common.CertificateExtension.ExtensionType.SUBJECTALTERNATIVENAME
and ext.subject_alternative_names is not None
):
for san in ext.subject_alternative_names:
san_list.append(san.to_context())
if san_list:
certificate_context['SubjectAlternativeName'] = san_list
if self.name:
certificate_context["Name"] = self.name
else: # autogenerate it
name = set() # type: Set[str]
# add subject alternative names
if san_list:
name = set([
sn['Value'] for sn in san_list
if (
'Value' in sn
and (
'Type' not in sn
or sn['Type'] in (Common.GeneralName.DNSNAME, Common.GeneralName.IPADDRESS)
)
)
])
# subject_dn is RFC4515 escaped
# replace \, and \+ with the long escaping \2c and \2b
long_escaped_subject_dn = self.subject_dn.replace("\\,", "\\2c")
long_escaped_subject_dn = long_escaped_subject_dn.replace("\\+", "\\2b")
# we then split RDN (separated by ,) and multi-valued RDN (sep by +)
rdns = long_escaped_subject_dn.replace('+', ',').split(',')
cn = next((rdn for rdn in rdns if rdn.startswith('CN=')), None)
if cn:
name.add(cn.split('=', 1)[-1])
if name:
certificate_context["Name"] = sorted(list(name))
if self.issuer_dn:
certificate_context["IssuerDN"] = self.issuer_dn
if self.serial_number:
certificate_context["SerialNumber"] = self.serial_number
if self.validity_not_before:
certificate_context["ValidityNotBefore"] = self.validity_not_before
if self.validity_not_after:
certificate_context["ValidityNotAfter"] = self.validity_not_after
if self.sha512:
certificate_context["SHA512"] = self.sha512
if self.sha256:
certificate_context["SHA256"] = self.sha256
if self.sha1:
certificate_context["SHA1"] = self.sha1
if self.md5:
certificate_context["MD5"] = self.md5
if self.publickey and isinstance(self.publickey, Common.CertificatePublicKey):
certificate_context["PublicKey"] = self.publickey.to_context()
if self.spki_sha256:
certificate_context["SPKISHA256"] = self.spki_sha256
sig = {} # type: Dict[str, str]
if self.signature_algorithm:
sig["Algorithm"] = self.signature_algorithm
if self.signature:
sig["Signature"] = self.signature
if sig:
certificate_context["Signature"] = sig
if self.extensions:
certificate_context["Extension"] = [e.to_context() for e in self.extensions]
if self.pem:
certificate_context["PEM"] = self.pem
if self.dbot_score and self.dbot_score.score == Common.DBotScore.BAD:
certificate_context['Malicious'] = {
'Vendor': self.dbot_score.integration_name,
'Description': self.dbot_score.malicious_description
}
ret_value = {
Common.Certificate.CONTEXT_PATH: certificate_context
}
if self.dbot_score:
ret_value.update(self.dbot_score.to_context())
return ret_value
def to_minimum_context(self):
certificate_context = {
"SubjectDN": self.subject_dn
}
if self.sha512:
certificate_context["SHA512"] = self.sha512
if self.sha256:
certificate_context["SHA256"] = self.sha256
if self.sha1:
certificate_context["SHA1"] = self.sha1
if self.md5:
certificate_context["MD5"] = self.md5
ret_value = {
Common.Certificate.CONTEXT_PATH: certificate_context
}
if self.dbot_score:
ret_value.update(self.dbot_score.to_context())
return ret_value
class Tactic(Indicator):
"""
Tactic indicator
:type stix_id: ``str``
:param stix_id: The Tactic STIX ID
:type first_seen_by_source: ``str``
:param first_seen_by_source: The Tactic first seen by source
:type description: ``str``
:param description: The Tactic description
:type publications: ``str``
:param publications: The Tactic publications
:type mitre_id: ``str``
:param mitre_id: The Tactic mitre id.
:type tags: ``str``
:param tags: The Tactic tags.
:type dbot_score: ``DBotScore``
:param dbot_score: If the address has reputation then create DBotScore object.
:type traffic_light_protocol: ``str``
:param traffic_light_protocol: The Traffic Light Protocol (TLP) color that is suitable for the Tactic.
:type community_notes: ``CommunityNotes``
:param community_notes: A list of community notes for the Tactic.
:type external_references: ``ExternalReference``
:param external_references: A list of id's and description of the Tactic via external refs.
:type value: ``str``
:param value: The Tactic value (name) - example: "Plist File Modification"
:return: None
:rtype: ``None``
"""
CONTEXT_PATH = 'Tactic(val.Name && val.Name == obj.Name)'
def __init__(self, stix_id, first_seen_by_source=None, description=None, publications=None, mitre_id=None, tags=None,
traffic_light_protocol=None, dbot_score=None, community_notes=None, external_references=None, value=None):
self.community_notes = community_notes
self.description = description
self.external_references = external_references
self.first_seen_by_source = first_seen_by_source
self.mitre_id = mitre_id
self.publications = publications
self.stix_id = stix_id
self.tags = tags
self.traffic_light_protocol = traffic_light_protocol
self.value = value
self.dbot_score = dbot_score
def to_context(self):
attack_pattern_context = {
'STIXID': self.stix_id,
"FirstSeenBySource": self.first_seen_by_source,
"Publications": self.publications,
"MITREID": self.mitre_id,
"Value": self.value,
"Tags": self.tags,
"Description": self.description
}
if self.external_references:
attack_pattern_context['ExternalReferences'] = self.create_context_table(self.external_references)
if self.traffic_light_protocol:
attack_pattern_context['TrafficLightProtocol'] = self.traffic_light_protocol
if self.dbot_score and self.dbot_score.score == Common.DBotScore.BAD:
attack_pattern_context['Malicious'] = {
'Vendor': self.dbot_score.integration_name,
'Description': self.dbot_score.malicious_description
}
ret_value = {
Common.AttackPattern.CONTEXT_PATH: attack_pattern_context
}
if self.dbot_score:
ret_value.update(self.dbot_score.to_context())
return ret_value
def to_minimum_context(self):
tactic_context = {
'STIXID': self.stix_id,
'Value': self.value
}
ret_value = {
Common.Tactic.CONTEXT_PATH: tactic_context
}
if self.dbot_score:
ret_value.update(self.dbot_score.to_context())
return ret_value
class ScheduledCommand:
"""
ScheduledCommand configuration class
Holds the scheduled command configuration for the command result - managing the way the command should be polled.
:type command: ``str``
:param command: The command that'll run after next_run_in_seconds has passed.
:type next_run_in_seconds: ``int``
:param next_run_in_seconds: How long to wait before executing the command.
:type args: ``Optional[Dict[str, Any]]``
:param args: Arguments to use when executing the command.
:type timeout_in_seconds: ``Optional[int]``
:param timeout_in_seconds: Number of seconds until the polling sequence will timeout.
:type items_remaining: ``Optional[int]``
:param items_remaining: Number of items that are remaining to be polled.
:return: None
:rtype: ``None``
"""
VERSION_MISMATCH_ERROR = 'This command is not supported by this XSOAR server version. Please update your server ' \
'version to 6.2.0 or later.'
def __init__(
self,
command, # type: str
next_run_in_seconds, # type: int
args=None, # type: Optional[Dict[str, Any]]
timeout_in_seconds=None, # type: Optional[int]
items_remaining=0, # type: Optional[int]
):
self.raise_error_if_not_supported()
self._command = command
if next_run_in_seconds < 10:
demisto.info('ScheduledCommandConfiguration provided value for next_run_in_seconds: '
'{} is '.format(next_run_in_seconds) + 'too low - minimum interval is 10 seconds. '
'next_run_in_seconds was set to 10 seconds.')
next_run_in_seconds = 10
self._next_run = str(next_run_in_seconds)
self._args = args
self._timeout = str(timeout_in_seconds) if timeout_in_seconds else None
self._items_remaining = items_remaining
@staticmethod
def raise_error_if_not_supported():
return True
@staticmethod
def supports_polling():
"""
Check if the integration supports polling (if server version is greater than 6.2.0).
Returns: Boolean
"""
return True
def to_results(self):
"""
Returns the result dictionary of the polling command
"""
return assign_params(
PollingCommand=self._command,
NextRun=self._next_run,
PollingArgs=self._args,
Timeout=self._timeout,
PollingItemsRemaining=self._items_remaining
)
def camelize_string(src_str, delim='_', upper_camel=True):
"""
Transform snake_case to CamelCase
:type src_str: ``str``
:param src_str: snake_case string to convert.
:type delim: ``str``
:param delim: indicator category.
:type upper_camel: ``bool``
:param upper_camel: When True then transforms string to camel case with the first letter capitalised
(for example: demisto_content to DemistoContent), otherwise the first letter will not be capitalised
(for example: demisto_content to demistoContent).
:return: A CammelCase string.
:rtype: ``str``
"""
if not src_str: # empty string
return ""
components = src_str.split(delim)
camelize_without_first_char = ''.join(map(lambda x: x.title(), components[1:]))
if upper_camel:
return components[0].title() + camelize_without_first_char
else:
return components[0].lower() + camelize_without_first_char
class IndicatorsTimeline:
"""
IndicatorsTimeline class - use to return Indicator Timeline object to be used in CommandResults
:type indicators: ``list``
:param indicators: expects a list of indicators.
:type category: ``str``
:param category: indicator category.
:type message: ``str``
:param message: indicator message.
:return: None
:rtype: ``None``
"""
def __init__(self, indicators=None, category=None, message=None):
# type: (list, str, str) -> None
if indicators is None:
indicators = []
# check if we are running from an integration or automation
try:
_ = demisto.params()
default_category = 'Integration Update'
except AttributeError:
default_category = 'Automation Update'
timelines = []
timeline = {}
for indicator in indicators:
timeline['Value'] = indicator
if category:
timeline['Category'] = category
else:
timeline['Category'] = default_category
if message:
timeline['Message'] = message
timelines.append(timeline)
self.indicators_timeline = timelines
def arg_to_number(arg, arg_name=None, required=False):
# type: (Any, Optional[str], bool) -> Optional[int]
"""Converts an XSOAR argument to a Python int
This function is used to quickly validate an argument provided to XSOAR
via ``demisto.args()`` into an ``int`` type. It will throw a ValueError
if the input is invalid. If the input is None, it will throw a ValueError
if required is ``True``, or ``None`` if required is ``False.
:type arg: ``Any``
:param arg: argument to convert
:type arg_name: ``str``
:param arg_name: argument name
:type required: ``bool``
:param required:
throws exception if ``True`` and argument provided is None
:return:
returns an ``int`` if arg can be converted
returns ``None`` if arg is ``None`` and required is set to ``False``
otherwise throws an Exception
:rtype: ``Optional[int]``
"""
if arg is None or arg == '':
if required is True:
if arg_name:
raise ValueError('Missing "{}"'.format(arg_name))
else:
raise ValueError('Missing required argument')
return None
arg = encode_string_results(arg)
if isinstance(arg, str):
if arg.isdigit():
return int(arg)
try:
return int(float(arg))
except Exception:
if arg_name:
raise ValueError('Invalid number: "{}"="{}"'.format(arg_name, arg))
else:
raise ValueError('"{}" is not a valid number'.format(arg))
if isinstance(arg, int):
return arg
if arg_name:
raise ValueError('Invalid number: "{}"="{}"'.format(arg_name, arg))
else:
raise ValueError('"{}" is not a valid number'.format(arg))
def arg_to_datetime(arg, arg_name=None, is_utc=True, required=False, settings=None):
# type: (Any, Optional[str], bool, bool, dict) -> Optional[datetime]
"""Converts an XSOAR argument to a datetime
This function is used to quickly validate an argument provided to XSOAR
via ``demisto.args()`` into an ``datetime``. It will throw a ValueError if the input is invalid.
If the input is None, it will throw a ValueError if required is ``True``,
or ``None`` if required is ``False.
:type arg: ``Any``
:param arg: argument to convert
:type arg_name: ``str``
:param arg_name: argument name
:type is_utc: ``bool``
:param is_utc: if True then date converted as utc timezone, otherwise will convert with local timezone.
:type required: ``bool``
:param required:
throws exception if ``True`` and argument provided is None
:type settings: ``dict``
:param settings: If provided, passed to dateparser.parse function.
:return:
returns an ``datetime`` if conversion works
returns ``None`` if arg is ``None`` and required is set to ``False``
otherwise throws an Exception
:rtype: ``Optional[datetime]``
"""
if arg is None:
if required is True:
if arg_name:
raise ValueError('Missing "{}"'.format(arg_name))
else:
raise ValueError('Missing required argument')
return None
if isinstance(arg, str) and arg.isdigit() or isinstance(arg, (int, float)):
# timestamp is a str containing digits - we just convert it to int
ms = float(arg)
if ms > 2000000000.0:
# in case timestamp was provided as unix time (in milliseconds)
ms = ms / 1000.0
if is_utc:
return datetime.fromtimestamp(ms, tz=timezone.utc)
else:
return datetime.fromtimestamp(ms)
if isinstance(arg, str):
# we use dateparser to handle strings either in ISO8601 format, or
# relative time stamps.
# For example: format 2019-10-23T00:00:00 or "3 days", etc
if settings:
date = dateparser.parse(arg, settings=settings) # type: ignore[arg-type]
else:
date = dateparser.parse(arg, settings={'TIMEZONE': 'UTC'})
if date is None:
# if d is None it means dateparser failed to parse it
if arg_name:
raise ValueError('Invalid date: "{}"="{}"'.format(arg_name, arg))
else:
raise ValueError('"{}" is not a valid date'.format(arg))
return date
if arg_name:
raise ValueError('Invalid date: "{}"="{}"'.format(arg_name, arg))
else:
raise ValueError('"{}" is not a valid date'.format(arg))
# -------------------------------- Relationships----------------------------------- #
class EntityRelationship:
"""
XSOAR entity relationship.
:type name: ``str``
:param name: Relationship name.
:type relationship_type: ``str``
:param relationship_type: Relationship type. (e.g. IndicatorToIndicator...).
:type entity_a: ``str``
:param entity_a: A value, A aka source of the relationship.
:type entity_a_family: ``str``
:param entity_a_family: Entity family of A, A aka source of the relationship. (e.g. Indicator...)
:type entity_a_type: ``str``
:param entity_a_type: Entity A type, A aka source of the relationship. (e.g. IP/URL/...).
:type entity_b: ``str``
:param entity_b: B value, B aka destination of the relationship.
:type entity_b_family: ``str``
:param entity_b_family: Entity family of B, B aka destination of the relationship. (e.g. Indicator...)
:type entity_b_type: ``str``
:param entity_b_type: Entity B type, B aka destination of the relationship. (e.g. IP/URL/...).
:type source_reliability: ``str``
:param source_reliability: Source reliability.
:type fields: ``dict``
:param fields: Custom fields. (Optional)
:type brand: ``str``
:param brand: Source brand name. (Optional)
:return: None
:rtype: ``None``
"""
class RelationshipsTypes(object):
"""
Relationships Types objects.
:return: None
:rtype: ``None``
"""
# dict which keys is a relationship type and the value is the reverse type.
RELATIONSHIP_TYPES = ['IndicatorToIndicator']
@staticmethod
def is_valid_type(_type):
# type: (str) -> bool
return _type in EntityRelationship.RelationshipsTypes.RELATIONSHIP_TYPES
class RelationshipsFamily(object):
"""
Relationships Family object list.
:return: None
:rtype: ``None``
"""
INDICATOR = ["Indicator"]
@staticmethod
def is_valid_type(_type):
# type: (str) -> bool
return _type in EntityRelationship.RelationshipsFamily.INDICATOR
class Relationships(object):
"""
Enum: Relations names and their reverse
:return: None
:rtype: ``None``
"""
APPLIED = 'applied'
ATTACHMENT_OF = 'attachment-of'
ATTACHES = 'attaches'
ATTRIBUTE_OF = 'attribute-of'
ATTRIBUTED_BY = 'attributed-by'
ATTRIBUTED_TO = 'attributed-to'
AUTHORED_BY = 'authored-by'
BEACONS_TO = 'beacons-to'
BUNDLED_IN = 'bundled-in'
BUNDLES = 'bundles'
COMMUNICATED_WITH = 'communicated-with'
COMMUNICATED_BY = 'communicated-by'
COMMUNICATES_WITH = 'communicates-with'
COMPROMISES = 'compromises'
CONTAINS = 'contains'
CONTROLS = 'controls'
CREATED_BY = 'created-by'
CREATES = 'creates'
DELIVERED_BY = 'delivered-by'
DELIVERS = 'delivers'
DETECTS = 'detects'
DETECTED_BY = 'detected-by'
DOWNLOADS = 'downloads'
DOWNLOADS_FROM = 'downloads-from'
DROPPED_BY = 'dropped-by'
DROPS = 'drops'
DUPLICATE_OF = 'duplicate-of'
EMBEDDED_IN = 'embedded-in'
EMBEDS = 'embeds'
EXECUTED = 'executed'
EXECUTED_BY = 'executed-by'
EXFILTRATES_TO = 'exfiltrates-to'
EXPLOITS = 'exploits'
HAS = 'has'
HOSTED_ON = 'hosted-on'
HOSTS = 'hosts'
IMPERSONATES = 'impersonates'
INDICATED_BY = 'indicated-by'
INDICATOR_OF = 'indicator-of'
INJECTED_FROM = 'injected-from'
INJECTS_INTO = 'injects-into'
INVESTIGATES = 'investigates'
IS_ALSO = 'is-also'
LOCATED_AT = 'located-at'
MITIGATED_BY = 'mitigated-by'
MITIGATES = 'mitigates'
ORIGINATED_FROM = 'originated-from'
OWNED_BY = 'owned-by'
OWNS = 'owns'
PART_OF = 'part-of'
RELATED_TO = 'related-to'
REMEDIATES = 'remediates'
RESOLVED_BY = 'resolved-by'
RESOLVED_FROM = 'resolved-from'
RESOLVES_TO = 'resolves-to'
SEEN_ON = 'seen-on'
SENT = 'sent'
SENT_BY = 'sent-by'
SENT_FROM = 'sent-from'
SENT_TO = 'sent-to'
SIMILAR_TO = 'similar-to'
SUB_DOMAIN_OF = 'sub-domain-of'
SUB_TECHNIQUE_OF = 'subtechnique-of'
PARENT_TECHNIQUE_OF = 'parent-technique-of'
SUPRA_DOMAIN_OF = 'supra-domain-of'
TARGETED_BY = 'targeted-by'
TARGETS = 'targets'
TYPES = 'Types'
UPLOADED_TO = 'uploaded-to'
USED_BY = 'used-by'
USED_ON = 'used-on'
USES = 'uses'
VARIANT_OF = 'variant-of'
RELATIONSHIPS_NAMES = {'applied': 'applied-on',
'attachment-of': 'attaches',
'attaches': 'attachment-of',
'attribute-of': 'owns',
'attributed-by': 'attributed-to',
'attributed-to': 'attributed-by',
'authored-by': 'author-of',
'beacons-to': 'communicated-by',
'bundled-in': 'bundles',
'bundles': 'bundled-in',
'communicated-with': 'communicated-by',
'communicated-by': 'communicates-with',
'communicates-with': 'communicated-by',
'compromises': 'compromised-by',
'contains': 'part-of',
'controls': 'controlled-by',
'created-by': 'creates',
'creates': 'created-by',
'delivered-by': 'delivers',
'delivers': 'delivered-by',
'detects': 'detected-by',
'detected-by': 'detects',
'downloads': 'downloaded-by',
'downloads-from': 'hosts',
'dropped-by': 'drops',
'drops': 'dropped-by',
'duplicate-of': 'duplicate-of',
'embedded-in': 'embeds',
'embeds': 'embedded-on',
'executed': 'executed-by',
'executed-by': 'executes',
'exfiltrates-to': 'exfiltrated-from',
'exploits': 'exploited-by',
'has': 'seen-on',
'hosted-on': 'hosts',
'hosts': 'hosted-on',
'impersonates': 'impersonated-by',
'indicated-by': 'indicator-of',
'indicator-of': 'indicated-by',
'injected-from': 'injects-into',
'injects-into': 'injected-from',
'investigates': 'investigated-by',
'is-also': 'is-also',
'mitigated-by': 'mitigates',
'mitigates': 'mitigated-by',
'originated-from': 'source-of',
'owned-by': 'owns',
'owns': 'owned-by',
'part-of': 'contains',
'related-to': 'related-to',
'remediates': 'remediated-by',
'resolved-by': 'resolves-to',
'resolved-from': 'resolves-to',
'resolves-to': 'resolved-from',
'seen-on': 'has',
'sent': 'attached-to',
'sent-by': 'sent',
'sent-from': 'received-by',
'sent-to': 'received-by',
'similar-to': 'similar-to',
'sub-domain-of': 'supra-domain-of',
'supra-domain-of': 'sub-domain-of',
'subtechnique-of': 'parent-technique-of',
'parent-technique-of': 'subtechnique-of',
'targeted-by': 'targets',
'targets': 'targeted-by',
'Types': 'Reverse',
'uploaded-to': 'hosts',
'used-by': 'uses',
'used-on': 'targeted-by',
'uses': 'used-by',
'variant-of': 'variant-of'}
@staticmethod
def is_valid(_type):
"""
:type _type: ``str``
:param _type: the data to be returned and will be set to context
:return: Is the given type supported
:rtype: ``bool``
"""
return _type in EntityRelationship.Relationships.RELATIONSHIPS_NAMES.keys()
@staticmethod
def get_reverse(name):
"""
:type name: ``str``
:param name: Relationship name
:return: Returns the reversed relationship name
:rtype: ``str``
"""
try:
return EntityRelationship.Relationships.RELATIONSHIPS_NAMES[name]
except KeyError:
demisto.debug('Cannot find a reverse name for relationship name, using "related-to" instead.')
return EntityRelationship.Relationships.RELATED_TO
def __init__(self, name, entity_a, entity_a_type, entity_b, entity_b_type,
reverse_name='', relationship_type='IndicatorToIndicator', entity_a_family='Indicator',
entity_b_family='Indicator', source_reliability="", fields=None, brand=""):
# Relationship
if not EntityRelationship.Relationships.is_valid(name):
demisto.debug("Unknown relationship name: " + name)
self._name = name
if reverse_name:
if not EntityRelationship.Relationships.is_valid(reverse_name):
demisto.debug("Unknown reverse relationship name: " + reverse_name)
self._reverse_name = reverse_name
else:
self._reverse_name = EntityRelationship.Relationships.get_reverse(name)
if not EntityRelationship.RelationshipsTypes.is_valid_type(relationship_type):
raise ValueError("Invalid relationship type: " + relationship_type)
self._relationship_type = relationship_type
# Entity A - Source
self._entity_a = entity_a
self._entity_a_type = entity_a_type
if not EntityRelationship.RelationshipsFamily.is_valid_type(entity_a_family):
raise ValueError("Invalid entity A Family type: " + entity_a_family)
self._entity_a_family = entity_a_family
# Entity B - Destination
if not entity_b:
demisto.info(
"WARNING: Invalid entity B - Relationships will not be created to entity A {} with relationship name {}".format(
str(entity_a), str(name)))
self._entity_b = entity_b
self._entity_b_type = entity_b_type
if not EntityRelationship.RelationshipsFamily.is_valid_type(entity_b_family):
raise ValueError("Invalid entity B Family type: " + entity_b_family)
self._entity_b_family = entity_b_family
# Custom fields
if fields:
self._fields = fields
else:
self._fields = {}
# Source
if brand:
self._brand = brand
else:
self._brand = ''
if source_reliability:
if not DBotScoreReliability.is_valid_type(source_reliability):
raise ValueError("Invalid source reliability value", source_reliability)
self._source_reliability = source_reliability
else:
self._source_reliability = ''
def to_entry(self):
""" Convert object to XSOAR entry
:return: XSOAR entry representation.
:rtype: ``dict``
"""
entry = {}
if self._entity_b:
entry = {
"name": self._name,
"reverseName": self._reverse_name,
"type": self._relationship_type,
"entityA": self._entity_a,
"entityAFamily": self._entity_a_family,
"entityAType": self._entity_a_type,
"entityB": self._entity_b,
"entityBFamily": self._entity_b_family,
"entityBType": self._entity_b_type,
"fields": self._fields,
}
if self._source_reliability:
entry["reliability"] = self._source_reliability
if self._brand:
entry["brand"] = self._brand
return entry
def to_indicator(self):
""" Convert object to XSOAR entry
:return: XSOAR entry representation.
:rtype: ``dict``
"""
indicator_relationship = {}
if self._entity_b:
indicator_relationship = {
"name": self._name,
"reverseName": self._reverse_name,
"type": self._relationship_type,
"entityA": self._entity_a,
"entityAFamily": self._entity_a_family,
"entityAType": self._entity_a_type,
"entityB": self._entity_b,
"entityBFamily": self._entity_b_family,
"entityBType": self._entity_b_type,
"fields": self._fields,
}
return indicator_relationship
def to_context(self):
""" Convert object to XSOAR context
:return: XSOAR context representation.
:rtype: ``dict``
"""
indicator_relationship_context = {}
if self._entity_b:
indicator_relationship_context = {
"Relationship": self._name,
"EntityA": self._entity_a,
"EntityAType": self._entity_a_type,
"EntityB": self._entity_b,
"EntityBType": self._entity_b_type,
}
return indicator_relationship_context
class CommandResults:
"""
CommandResults class - use to return results to warroom
:type outputs_prefix: ``str``
:param outputs_prefix: should be identical to the prefix in the yml contextPath in yml file. for example:
CortexXDR.Incident
:type outputs_key_field: ``str`` or ``list[str]``
:param outputs_key_field: primary key field in the main object. If the command returns Incidents, and of the
properties of Incident is incident_id, then outputs_key_field='incident_id'. If object has multiple
unique keys, then list of strings is supported outputs_key_field=['id1', 'id2']
:type outputs: ``list`` or ``dict``
:param outputs: the data to be returned and will be set to context
:type indicators: ``list``
:param indicators: DEPRECATED: use 'indicator' instead.
:type indicator: ``Common.Indicator``
:param indicator: single indicator like Common.IP, Common.URL, Common.File, etc.
:type readable_output: ``str``
:param readable_output: (Optional) markdown string that will be presented in the warroom, should be human readable -
(HumanReadable) - if not set, readable output will be generated
:type raw_response: ``dict`` | ``list``
:param raw_response: must be dictionary, if not provided then will be equal to outputs. usually must be the original
raw response from the 3rd party service (originally Contents)
:type indicators_timeline: ``IndicatorsTimeline``
:param indicators_timeline: must be an IndicatorsTimeline. used by the server to populate an indicator's timeline.
:type ignore_auto_extract: ``bool``
:param ignore_auto_extract: must be a boolean, default value is False. Used to prevent AutoExtract on output.
:type relationships: ``list of EntityRelationship``
:param relationships: List of relationships of the indicator.
:type mark_as_note: ``bool``
:param mark_as_note: must be a boolean, default value is False. Used to mark entry as note.
:type tags: ``list``
:param tags: must be a list, default value is None. Used to tag war room entries.
:type entry_type: ``int`` code of EntryType
:param entry_type: type of return value, see EntryType
:type scheduled_command: ``ScheduledCommand``
:param scheduled_command: manages the way the command should be polled.
:type extended_payload: ``dict``
:param extended_payload: (Optional) A dictionary representing the contents of ExtendedPayload for synchronization.
:type execution_metrics: ``ExecutionMetrics``
:param execution_metrics: contains metric data about a command's execution
:type replace_existing: ``bool``
:param replace_existing: Replace the context value at outputs_prefix if it exists.
Works only if outputs_prefix is a path to a nested value i.e., contains a period.
For example, the "next token" result should always be overwritten. This response can be returned as follows:
>>> CommandResults(
>>> readable_output=f'Next Token: {next_token}',
>>> outputs=next_token,
>>> outputs_prefix='Path.To.NextToken',
>>> replace_existing=True,
>>> )
:return: None
:rtype: ``None``
"""
def __init__(self, outputs_prefix=None,
outputs_key_field=None,
outputs=None,
indicators=None,
readable_output=None,
raw_response=None,
indicators_timeline=None,
indicator=None,
ignore_auto_extract=False,
mark_as_note=False,
tags=None,
scheduled_command=None,
relationships=None,
entry_type=None,
content_format=None,
extended_payload=None,
execution_metrics=None,
replace_existing=False):
# type: (str, object, object, list, str, object, IndicatorsTimeline, Common.Indicator, bool, bool, List[str], ScheduledCommand, list, int, str, dict, List[Any], bool) -> None # noqa: E501
if raw_response is None:
raw_response = outputs
if outputs is not None:
if not isinstance(outputs, dict) and not outputs_prefix:
raise ValueError('outputs_prefix is missing')
if outputs_prefix == '.':
raise ValueError('outputs_prefix cannot be a period.')
if indicators and indicator:
raise ValueError('indicators is DEPRECATED, use only indicator')
if entry_type is None:
entry_type = EntryType.NOTE
self.indicators = indicators # type: Optional[List[Common.Indicator]]
self.indicator = indicator # type: Optional[Common.Indicator]
self.entry_type = entry_type # type: int
self.outputs_prefix = outputs_prefix
# this is public field, it is used by a lot of unit tests, so I don't change it
self.outputs_key_field = outputs_key_field
self._outputs_key_field = None # type: Optional[List[str]]
if not outputs_key_field:
self._outputs_key_field = None
elif isinstance(outputs_key_field, STRING_TYPES):
self._outputs_key_field = [outputs_key_field] # type: ignore[list-item]
elif isinstance(outputs_key_field, list):
self._outputs_key_field = outputs_key_field
else:
raise TypeError('outputs_key_field must be of type str or list')
self.outputs = outputs
self.raw_response = raw_response
self.readable_output = readable_output
self.indicators_timeline = indicators_timeline
self.ignore_auto_extract = ignore_auto_extract
self.mark_as_note = mark_as_note
self.tags = tags
self.scheduled_command = scheduled_command
self.relationships = relationships
self.extended_payload = extended_payload
self.execution_metrics = execution_metrics
self.replace_existing = replace_existing
if content_format is not None and not EntryFormat.is_valid_type(content_format):
raise TypeError('content_format {} is invalid, see CommonServerPython.EntryFormat'.format(content_format))
self.content_format = content_format
def to_context(self):
outputs = {} # type: dict
relationships = [] # type: list
tags = [] # type: list
if self.readable_output:
human_readable = self.readable_output
else:
human_readable = None # type: ignore[assignment]
raw_response = self.raw_response # type: ignore[assignment]
indicators_timeline = [] # type: ignore[assignment]
ignore_auto_extract = False # type: bool
mark_as_note = False # type: bool
exec_metrics = None # type: ignore[assignment]
indicators = [self.indicator] if self.indicator else self.indicators
if indicators:
# Get InsightCacheSize from demisto.callingContext (in KB, default 3 MB = 3072 KB)
insight_cache_size_kb = demisto.callingContext.get("context", {}).get("InsightCacheSize", DEFAULT_INSIGHT_CACHE_SIZE)
insight_cache_size_bytes = insight_cache_size_kb * 1024
virtual_outputs = {}
for indicator in indicators:
context_outputs = indicator.to_context()
for key, value in context_outputs.items():
if key not in virtual_outputs:
virtual_outputs[key] = []
virtual_outputs[key].append(value)
context_size = len(json.dumps(virtual_outputs, default=str, separators=JSON_SEPARATORS, ensure_ascii=False))
# If the context size exceeds the limit, use to_minimum_context() instead
if context_size > insight_cache_size_bytes:
# Measure performance of to_minimum_context() - minimal context generation
for indicator in indicators:
context_outputs = indicator.to_minimum_context()
for key, value in context_outputs.items():
if key not in outputs:
outputs[key] = []
outputs[key].append(value)
if human_readable:
human_readable += "\nNote! some of the context data was not included because it went over the {}KB limit.".format(insight_cache_size_kb)
else:
human_readable = "Note! some of the context data was not included because it went over the {}KB limit.".format(insight_cache_size_kb)
demisto.debug(
"Context size ({} chars) exceeded limit ({} bytes). Will use the minimum context.".format(context_size, insight_cache_size_bytes)
)
else:
# Use the already calculated full context
outputs = virtual_outputs
demisto.debug(
"Using full context ({} chars within {} bytes limit). ".format(context_size, insight_cache_size_bytes)
)
if self.tags:
tags = self.tags # type: ignore[assignment]
if self.ignore_auto_extract:
ignore_auto_extract = True
if self.mark_as_note:
mark_as_note = True
if self.indicators_timeline:
indicators_timeline = self.indicators_timeline.indicators_timeline
if self.outputs is not None and self.outputs != []:
if not self.readable_output:
# if markdown is not provided then create table by default
if isinstance(self.outputs, (dict, list)):
human_readable = tableToMarkdown('Results', self.outputs)
else:
human_readable = self.outputs # type: ignore[assignment]
if self.outputs_prefix and self.replace_existing:
next_token_path, _, next_token_key = self.outputs_prefix.rpartition('.')
if not next_token_path:
raise DemistoException('outputs_prefix must be a nested path to replace an existing key.')
outputs[next_token_path + '(true)'] = {next_token_key: self.outputs}
elif self.outputs_prefix and self._outputs_key_field:
# if both prefix and key field provided then create DT key
formatted_outputs_key = ' && '.join(['val.{0} && val.{0} == obj.{0}'.format(key_field)
for key_field in self._outputs_key_field])
outputs_key = '{0}({1})'.format(self.outputs_prefix, formatted_outputs_key)
outputs[outputs_key] = self.outputs
elif self.outputs_prefix:
outputs[str(self.outputs_prefix)] = self.outputs
else:
outputs.update(self.outputs) # type: ignore[call-overload]
if self.relationships:
relationships = [relationship.to_entry() for relationship in self.relationships if relationship.to_entry()]
# using a local variable to avoid changing the object's attribute, see discussion on PR #18544.
content_format = self.content_format
if content_format is None:
if isinstance(raw_response, STRING_TYPES + (int,)):
content_format = EntryFormat.TEXT
else:
content_format = EntryFormat.JSON
if self.execution_metrics:
exec_metrics = self.execution_metrics
self.entry_type = EntryType.EXECUTION_METRICS
raw_response = 'Metrics reported successfully.'
content_format = EntryFormat.TEXT
return_entry = {
'Type': self.entry_type,
'ContentsFormat': content_format,
'Contents': raw_response,
'HumanReadable': human_readable,
'EntryContext': outputs,
'IndicatorTimeline': indicators_timeline,
'IgnoreAutoExtract': bool(ignore_auto_extract),
'Note': mark_as_note,
'Relationships': relationships
}
if self.extended_payload:
return_entry['ExtendedPayload'] = self.extended_payload
if tags:
# This is for backward compatibility reasons
return_entry['Tags'] = tags
if self.scheduled_command:
return_entry.update(self.scheduled_command.to_results())
if exec_metrics:
return_entry.update({'APIExecutionMetrics': exec_metrics})
return return_entry
def is_integration_command_execution():
"""
This function determines whether the current execution a script execution or a integration command execution.
:return: Is the current execution a script execution or a integration command execution.
:rtype: ``bool``
"""
try:
return demisto.callingContext['context']['ExecutedCommands'][0]['moduleBrand'] != 'Scripts'
except (KeyError, IndexError, TypeError):
try:
# In dynamic-section scripts ExecutedCommands is None and another way is needed to verify if we are in a Script.
return demisto.callingContext['context']['ScriptName'] == ''
except (KeyError, IndexError, TypeError):
return True
EXECUTION_METRICS_SCRIPT_SKIP_MSG = "returning results with Type=EntryType.EXECUTION_METRICS isn't fully supported for scripts. dropping result."
def return_results(results):
"""
This function wraps the demisto.results(), supports.
:type results: ``CommandResults`` or ``PollResult`` or ``str`` or ``dict`` or ``BaseWidget`` or ``list`` or ``GetMappingFieldsResponse`` or ``GetModifiedRemoteDataResponse`` or ``GetRemoteDataResponse``
:param results: A result object to return as a War-Room entry.
:return: None
:rtype: ``None``
"""
is_integration = is_integration_command_execution()
if results is None:
# backward compatibility reasons
demisto.results(None)
return
elif results and isinstance(results, list):
result_list = [] # type: list
for result in results:
# Results of type dict or str are of the old results format and work with demisto.results()
if isinstance(result, dict):
if is_integration or result.get('Type') != EntryType.EXECUTION_METRICS:
result_list.append(result)
else:
demisto.debug(EXECUTION_METRICS_SCRIPT_SKIP_MSG)
elif isinstance(result, str):
result_list.append(result)
else:
# The rest are of the new format and have a corresponding function (to_context, to_display, etc...)
return_results(result)
if result_list:
demisto.results(result_list)
elif isinstance(results, CommandResults):
context = results.to_context()
if is_integration or context.get('Type') != EntryType.EXECUTION_METRICS:
demisto.results(context)
else:
demisto.debug(EXECUTION_METRICS_SCRIPT_SKIP_MSG)
elif isinstance(results, BaseWidget):
demisto.results(results.to_display())
elif isinstance(results, GetMappingFieldsResponse):
demisto.results(results.extract_mapping())
elif isinstance(results, GetRemoteDataResponse):
demisto.results(results.extract_for_local())
elif isinstance(results, GetModifiedRemoteDataResponse):
demisto.results(results.to_entry())
elif hasattr(results, 'to_entry'):
entry = results.to_entry()
if is_integration or entry.get('Type') != EntryType.EXECUTION_METRICS:
demisto.results(entry)
else:
demisto.debug(EXECUTION_METRICS_SCRIPT_SKIP_MSG)
elif not is_integration and isinstance(results, dict) and results.get('Type') == EntryType.EXECUTION_METRICS:
demisto.debug(EXECUTION_METRICS_SCRIPT_SKIP_MSG)
else:
demisto.results(results)
# deprecated
def return_outputs(readable_output, outputs=None, raw_response=None, timeline=None, ignore_auto_extract=False):
"""
DEPRECATED: use return_results() instead
This function wraps the demisto.results(), makes the usage of returning results to the user more intuitively.
:type readable_output: ``str`` | ``int``
:param readable_output: markdown string that will be presented in the warroom, should be human readable -
(HumanReadable)
:type outputs: ``dict``
:param outputs: the outputs that will be returned to playbook/investigation context (originally EntryContext)
:type raw_response: ``dict`` | ``list`` | ``str``
:param raw_response: must be dictionary, if not provided then will be equal to outputs. usually must be the original
raw response from the 3rd party service (originally Contents)
:type timeline: ``dict`` | ``list``
:param timeline: expects a list, if a dict is passed it will be put into a list. used by server to populate an
indicator's timeline. if the 'Category' field is not present in the timeline dict(s), it will automatically
be be added to the dict(s) with its value set to 'Integration Update'.
:type ignore_auto_extract: ``bool``
:param ignore_auto_extract: expects a bool value. if true then the warroom entry readable_output will not be auto enriched.
:return: None
:rtype: ``None``
"""
timeline_list = [timeline] if isinstance(timeline, dict) else timeline
if timeline_list:
for tl_obj in timeline_list:
if 'Category' not in tl_obj.keys():
tl_obj['Category'] = 'Integration Update'
return_entry = {
"Type": entryTypes["note"],
"HumanReadable": readable_output,
"ContentsFormat": formats["text"] if isinstance(raw_response, STRING_TYPES) else formats['json'],
"Contents": raw_response,
"EntryContext": outputs,
'IgnoreAutoExtract': ignore_auto_extract,
"IndicatorTimeline": timeline_list
}
# Return 'readable_output' only if needed
if readable_output and not outputs and not raw_response:
return_entry["Contents"] = readable_output
return_entry["ContentsFormat"] = formats["text"]
elif outputs and raw_response is None:
# if raw_response was not provided but outputs were provided then set Contents as outputs
return_entry["Contents"] = outputs
demisto.results(return_entry)
def _get_auto_error_message(error):
"""Return the automatic (unified) error message for a content error, if any.
Looks for a :class:`CortexError` first in the ``error`` argument and then
in the currently-handled exception, and returns its automatic, unified
message built by :meth:`CortexError.auto_message` - which is built purely
from the error's structured arguments and *always ignores* any custom
message supplied by the caller (so Agentix always gets the standardized
message).
:type error: ``str`` or ``Exception``
:param error: The ``error`` value passed to :func:`return_error`.
:return: The automatic message, or ``None`` if no content error is available.
:rtype: ``str`` or ``None``
"""
content_error = None
if isinstance(error, CortexError):
content_error = error
else:
exc = sys.exc_info()[1]
if isinstance(exc, CortexError):
content_error = exc
if content_error is None:
return None
try:
return content_error.auto_message()
except Exception as exc:
demisto.debug('_get_auto_error_message failed to build message: {}'.format(exc))
return str(content_error)
def _select_error_message(message, error):
"""Choose which message ``return_error`` should surface.
Selection rules:
* **Agentix caller** -> always use the automatic, unified message built
from the :class:`CortexError` (when one is available); otherwise fall
back to the explicitly-passed ``message``.
* **Non-Agentix caller** -> prefer the explicitly-passed ``message``; if
none was provided, fall back to the automatic message.
:type message: ``str``
:param message: The explicit message passed to :func:`return_error`.
:type error: ``str`` or ``Exception``
:param error: The ``error`` value passed to :func:`return_error`.
:return: The message to display in the error entry.
:rtype: ``str``
"""
auto_message = _get_auto_error_message(error)
if is_caller_agentix():
# Agentix prefers the machine-friendly, standardized message.
return auto_message or message
# Non-Agentix: respect an explicit message, otherwise use the auto one.
if message:
return message
return auto_message or message
def return_error(message, error='', outputs=None):
"""
Returns error entry with given message and exits the script
:type message: ``str``
:param message: The message to return to the entry (required).
Message selection: when the caller is Agentix
(see :func:`is_caller_agentix`), the automatic, unified message
built from the :class:`CortexError` is used (falling back to this
``message`` if none is available). Otherwise this explicit
``message`` is used, falling back to the automatic message when it
is empty.
:type error: ``str`` or Exception
:param error: The raw error message to log (optional).
When *error* is a :class:`CortexError`, its ``error_code`` is
automatically attached to the error entry via ``ExtendedPayload``
and its :meth:`CortexError.build_message` provides the automatic,
unified message used for Agentix callers.
:type outputs: ``dict or None``
:param outputs: the outputs that will be returned to playbook/investigation context (optional)
:return: Error entry object
:rtype: ``dict``
"""
is_command = hasattr(demisto, 'command')
try:
is_server_handled = is_command and (demisto.command() in FETCH_COMMANDS or demisto.command() == LONG_RUNNING_COMMAND)
except Exception:
is_server_handled = False
# Decide which message to surface based on the caller (Agentix vs. others)
# and whether a CortexError with an automatic, unified message is present.
message = _select_error_message(message, error)
message = LOG(message)
if error:
LOG(str(error))
if any(sys.exc_info()): # Checking that an exception occurred
fixed_traceback = fix_traceback_line_numbers(traceback.format_exc())
LOG('\n{}'.format(fixed_traceback))
if is_debug_mode():
message = '{}\n\n{}'.format(message, fixed_traceback)
LOG.print_log()
if not isinstance(message, str):
message = message.encode('utf8') if hasattr(message, 'encode') else str(message)
if is_command and demisto.command() == 'get-modified-remote-data':
if (error and not isinstance(error, NotImplementedError)) or sys.exc_info()[0] != NotImplementedError:
message = 'skip update. error: ' + message
if is_server_handled:
raise Exception(message)
else:
if len(message) > MAX_ERROR_MESSAGE_LENGTH:
half_length = MAX_ERROR_MESSAGE_LENGTH // 2
message = message[:half_length] + "...This error body was truncated..." + message[half_length * (-1):]
error_entry = {
'Type': entryTypes['error'],
'ContentsFormat': formats['text'],
'Contents': message,
'EntryContext': outputs,
}
# Attach the content error type (and, when available, the retryable
# flag) inside ExtendedPayload so LLM agents can classify the failure
# and decide whether to retry, without parsing the human-readable
# Contents string.
# Priority for the CortexError source: error param -> current exception.
_cortex_error = error if isinstance(error, CortexError) else None
if _cortex_error is None:
exc = sys.exc_info()[1]
if isinstance(exc, CortexError):
_cortex_error = exc
# Error code: CortexError first, then a text-based heuristic fallback.
_error_type = _cortex_error.error_code if _cortex_error else None
if not _error_type:
_error_type = _classify_error_message(message)
extended_payload = {}
if _error_type:
extended_payload[EXTENDED_PAYLOAD_ERROR_CODE_KEY] = _error_type
# Expose only the boolean retryable flag (true/false), taken from the
# CortexError details.
if _cortex_error is not None and EXTENDED_PAYLOAD_RETRYABLE_KEY in _cortex_error.details:
extended_payload[EXTENDED_PAYLOAD_RETRYABLE_KEY] = \
_cortex_error.details[EXTENDED_PAYLOAD_RETRYABLE_KEY]
if extended_payload:
error_entry['ExtendedPayload'] = extended_payload
demisto.results(error_entry)
sys.exit(0)
def return_warning(message, exit=False, warning='', outputs=None, ignore_auto_extract=False):
"""
Returns a warning entry with the specified message, and exits the script.
:type message: ``str``
:param message: The message to return in the entry (required).
:type exit: ``bool``
:param exit: Determines if the program will terminate after the command is executed. Default is False.
:type warning: ``str``
:param warning: The warning message (raw) to log (optional).
:type outputs: ``dict or None``
:param outputs: The outputs that will be returned to playbook/investigation context (optional).
:type ignore_auto_extract: ``bool``
:param ignore_auto_extract: Determines if the War Room entry will be auto-enriched. Default is false.
:return: Warning entry object
:rtype: ``dict``
"""
LOG(message)
if warning:
LOG(warning)
LOG.print_log()
demisto.results({
'Type': entryTypes['warning'],
'ContentsFormat': formats['text'],
'IgnoreAutoExtract': ignore_auto_extract,
'Contents': str(message),
"EntryContext": outputs
})
if exit:
sys.exit(0)
class ExecutionMetrics(object):
"""
ExecutionMetrics is used to collect and format metric data to be reported to the XSOAR server.
:return: None
:rtype: ``None``
"""
def __init__(self, success=0, quota_error=0, general_error=0, auth_error=0, service_error=0, connection_error=0,
proxy_error=0, ssl_error=0, timeout_error=0, retry_error=0):
self._metrics = []
self.metrics = None
self.success = success
self.quota_error = quota_error
self.general_error = general_error
self.auth_error = auth_error
self.service_error = service_error
self.connection_error = connection_error
self.proxy_error = proxy_error
self.ssl_error = ssl_error
self.timeout_error = timeout_error
self.retry_error = retry_error
"""
Initializes an ExecutionMetrics object. Once initialized, you may increment each metric type according to the
metric you'd like to report. Afterwards, pass the `metrics` value to CommandResults.
:type success: ``int``
:param success: Quantity of Successful metrics
:type quota_error: ``int``
:param quota_error: Quantity of Quota Error (Rate Limited) metrics
:type general_error: ``int``
:param general_error: Quantity of General Error metrics
:type auth_error: ``int``
:param auth_error: Quantity of Authentication Error metrics
:type service_error: ``int``
:param service_error: Quantity of Service Error metrics
:type connection_error: ``int``
:param connection_error: Quantity of Connection Error metrics
:type proxy_error: ``int``
:param proxy_error: Quantity of Proxy Error metrics
:type ssl_error: ``int``
:param ssl_error: Quantity of SSL Error metrics
:type timeout_error: ``int``
:param timeout_error: Quantity of Timeout Error metrics
:type retry_error: ``int``
:param retry_error: Quantity of Retry Error metrics
:type metrics: ``CommandResults``
:param metrics: Append this value to your CommandResults list to report the metrics to your server.
"""
@staticmethod
def is_supported():
if is_demisto_version_ge('6.8.0'):
return True
return False
@property
def success(self):
return self._success
@success.setter
def success(self, value):
self._success = value
self.update_metrics('Successful', self._success)
@property
def quota_error(self):
return self._quota_error
@quota_error.setter
def quota_error(self, value):
self._quota_error = value
self.update_metrics(ErrorTypes.QUOTA_ERROR, self._quota_error)
@property
def general_error(self):
return self._general_error
@general_error.setter
def general_error(self, value):
self._general_error = value
self.update_metrics(ErrorTypes.GENERAL_ERROR, self._general_error)
@property
def auth_error(self):
return self._auth_error
@auth_error.setter
def auth_error(self, value):
self._auth_error = value
self.update_metrics(ErrorTypes.AUTH_ERROR, self._auth_error)
@property
def service_error(self):
return self._service_error
@service_error.setter
def service_error(self, value):
self._service_error = value
self.update_metrics(ErrorTypes.SERVICE_ERROR, self._service_error)
@property
def connection_error(self):
return self._connection_error
@connection_error.setter
def connection_error(self, value):
self._connection_error = value
self.update_metrics(ErrorTypes.CONNECTION_ERROR, self._connection_error)
@property
def proxy_error(self):
return self._proxy_error
@proxy_error.setter
def proxy_error(self, value):
self._proxy_error = value
self.update_metrics(ErrorTypes.PROXY_ERROR, self._proxy_error)
@property
def ssl_error(self):
return self._ssl_error
@ssl_error.setter
def ssl_error(self, value):
self._ssl_error = value
self.update_metrics(ErrorTypes.SSL_ERROR, self._ssl_error)
@property
def timeout_error(self):
return self._timeout_error
@timeout_error.setter
def timeout_error(self, value):
self._timeout_error = value
self.update_metrics(ErrorTypes.TIMEOUT_ERROR, self._timeout_error)
@property
def retry_error(self):
return self._retry_error
@retry_error.setter
def retry_error(self, value):
self._retry_error = value
self.update_metrics(ErrorTypes.RETRY_ERROR, self._retry_error)
def get_metric_list(self):
return self._metrics
def update_metrics(self, metric_type, metric_value):
if metric_value > 0:
if len(self._metrics) == 0:
self._metrics.append({'Type': metric_type, 'APICallsCount': metric_value})
else:
for metric in self._metrics:
if metric['Type'] == metric_type:
metric['APICallsCount'] = metric_value
break
else:
self._metrics.append({'Type': metric_type, 'APICallsCount': metric_value})
self.metrics = CommandResults(execution_metrics=self._metrics)
def append_metrics(execution_metrics, results):
"""
Returns a 'CommandResults' list appended with metrics.
:type execution_metrics: ``ExecutionMetrics``
:param execution_metrics: Metrics object to be added to CommandResults list(optional).
:type results: ``list``
:param results: 'CommandResults' list to append metrics to (required).
:return: results appended with the metrics if the server version is supported.
:rtype: ``list``
"""
if execution_metrics.metrics is not None and execution_metrics.is_supported():
results.append(execution_metrics.metrics)
return results
def convert_dict_values_bytes_to_str(input_dict): # type: ignore
"""
Converts byte dict values to str
:type input_dict: ``dict``
:param input_dict: dict to converts its values.
:return: dict contains str instead of bytes.
:rtype: ``dict``
"""
output_dict = {}
for key, value in input_dict.items():
if isinstance(value, dict):
output_dict[key] = convert_dict_values_bytes_to_str(value)
elif isinstance(value, bytes):
output_dict[key] = value.decode()
elif isinstance(value, list):
output_dict[key] = [
convert_dict_values_bytes_to_str(item) if isinstance(item, dict)
else item.decode() if isinstance(item, bytes) else item
for item in value]
else:
output_dict[key] = value
return output_dict
class CommandRunner:
"""
Class for executing multiple commands and save the results of each command.
:return: None
:rtype: ``None``
"""
class Command:
"""
Data class with the data required to execute a command.
:return: None
:rtype: ``None``
"""
def __init__(self, commands, args_lst, brand=None, instance=None):
"""
:param commands: The commands list or a single command
:type commands: str + List[str]
:param args_lst: The args list or a single args
:type args_lst: List[Dict] + Dict
:param brand: The brand to use
:type brand: str
:param instance: The instance to use
:type instance: str
"""
if isinstance(commands, str) and isinstance(args_lst, dict):
commands = [commands]
args_lst = [args_lst]
if isinstance(commands, str) and isinstance(args_lst, list):
commands = [commands for _ in range(len(args_lst))]
self._is_valid(commands, args_lst)
self.commands = commands
self.args_lst = args_lst
if brand:
for args in self.args_lst:
args.update({'using-brand': brand})
if instance:
for args in self.args_lst:
args.update({'using': instance})
@staticmethod
def _is_valid(commands, args_lst):
"""
Error handling of the given arguments
:param commands: list of commands
:param args_lst: list of args
:return:
"""
if not isinstance(commands, list):
raise DemistoException('Expected "commands" argument to be a list. received: {}'.format(type(commands)))
if not isinstance(args_lst, list):
raise DemistoException('Expected "args_lst" argument to be a list. received: {}'.format(type(args_lst)))
if len(commands) != len(args_lst):
raise DemistoException('"commands" and "args_lst" should be in the same size')
class Result:
"""
Class for the result of the command.
:return: None
:rtype: ``None``
"""
def __init__(self, command, args, brand, instance, result):
"""
:param command: command that was run.
:type command ``str``
:param args: args that was run.
:type args: ``dict``
:param brand: The brand that was used.
:type brand: ``str``
:param instance: The instance that was used.
:type instance: ``str``
:param result: The result of the command.
:type result: ``object``
"""
self.command = command
self.args = args
self.brand = brand
self.instance = instance
self.result = result
@staticmethod
def execute_commands(command, extract_contents=True):
"""
Runs the `demisto.executeCommand()` and gets all the results, including the errors, returned from the command.
:type command: ``Command``
:param command: The commands to run. (required)
:type extract_contents: ``bool``
:param extract_contents: Whether to extract Contents part of the results. Default is True.
:return: A tuple of two lists: the command results list and command errors list.
:rtype: ``Tuple[List[ResultWrapper], List[ResultWrapper]]``
"""
results, errors = [], []
for command, args in zip(command.commands, command.args_lst):
try:
demisto.debug(' '.join(('calling', command, 'with args=', ','.join(args))))
execute_command_results = demisto.executeCommand(command, args)
for res in execute_command_results:
brand_name = res.get('Brand', 'Unknown') if isinstance(res, dict) else 'Unknown'
module_name = res.get('ModuleName', 'Unknown') if isinstance(res, dict) else 'Unknown'
if is_error(res):
error = get_error(res)
demisto.debug('error: ' + error)
errors.append(CommandRunner.Result(command, args, brand_name, module_name, error))
else:
if extract_contents:
res = res.get('Contents', {})
if res is None:
res = {}
results.append(CommandRunner.Result(command, args, brand_name, module_name, res))
except ValueError as e:
# We expect this error when the command is not supported.
demisto.debug('demisto.executeCommand received an error because the command is not supported:'
' {e}'.format(e=str(e)))
return results, errors
@staticmethod
def run_commands_with_summary(commands):
"""
Given a list of commands, return a list of results (to pass to return_results).
In addition, it will create a `CommandResult` of the summary of the commands, which is a `readable_output`.
:param commands: A list of commands.
:type commands: ``List[Command]``
:return: list of results
"""
full_results, full_errors = [], []
for command in commands:
results, errors = CommandRunner.execute_commands(command, extract_contents=False)
full_results.extend(results)
full_errors.extend(errors)
summary_md = CommandRunner.get_results_summary(full_results, full_errors)
command_results = []
for res in full_results:
if isinstance(res.result, dict) and res.result.get('HumanReadable'):
res.result['HumanReadable'] = "***{brand} ({instance})***\n{human_readable}".format(
brand=res.brand,
instance=res.instance,
human_readable=res.result.get('HumanReadable'))
command_results.append(res.result)
if not command_results:
if full_errors: # no results were given but there are errors
errors = ["{instance}: {msg}".format(instance=err.instance, msg=err.result) for err in full_errors]
error_msg = '\n'.join(['Script failed. The following errors were encountered: '] + errors)
else:
error_msg = 'The commands that run are not supported in this Instance. ' \
'Try to configure the integrations in XSOAR settings.'
raise DemistoException(error_msg)
command_results.append(CommandResults(readable_output=summary_md))
return command_results
@staticmethod
def get_results_summary(results, errors):
"""
Get a Human Readable result for all the results of the commands.
:param results: list of returned results
:param errors: list of returned errors
:return: `CommandResults` of HumanReadable that summarizes the results and errors.
"""
results_summary_table = []
headers = ['Instance', 'Command', 'Result', 'Comment']
for res in results:
# don't care about using arg in command
res.args.pop('using', None)
res.args.pop('using-brand', None)
command = {'command': res.command,
'args': res.args}
comment = res.result.get('Contents', 'No contents found, see entry.') if isinstance(res.result, dict) else None
results_summary_table.append({'Instance': '***{brand}***: {instance}'.format(brand=res.brand,
instance=res.instance),
'Command': command,
'Result': 'Success',
'Comment': comment})
for err in errors:
# don't care about using arg in command
err.args.pop('using', None)
err.args.pop('using-brand', None)
command = {'command': err.command,
'args': err.args}
results_summary_table.append({'Instance': '***{brand}***: {instance}'.format(brand=err.brand,
instance=err.instance),
'Command': command,
'Result': 'Error',
'Comment': err.result})
summary_md = tableToMarkdown('Results Summary', results_summary_table, headers=headers, is_auto_json_transform=True)
return summary_md
def execute_command(command, args, extract_contents=True, fail_on_error=True):
"""
Runs the `demisto.executeCommand()` function and checks for errors.
:type command: ``str``
:param command: The command to run. (required)
:type args: ``dict``
:param args: The command arguments. (required)
:type extract_contents: ``bool``
:param extract_contents: Whether to return only the Contents part of the results. Default is True.
:type fail_on_error: ``bool``
:param fail_on_error: Whether to fail the command when receiving an error from the command. Default is True.
:return: The command results.
:rtype:
- When `fail_on_error` is True - ``list`` or ``dict`` or ``str``.
- When `fail_on_error` is False -``bool`` and ``str``.
Note:
For backward compatibility, only when `fail_on_error` is set to False, two values will be returned.
"""
if not hasattr(demisto, 'executeCommand'):
raise DemistoException('Cannot run demisto.executeCommand() from integrations.')
res = demisto.executeCommand(command, args)
if is_error(res):
error_message = get_error(res)
if fail_on_error:
# Reuse the standardized error code from the failing sub-command's
# ExtendedPayload when present, otherwise classify from the message.
error_code = get_error_code(res)
if not error_code:
error_code = _classify_error_message(error_message)
if error_code:
# Use the base CortexError so the standardized error_code taken
# from the failing sub-command (which may be any category, e.g.
# AUTH_ERROR/QUOTA_ERROR) is carried as-is.
raise CortexError(
# Original message kept for backward compatibility; the
# error_code carries the standardized classification.
override_message='Failed to execute {}. Error details:\n{}'.format(command, error_message),
error_code=error_code,
details={"command": command, "raw_error": error_message},
)
raise DemistoException('Failed to execute {}. Error details:\n{}'.format(command, error_message))
else:
return False, error_message
if not extract_contents:
if fail_on_error:
return res
else:
return True, res
contents = [entry.get('Contents', {}) for entry in res if entry['Type'] != entryTypes['debug']]
contents = contents[0] if len(contents) == 1 else contents
if fail_on_error:
return contents
return True, contents
def camelize(src, delim=' ', upper_camel=True):
"""
Convert all keys of a dictionary (or list of dictionaries) to CamelCase (with capital first letter)
:type src: ``dict`` or ``list``
:param src: The dictionary (or list of dictionaries) to convert the keys for. (required)
:type delim: ``str``
:param delim: The delimiter between two words in the key (e.g. delim=' ' for "Start Date"). Default ' '.
:type upper_camel: ``bool``
:param upper_camel: When True then transforms dictionary keys to camel case with the first letter capitalised
(for example: demisto_content to DemistoContent), otherwise the first letter will not be capitalised
(for example: demisto_content to demistoContent).
:return: The dictionary (or list of dictionaries) with the keys in CamelCase.
:rtype: ``dict`` or ``list``
"""
def camelize_str(src_str):
if callable(getattr(src_str, "decode", None)):
src_str = src_str.decode('utf-8')
components = src_str.split(delim)
camelize_without_first_char = ''.join(map(lambda x: x.title(), components[1:]))
if upper_camel:
return components[0].title() + camelize_without_first_char
else:
return components[0].lower() + camelize_without_first_char
if isinstance(src, list):
return [camelize(phrase, delim, upper_camel=upper_camel) for phrase in src]
return {camelize_str(key): value for key, value in src.items()}
# Constants for common merge paths
outputPaths = {
'file': 'File(val.MD5 && val.MD5 == obj.MD5 || val.SHA1 && val.SHA1 == obj.SHA1 || '
'val.SHA256 && val.SHA256 == obj.SHA256 || val.SHA512 && val.SHA512 == obj.SHA512 || '
'val.CRC32 && val.CRC32 == obj.CRC32 || val.CTPH && val.CTPH == obj.CTPH || '
'val.SSDeep && val.SSDeep == obj.SSDeep)',
'ip': 'IP(val.Address && val.Address == obj.Address)',
'url': 'URL(val.Data && val.Data == obj.Data)',
'domain': 'Domain(val.Name && val.Name == obj.Name)',
'cve': 'CVE(val.ID && val.ID == obj.ID)',
'email': 'Account.Email(val.Address && val.Address == obj.Address)',
'dbotscore': 'DBotScore'
}
def replace_in_keys(src, existing='.', new='_'):
"""
Replace a substring in all of the keys of a dictionary (or list of dictionaries)
:type src: ``dict`` or ``list``
:param src: The dictionary (or list of dictionaries) with keys that need replacement. (required)
:type existing: ``str``
:param existing: substring to replace.
:type new: ``str``
:param new: new substring that will replace the existing substring.
:return: The dictionary (or list of dictionaries) with keys after substring replacement.
:rtype: ``dict`` or ``list``
"""
def replace_str(src_str):
if callable(getattr(src_str, "decode", None)):
src_str = src_str.decode('utf-8')
return src_str.replace(existing, new)
if isinstance(src, list):
return [replace_in_keys(x, existing, new) for x in src]
return {replace_str(k): v for k, v in src.items()}
# ############################## REGEX FORMATTING ###############################
regexFlags = re.M # Multi line matching
# for the global(/g) flag use re.findall({regex_format},str)
# else, use re.match({regex_format},str)
ipv4Regex = r'^(?P<ipv4>(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?))[:]?(?P<port>\d+)?$' # noqa: E501
ipv4cidrRegex = r'^([0-9]{1,3}\.){3}[0-9]{1,3}(\/([0-9]|[1-2][0-9]|3[0-2]))$'
ipv6Regex = r'^(?:(?:[0-9a-fA-F]{1,4}:){7,7}[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,7}:|(?:[0-9a-fA-F]{1,4}:){1,6}:[0-9a-fA-F]{1,4}|(?:[0-9a-fA-F]{1,4}:){1,5}(:[0-9a-fA-F]{1,4}){1,2}|(?:[0-9a-fA-F]{1,4}:){1,4}(:[0-9a-fA-F]{1,4}){1,3}|(?:[0-9a-fA-F]{1,4}:){1,3}(:[0-9a-fA-F]{1,4}){1,4}|(?:[0-9a-fA-F]{1,4}:){1,2}(:[0-9a-fA-F]{1,4}){1,5}|[0-9a-fA-F]{1,4}:((:[0-9a-fA-F]{1,4}){1,6})|:(?:(:[0-9a-fA-F]{1,4}){1,7}|:)|fe80:(:[0-9a-fA-F]{0,4}){0,4}%[0-9a-zA-Z]{1,}|::(ffff(:0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])|([0-9a-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[0-9]){0,1}[0-9]))$' # noqa: E501
ipv6cidrRegex = r'^s*((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]d|1dd|[1-9]?d)(.(25[0-5]|2[0-4]d|1dd|[1-9]?d)){3}))|:)))(%.+)?s*(\/([0-9]|[1-9][0-9]|1[0-1][0-9]|12[0-8]))$' # noqa: E501
emailRegex = r'''(?i)(?:[a-z0-9!#$%&'*+/=?^_\x60{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_\x60{|}~-]+)*|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\[(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\])''' # noqa: E501
urlRegex = r'(?i)(?:(?P<url_with_path>(?P<scheme>(?:https?|hxxps?|s?ftps?|meows?)\[?[:-]]?(?:\/\/|\\\\|3A__))?(?P<userinfo>[\w]+@)?(?P<host>(?P<simple_domain>(?:(?:[^\W_]|-)+\[?\.\]?)+[^\W\d_-]{2,})|(?P<ipv4>(?:(?:25[0-5]|2[0-4][\d]|[01]?[\d][\d]?)\[?[.]]?){3}(?:25[0-5]|2[0-4][\d]|[01]?[\d][\d]?))|(?P<HEXIPv4>0\[?x]?[\da-f]{8})|(?P<ipv6>\[?(?:(?:[\da-fA-F]{1,4}:){7,7}[\da-fA-F]{1,4}|(?:[\da-fA-F]{1,4}:){1,7}:|([\da-fA-F]{1,4}:){1,6}:[\da-fA-F]{1,4}|([\da-fA-F]{1,4}:){1,5}(:[\da-fA-F]{1,4}){1,2}|([\da-fA-F]{1,4}:){1,4}(:[\da-fA-F]{1,4}){1,3}|([\da-fA-F]{1,4}:){1,3}(:[\da-fA-F]{1,4}){1,4}|([\da-fA-F]{1,4}:){1,2}(:[\da-fA-F]{1,4}){1,5}|[\da-fA-F]{1,4}:(?:(:[\da-fA-F]{1,4}){1,6})|:(?:(:[\da-fA-F]{1,4}){1,7}|:)|fe80:(?::[\da-fA-F]{0,4}){0,4}%[\da-zA-Z]{1,}|::(?:ffff(?::0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[\d]){0,1}[\d])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[\d]){0,1}[\d])|([\da-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[\d]){0,1}[\d])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[\d]){0,1}[\d]))\]?))(?P<port>:(?:6[0-5][\d]{3}|[1-5][\d]{4}|[1-9][\d]{,3}))?/(?P<path>(?:[\w\/%-]+)(?P<extension>\[?[.]]?[^\W\d_-]+)?)?(?P<query>\?[^\s#]*)?(?P<fragment>#[\w\d]*)?)|(?P<no_path_url>(?:(?:https?|hxxps?|s?ftps?|meows?)\[?[:-]]?(?:\/\/|\\\\|3A__))(?:[\w]+@)?(?:(?:(?:[^\W_]+\[?\.\]?)+[^\W\d_-]{2,})[\/.]?)|(?:(?:(?:https?|hxxps?|s?ftps?|meows?)\[?[:-]]?(?:\/\/|\\\\|3A__))(?:(?:(?:(?:25[0-5]|2[0-4][\d]|[01]?[\d][\d]?)\[?[.]]?){3}(?:25[0-5]|2[0-4][\d]|[01]?[\d][\d]?))|(?:0\[?x]?[\da-f]{8})|(?:\[?(?:(?:[\da-fA-F]{1,4}:){7,7}[\da-fA-F]{1,4}|(?:[\da-fA-F]{1,4}:){1,7}:|([\da-fA-F]{1,4}:){1,6}:[\da-fA-F]{1,4}|([\da-fA-F]{1,4}:){1,5}(:[\da-fA-F]{1,4}){1,2}|([\da-fA-F]{1,4}:){1,4}(:[\da-fA-F]{1,4}){1,3}|([\da-fA-F]{1,4}:){1,3}(:[\da-fA-F]{1,4}){1,4}|([\da-fA-F]{1,4}:){1,2}(:[\da-fA-F]{1,4}){1,5}|[\da-fA-F]{1,4}:(?:(:[\da-fA-F]{1,4}){1,6})|:(?:(:[\da-fA-F]{1,4}){1,7}|:)|fe80:(?::[\da-fA-F]{0,4}){0,4}%[\da-zA-Z]{1,}|::(?:ffff(?::0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[\d]){0,1}[\d])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[\d]){0,1}[\d])|([\da-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[\d]){0,1}[\d])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[\d]){0,1}[\d]))\]?))))$|(?P<ip_port>(?:(?:https?|hxxps?|s?ftps?|meows?)\[?[:-]]?(?:\/\/|\\\\|3A__))?(?:(?:(?:(?:25[0-5]|2[0-4][\d]|[01]?[\d][\d]?)\[?[.]]?){3}(?:25[0-5]|2[0-4][\d]|[01]?[\d][\d]?))|(?:0\[?x]?[\da-f]{8})|(?:\[?(?:(?:[\da-fA-F]{1,4}:){7,7}[\da-fA-F]{1,4}|(?:[\da-fA-F]{1,4}:){1,7}:|([\da-fA-F]{1,4}:){1,6}:[\da-fA-F]{1,4}|([\da-fA-F]{1,4}:){1,5}(:[\da-fA-F]{1,4}){1,2}|([\da-fA-F]{1,4}:){1,4}(:[\da-fA-F]{1,4}){1,3}|([\da-fA-F]{1,4}:){1,3}(:[\da-fA-F]{1,4}){1,4}|([\da-fA-F]{1,4}:){1,2}(:[\da-fA-F]{1,4}){1,5}|[\da-fA-F]{1,4}:(?:(:[\da-fA-F]{1,4}){1,6})|:(?:(:[\da-fA-F]{1,4}){1,7}|:)|fe80:(?::[\da-fA-F]{0,4}){0,4}%[\da-zA-Z]{1,}|::(?:ffff(?::0{1,4}){0,1}:){0,1}((25[0-5]|(2[0-4]|1{0,1}[\d]){0,1}[\d])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[\d]){0,1}[\d])|([\da-fA-F]{1,4}:){1,4}:((25[0-5]|(2[0-4]|1{0,1}[\d]){0,1}[\d])\.){3,3}(25[0-5]|(2[0-4]|1{0,1}[\d]){0,1}[\d]))\]?))(?::(?:6[0-5][\d]{3}|[1-5][\d]{4}|[1-9][\d]{,3})))$)' # noqa: E501
cveRegex = r'(?i)^cve-\d{4}-([1-9]\d{4,}|\d{4})$'
domainRegex = r'(?i)(?P<scheme>(?:http|hxxp|s?ftp)s?(?::|%3A)(?:%2F%2F|//))?(?P<fqdn>(?:(?P<domain>(?:[^\W_]|-)+)\[?\.\]?)+(?P<tld>[^\W\d_-]{2,}))'
hashRegex = r'\b[0-9a-fA-F]+\b'
md5Regex = re.compile(r'\b[0-9a-fA-F]{32}\b', regexFlags)
sha1Regex = re.compile(r'\b[0-9a-fA-F]{40}\b', regexFlags)
sha256Regex = re.compile(r'\b[0-9a-fA-F]{64}\b', regexFlags)
sha512Regex = re.compile(r'\b[0-9a-fA-F]{128}\b', regexFlags)
pascalRegex = re.compile('([A-Z]?[a-z]+)')
# ############################## REGEX FORMATTING end ###############################
def is_filename_valid(filename):
"""
Checking if the file name contains invalid characters.
:param filename: The file name
:type filename: ``str``
:return: True if valid otherwise False.
:rtype: ``bool``
"""
if not re.match(r"^[^~)('\\!*<>:;,?\"*|/]+$", filename):
return False
return True
def underscoreToCamelCase(s, upper_camel=True):
"""
Convert an underscore separated string to camel case
:type s: ``str``
:param s: The string to convert (e.g. hello_world) (required)
:type upper_camel: ``bool``
:param upper_camel: When True then transforms dictionarykeys to camel case with the first letter capitalised
(for example: demisto_content to DemistoContent), otherwise the first letter will not be capitalised
(for example: demisto_content to demistoContent).
:return: The converted string (e.g. HelloWorld)
:rtype: ``str``
"""
if not isinstance(s, STRING_OBJ_TYPES):
return s
components = s.split('_')
camel_without_first_char = ''.join(x.title() for x in components[1:])
if upper_camel:
return components[0].title() + camel_without_first_char
else:
return components[0].lower() + camel_without_first_char
def camel_case_to_underscore(s):
"""Converts a camelCase string to snake_case
:type s: ``str``
:param s: The string to convert (e.g. helloWorld) (required)
:return: The converted string (e.g. hello_world)
:rtype: ``str``
"""
s1 = re.sub('(.)([A-Z][a-z]+)', r'\1_\2', s)
return re.sub('([a-z0-9])([A-Z])', r'\1_\2', s1).lower()
def snakify(src):
"""Convert all keys of a dictionary to snake_case (underscored separated)
:type src: ``dict``
:param src: The dictionary to convert the keys for. (required)
:return: The dictionary (or list of dictionaries) with the keys in CamelCase.
:rtype: ``dict``
"""
return {camel_case_to_underscore(k): v for k, v in src.items()}
def pascalToSpace(s):
"""
Converts pascal strings to human readable (e.g. "ThreatScore" -> "Threat Score", "thisIsIPAddressName" ->
"This Is IP Address Name"). Could be used as headerTransform
:type s: ``str``
:param s: The string to be converted (required)
:return: The converted string
:rtype: ``str``
"""
if not isinstance(s, STRING_OBJ_TYPES):
return s
# double space to handle capital words like IP/URL/DNS that not included in the regex
s = re.sub(pascalRegex, lambda match: r' {} '.format(match.group(1).title()), s)
# split and join: to remove double spacing caused by previous workaround
s = ' '.join(s.split())
return s
def string_to_table_header(string):
"""
Checks if string, change underscores to spaces, capitalize every word.
Example: "one_two" to "One Two"
:type string: ``str``
:param string: The string to be converted (required)
:return: The converted string
:rtype: ``str``
"""
if isinstance(string, STRING_OBJ_TYPES):
return " ".join(word.capitalize() for word in string.replace("_", " ").split())
else:
raise Exception('The key is not a string: {}'.format(string))
def string_to_context_key(string):
"""
Checks if string, removes underscores, capitalize every word.
Example: "one_two" to "OneTwo"
:type string: ``str``
:param string: The string to be converted (required)
:return: The converted string
:rtype: ``str``
"""
if isinstance(string, STRING_OBJ_TYPES):
return "".join(word.capitalize() for word in string.split('_'))
else:
raise Exception('The key is not a string: {}'.format(string))
def response_to_context(reponse_obj, user_predefiend_keys=None):
"""
Recursively creates a data dictionary where all key starts with capital letters.
If a key include underscores, removes underscores, capitalize every word.
Example: "one_two" to "OneTwo
:type reponse_obj: ``Any``
:param reponse_obj: The response object to update.
:type reponse_obj: ``dict``
:user_predefiend_keys: An optional argument,
a dict with predefined keys where the key is the key in the response and value is the key we want to turn the key into.
:return: A response with all keys (if there're any) starts with a capital letter.
:rtype: ``Any``
"""
predefined_keys = {"id": "ID", "uuid": "UUID"}
if user_predefiend_keys:
predefined_keys.update(user_predefiend_keys)
parsed_dict = {}
key = ""
if isinstance(reponse_obj, dict):
for key, value in reponse_obj.items():
if key in predefined_keys:
key = predefined_keys.get(key, "") # type: ignore
else:
key = string_to_context_key(key)
if isinstance(value, dict):
parsed_dict[key] = response_to_context(value)
elif isinstance(value, list):
parsed_dict[key] = [response_to_context(list_item) for list_item in value]
else:
parsed_dict[key] = value
return parsed_dict
elif isinstance(reponse_obj, list):
return [response_to_context(list_item) for list_item in reponse_obj]
else:
return reponse_obj
def safe_strptime(date_str, datetime_format, strptime=datetime.strptime):
"""
Parses a date string to a datetime object, handling cases where the microsecond component is missing.
:type date_str: ``str``
:param date_str: The date string to parse (required)
:type datetime_format: ``str``
:param datetime_format: The format of the date string (required)
:type strptime: ``Callable``
:param strptime: The function to use for parsing the date string (optional)
:return: The parsed datetime object
:rtype: ``datetime.datetime``
"""
try:
return strptime(date_str, datetime_format)
except ValueError as e:
demisto.debug('Failed to parse date {} with format {}: {}'.format(date_str, datetime_format, str(e)))
return strptime(date_str, datetime_format.replace('.%f', ''))
def parse_date_range(date_range, date_format=None, to_timestamp=False, timezone=0, utc=True):
"""
THIS FUNCTTION IS DEPRECATED - USE dateparser.parse instead
Parses date_range string to a tuple date strings (start, end). Input must be in format 'number date_range_unit')
Examples: (2 hours, 4 minutes, 6 month, 1 day, etc.)
:type date_range: ``str``
:param date_range: The date range to be parsed (required)
:type date_format: ``str``
:param date_format: Date format to convert the date_range to. (optional)
:type to_timestamp: ``bool``
:param to_timestamp: If set to True, then will return time stamp rather than a datetime.datetime. (optional)
:type timezone: ``int``
:param timezone: timezone should be passed in hours (e.g if +0300 then pass 3, if -0200 then pass -2).
:type utc: ``bool``
:param utc: If set to True, utc time will be used, otherwise local time.
:return: The parsed date range.
:rtype: ``(datetime.datetime, datetime.datetime)`` or ``(int, int)`` or ``(str, str)``
"""
range_split = date_range.strip().split(' ')
if len(range_split) != 2:
return_error('date_range must be "number date_range_unit", examples: (2 hours, 4 minutes,6 months, 1 day, '
'etc.)')
try:
number = int(range_split[0])
except ValueError:
return_error('The time value is invalid. Must be an integer.')
unit = range_split[1].lower()
if unit not in ['minute', 'minutes',
'hour', 'hours',
'day', 'days',
'month', 'months',
'year', 'years',
]:
return_error('The unit of date_range is invalid. Must be minutes, hours, days, months or years.')
if not isinstance(timezone, (int, float)):
return_error('Invalid timezone "{}" - must be a number (of type int or float).'.format(timezone))
if utc:
utc_now = datetime.utcnow()
end_time = utc_now + timedelta(hours=timezone)
start_time = utc_now + timedelta(hours=timezone)
else:
now = datetime.now()
end_time = now + timedelta(hours=timezone)
start_time = now + timedelta(hours=timezone)
if 'minute' in unit:
start_time = end_time - timedelta(minutes=number)
elif 'hour' in unit:
start_time = end_time - timedelta(hours=number)
elif 'day' in unit:
start_time = end_time - timedelta(days=number)
elif 'month' in unit:
start_time = end_time - timedelta(days=number * 30)
elif 'year' in unit:
start_time = end_time - timedelta(days=number * 365)
if to_timestamp:
return date_to_timestamp(start_time), date_to_timestamp(end_time)
if date_format:
return datetime.strftime(start_time, date_format), datetime.strftime(end_time, date_format)
return start_time, end_time
def timestamp_to_datestring(timestamp, date_format="%Y-%m-%dT%H:%M:%S.000Z", is_utc=False):
"""
Parses timestamp (milliseconds) to a date string in the provided date format (by default: ISO 8601 format)
Examples: (1541494441222, 1541495441000, etc.)
:type timestamp: ``int`` or ``str``
:param timestamp: The timestamp to be parsed (required)
:type date_format: ``str``
:param date_format: The date format the timestamp should be parsed to. (optional)
:type is_utc: ``bool``
:param is_utc: Should the string representation of the timestamp use UTC time or the local machine time
:return: The parsed timestamp in the date_format
:rtype: ``str``
"""
use_utc_time = is_utc or date_format.endswith('Z')
if use_utc_time:
return datetime.utcfromtimestamp(int(timestamp) / 1000.0).strftime(date_format)
return datetime.fromtimestamp(int(timestamp) / 1000.0).strftime(date_format)
def date_to_timestamp(date_str_or_dt, date_format='%Y-%m-%dT%H:%M:%S'):
"""
Parses date_str_or_dt in the given format (default: %Y-%m-%dT%H:%M:%S) to milliseconds
Examples: ('2018-11-06T08:56:41', '2018-11-06T08:56:41', etc.)
:type date_str_or_dt: ``str`` or ``datetime.datetime``
:param date_str_or_dt: The date to be parsed. (required)
:type date_format: ``str``
:param date_format: The date format of the date string (will be ignored if date_str_or_dt is of type
datetime.datetime). (optional)
:return: The parsed timestamp.
:rtype: ``int``
"""
if isinstance(date_str_or_dt, STRING_OBJ_TYPES):
return int(time.mktime(safe_strptime(date_str_or_dt, date_format, time.strptime)) * 1000)
# otherwise datetime.datetime
return int(time.mktime(date_str_or_dt.timetuple()) * 1000)
def remove_nulls_from_dictionary(data):
"""
Remove Null values from a dictionary. (updating the given dictionary)
:type data: ``dict``
:param data: The data to be added to the context (required)
:return: No data returned
:rtype: ``None``
"""
list_of_keys = list(data.keys())[:]
for key in list_of_keys:
if data[key] in ('', None, [], {}, ()):
del data[key]
def assign_params(keys_to_ignore=None, values_to_ignore=None, **kwargs):
"""Creates a dictionary from given kwargs without empty values.
empty values are: None, '', [], {}, ()
` Examples:
>>> assign_params(a='1', b=True, c=None, d='')
{'a': '1', 'b': True}
>>> since_time = 'timestamp'
>>> assign_params(values_to_ignore=(15, ), sinceTime=since_time, b=15)
{'sinceTime': 'timestamp'}
>>> item_id = '1236654'
>>> assign_params(keys_to_ignore=['rnd'], ID=item_id, rnd=15)
{'ID': '1236654'}
:type keys_to_ignore: ``tuple`` or ``list``
:param keys_to_ignore: Keys to ignore if exists
:type values_to_ignore: ``tuple`` or ``list``
:param values_to_ignore: Values to ignore if exists
:type kwargs: ``kwargs``
:param kwargs: kwargs to filter
:return: dict without empty values
:rtype: ``dict``
"""
if values_to_ignore is None:
values_to_ignore = (None, '', [], {}, ())
if keys_to_ignore is None:
keys_to_ignore = tuple()
return {
key: value for key, value in kwargs.items()
if value not in values_to_ignore and key not in keys_to_ignore
}
class GetDemistoVersion:
"""
Callable class to replace get_demisto_version function
"""
def __init__(self):
self._version = None
def __call__(self):
"""Returns the Demisto version and build number.
:return: Demisto version object if Demisto class has attribute demistoVersion, else raises AttributeError
:rtype: ``dict``
"""
if self._version is None:
if hasattr(demisto, 'demistoVersion'):
self._version = demisto.demistoVersion()
else:
raise AttributeError('demistoVersion attribute not found.')
return self._version
get_demisto_version = GetDemistoVersion()
def get_demisto_version_as_str():
"""Get the Demisto Server version as a string `<version>-<build>`. If unknown will return: 'Unknown'.
Meant to be use in places where we want to display the version. If you want to perform logic based upon vesrion
use: is_demisto_version_ge.
:return: Demisto version as string
:rtype: ``dict``
"""
try:
ver_obj = get_demisto_version()
return '{}-{}'.format(ver_obj.get('version', 'Unknown'),
ver_obj.get("buildNumber", 'Unknown'))
except AttributeError:
return "Unknown"
def is_demisto_version_ge(version, build_number=''):
"""Utility function to check if current running integration is at a server greater or equal to the passed version
:type version: ``str``
:param version: Version to check
:type build_number: ``str``
:param build_number: Build number to check
:return: True if running within a Server version greater or equal than the passed version
:rtype: ``bool``
"""
def versionzfill(v):
""" Split version on . and zfill. So we can compare 6.10 to 6.5 properly.
:type v: ``srt``
:param v: Version str to fill
:return: Version with zfill
:rtype: ``str``
"""
vzfill = []
for ver in v.split("."):
vzfill.append(ver.zfill(8))
return tuple(vzfill)
server_version = {}
try:
server_version = get_demisto_version()
server_zfill = versionzfill(server_version.get('version'))
ver_zfill = versionzfill(version)
if server_zfill > ver_zfill:
return True
elif server_zfill == ver_zfill:
if build_number:
return int(server_version.get('buildNumber')) >= int(build_number) # type: ignore[arg-type]
return True # No build number
else:
return False
except AttributeError:
# demistoVersion was added in 5.0.0. We are currently running in 4.5.0 and below
if version >= "5.0.0":
return False
raise
except ValueError:
# dev editions are not comparable
demisto.log( # pylint: disable=E9012
'is_demisto_version_ge: ValueError. \n '
'input: server version: {} build number: {}\n'
'server version: {}'.format(version, build_number, server_version)
)
return True
def is_xsiam_or_xsoar_saas():
"""Determines whether or not the platform is XSIAM or XSOAR SAAS.
:return: True iff the platform is XSIAM or XSOAR SAAS.
:rtype: ``bool``
"""
return is_demisto_version_ge('8.0.0')
def is_xsoar():
"""Determines whether or not the platform is XSOAR.
:return: True iff the platform is XSOAR.
:rtype: ``bool``
"""
return "xsoar" in demisto.demistoVersion().get("platform", "")
def is_xsoar_on_prem():
"""Determines whether or not the platform is a XSOAR on-prem.
:return: True iff the platform is XSOAR on-prem.
:rtype: ``bool``
"""
return demisto.demistoVersion().get("platform") == "xsoar" and not is_demisto_version_ge('8.0.0')
def is_xsoar_hosted():
"""Determines whether or not the platform is XSOAR hosted.
:return: True iff the platform is XSOAR hosted.
:rtype: ``bool``
"""
return demisto.demistoVersion().get("platform") == "xsoar_hosted"
def is_xsoar_saas():
"""Determines whether or not the platform is XSOAR SAAS.
:return: True iff the platform is XSOAR SAAS.
:rtype: ``bool``
"""
return demisto.demistoVersion().get("platform") == "xsoar" and is_xsiam_or_xsoar_saas()
def is_platform():
"""Determines whether running on a unified Cortex platform tenant.
:return: True iff the platform type is 'unified_platform'.
:rtype: ``bool``
"""
return demisto.demistoVersion().get("platform") == "unified_platform"
def is_xsiam():
"""Determines whether or not the platform is XSIAM.
:return: True iff the platform is XSIAM.
:rtype: ``bool``
"""
XSIAM_PLATFORM_CODES = {"x1", "x3", "x5"}
version_info = demisto.demistoVersion()
is_xsiam_legacy = version_info.get("platform") == "x2"
is_xsiam_platform = is_platform() and (version_info.get("module") in XSIAM_PLATFORM_CODES)
return is_xsiam_legacy or is_xsiam_platform
def resolve_should_push_events(args):
"""Resolve the ``should_push_events`` flag from command args based on the current platform.
Extracts the ``should_push_events`` argument (defaulting to ``False``) and converts it
to a boolean. If it is ``True`` but the platform is not Cortex XSIAM, the value is
silently overridden to ``False`` and a debug log is emitted.
Use this in ``get-events`` debug commands to gracefully degrade on
non-XSIAM platforms (e.g., Cortex XSOAR) instead of raising an error.
:param args: The command arguments dict (``demisto.args()``), expected to optionally
contain a ``should_push_events`` key.
:type args: ``dict``
:return: The resolved push-events flag (``False`` on unsupported platforms).
:rtype: ``bool``
Example::
should_push = resolve_should_push_events(args)
"""
should_push_events = argToBoolean(args.get("should_push_events", False))
if should_push_events and not is_xsiam():
demisto.debug("[Events Push Check] "
"should_push_events is not supported on this platform. Overriding to False.")
return False
return should_push_events
def is_using_engine():
"""Determines whether or not the platform is using engine.
NOTE:
- This method works only for system integrations (not custom).
- On xsoar 8, this method works only for integrations that runs on the xsoar pod - not on the engine-0 (mainly long running
integrations) such as: EDL, Cortex Core - IOC, Cortex Core - IR, ExportIndicators, Generic Webhook, PingCastle,
Publish List, Simple API Proxy, Syslog v2, TAXII Server, TAXII2 Server, Web File Repository, Workday_IAM_Event_Generator,
XSOAR-Web-Server, Microsoft Teams, AWS-SNS-Listener.
:return: True iff the platform is using engine.
:rtype: ``bool``
"""
return demisto.demistoVersion().get("engine")
def is_integration_instance_running_on_engine():
"""Determines whether the current integration instance runs on an xsoar engine.
If yes - returns the engine id.
:return: The engine id iff the instance is running on an xsaor engine.
:rtype: ``str``
"""
engine_id = ''
integrations_raw_response = demisto.internalHttpRequest(
'POST', uri='/settings/integration/search', body='{\"size\":1000}'
)
integrations_body_raw_response = integrations_raw_response.get('body', '{}')
try:
integrations_body_response = json.loads(integrations_body_raw_response) # type: ignore
except json.JSONDecodeError: # type: ignore[attr-defined]
demisto.debug('Unable to load response {}'.format(integrations_body_raw_response))
integrations_body_response = {}
instances = integrations_body_response.get('instances', [])
instance_name = demisto.integrationInstance()
demisto.debug("Search for the data of the {} instance.".format(instance_name))
for instance in instances:
if instance_name == instance.get('name', ''):
engine_id = instance.get('engine', '')
break
if engine_id: # engine_id = '' for instances that don't run on engine
demisto.debug("The {} instance runs on an xsoar engine, engine ID is: {}".format(instance_name, engine_id))
else:
demisto.debug("The {} instance does not run on an xsoar engine.".format(instance_name))
return engine_id
def get_engine_base_url(engine_id):
"""Gets the xsoar engine id and returns it's base url.
For example: for engine_id = '4ccccccc-5aaa-4000-b666-dummy_id', base url = '11.180.111.111:1443'.
:type engine_id: ``str``
:param engine_id: The xsoar engine id.
:return: The base URL of the engine.
:rtype: ``str`
"""
engines_raw_response = demisto.internalHttpRequest(
'GET', uri='/engines', body=json.dumps({})
)
engines_body_raw_response = engines_raw_response.get('body', '{}')
try:
engines_body_response = json.loads(engines_body_raw_response) # type: ignore
except json.JSONDecodeError: # type: ignore[attr-defined]
demisto.debug('Unable to load response {}'.format(engines_body_raw_response))
engines_body_response = {}
engines = engines_body_response.get('engines', [])
demisto.debug("Search for the data of engine ID {}.".format(engine_id))
for engine in engines:
if engine.get('id') == engine_id:
engine_base_url = engine.get('baseUrl', '')
demisto.debug("The base URL of engine ID {} is: {}.".format(engine_id, engine_base_url))
return engine_base_url
demisto.debug("Couldn't find a base url for engine ID {}.".format(engine_id))
return ''
class DemistoHandler(logging.Handler):
"""
Handler to route logging messages to an IntegrationLogger or demisto.debug if not supplied
"""
def __init__(self, int_logger=None):
logging.Handler.__init__(self)
self.int_logger = int_logger
def emit(self, record):
msg = self.format(record)
try:
if self.int_logger:
self.int_logger(msg)
else:
demisto.debug(msg)
except Exception: # noqa: disable=broad-except
pass
def censor_request_logs(request_log):
"""
Censors the request logs generated from the urllib library directly by replacing sensitive information such as tokens and cookies with a mask.
In most cases, the sensitive value is the first word after the keyword, but in some cases, it is the second one.
:param request_log: The request log to censor
:type request_log: ``str``
:return: The censored request log
:rtype: ``str``
"""
keywords_to_censor = ['Authorization:', 'Cookie', "Token", "username",
"password", "Key", "identifier", "credential", "client"]
lower_keywords_to_censor = [word.lower() for word in keywords_to_censor]
trimed_request_log = request_log.lstrip(SEND_PREFIX)
request_log_with_spaces = trimed_request_log.replace("\\r\\n", " \\r\\n")
request_log_lst = request_log_with_spaces.split()
for i, word in enumerate(request_log_lst):
# Check if the word is a keyword or contains a keyword (e.g "Cookies" containes "Cookie")
if any(keyword in word.lower() for keyword in lower_keywords_to_censor):
next_word = request_log_lst[i + 1] if i + 1 < len(request_log_lst) else None
if next_word:
# If the next word is "Bearer", "JWT", "Basic" or "LOG" then we replace the word after it since thats the token
if next_word.lower() in ["bearer", "jwt", "basic", "log"] and i + 2 < len(request_log_lst):
request_log_lst[i + 2] = MASK
elif request_log_lst[i + 1].endswith("}'"):
request_log_lst[i + 1] = "\"{}\"}}'".format(MASK)
else:
request_log_lst[i + 1] = MASK
# Rebuild the request log so that the only change is the masked information.
censored_string = SEND_PREFIX + \
' '.join(request_log_lst) if request_log.startswith(SEND_PREFIX) else ' '.join(request_log_lst)
censored_string = censored_string.replace(" \\r\\n", "\\r\\n")
return censored_string
class DebugLogger(object):
"""
Wrapper to initiate logging at logging.DEBUG level.
Is used when `debug-mode=True`.
"""
def __init__(self):
self.handler = None # just in case our http_client code throws an exception. so we don't error in the __del__
self.int_logger = IntegrationLogger()
self.int_logger.set_buffering(False)
self.http_client_print = None
self.http_client = None
if IS_PY3:
# pylint: disable=import-error
import http.client as http_client
# pylint: enable=import-error
self.http_client = http_client
self.http_client.HTTPConnection.debuglevel = 1
self.http_client_print = getattr(http_client, 'print', None) # save in case someone else patched it already
setattr(http_client, 'print', self.int_logger.print_override)
self.handler = DemistoHandler(self.int_logger)
demisto_formatter = logging.Formatter(fmt='python logging: %(levelname)s [%(name)s] - %(message)s', datefmt=None)
self.handler.setFormatter(demisto_formatter)
self.root_logger = logging.getLogger()
self.prev_log_level = self.root_logger.getEffectiveLevel()
self.root_logger.setLevel(logging.DEBUG)
self.org_handlers = list()
if self.root_logger.handlers:
self.org_handlers.extend(self.root_logger.handlers)
for h in self.org_handlers:
self.root_logger.removeHandler(h)
self.root_logger.addHandler(self.handler)
def __del__(self):
if self.handler:
self.root_logger.setLevel(self.prev_log_level)
self.root_logger.removeHandler(self.handler)
self.handler.flush()
self.handler.close()
if self.org_handlers:
for h in self.org_handlers:
self.root_logger.addHandler(h)
if self.http_client:
self.http_client.HTTPConnection.debuglevel = 0
if self.http_client_print:
setattr(self.http_client, 'print', self.http_client_print)
else:
delattr(self.http_client, 'print')
if self.int_logger.curl:
for curl in self.int_logger.curl:
demisto.info('cURL:\n' + curl)
def log_start_debug(self):
"""
Utility function to log start of debug mode logging
"""
msg = "debug-mode started.\n#### http client print found: {}.\n#### Env {}.".format(self.http_client_print is not None,
os.environ)
if hasattr(demisto, 'params'):
msg += "\n#### Params: {}.".format(json.dumps(demisto.params(), indent=2))
calling_context = demisto.callingContext.get('context', {})
msg += "\n#### Docker image: [{}]".format(calling_context.get('DockerImage'))
brand = calling_context.get('IntegrationBrand')
if brand:
msg += "\n#### Integration: brand: [{}] instance: [{}]".format(brand, calling_context.get('IntegrationInstance'))
sm = get_schedule_metadata(context=calling_context)
if sm.get('is_polling'):
msg += "\n#### Schedule Metadata: scheduled command: [{}] args: [{}] times ran: [{}] scheduled: [{}] end " \
"date: [{}]".format(sm.get('polling_command'),
sm.get('polling_args'),
sm.get('times_ran'),
sm.get('start_date'),
sm.get('end_date')
)
self.int_logger.write(msg)
_requests_logger = None
try:
if is_debug_mode():
_requests_logger = DebugLogger()
_requests_logger.log_start_debug()
except Exception as ex:
# Should fail silently so that if there is a problem with the logger it will
# not affect the execution of commands and playbooks
demisto.info('Failed initializing DebugLogger: {}'.format(ex))
def add_sensitive_log_strs(sensitive_str):
"""
Adds the received string to both LOG and DebugLogger. The logger will mask the string each time he encounters it.
:type sensitive_str: ``str``
:param sensitive_str: The string to be replaced.
:return: No data returned
:rtype: ``None``
"""
if _requests_logger:
_requests_logger.int_logger.add_replace_strs(sensitive_str)
LOG.add_replace_strs(sensitive_str)
def parse_date_string(date_string, date_format='%Y-%m-%dT%H:%M:%S'):
"""
Parses the date_string function to the corresponding datetime object.
Note: If possible (e.g. running Python 3), it is suggested to use
dateutil.parser.parse or dateparser.parse functions instead.
Examples:
>>> parse_date_string('2019-09-17T06:16:39Z')
datetime.datetime(2019, 9, 17, 6, 16, 39)
>>> parse_date_string('2019-09-17T06:16:39.22Z')
datetime.datetime(2019, 9, 17, 6, 16, 39, 220000)
>>> parse_date_string('2019-09-17T06:16:39.4040+05:00', '%Y-%m-%dT%H:%M:%S+02:00')
datetime.datetime(2019, 9, 17, 6, 16, 39, 404000)
:type date_string: ``str``
:param date_string: The date string to parse. (required)
:type date_format: ``str``
:param date_format:
The date format of the date string. If the date format is known, it should be provided. (optional)
:return: The parsed datetime.
:rtype: ``(datetime.datetime, datetime.datetime)``
"""
try:
return safe_strptime(date_string, date_format)
except ValueError as e:
error_message = str(e)
date_format = '%Y-%m-%dT%H:%M:%S'
time_data_regex = r'time data \'(.*?)\''
time_data_match = re.findall(time_data_regex, error_message)
sliced_time_data = ''
if time_data_match:
# found time date which does not match date format
# example of caught error message:
# "time data '2019-09-17T06:16:39Z' does not match format '%Y-%m-%dT%H:%M:%S.%fZ'"
time_data = time_data_match[0]
# removing YYYY-MM-DDThh:mm:ss from the time data to keep only milliseconds and time zone
sliced_time_data = time_data[19:]
else:
unconverted_data_remains_regex = r'unconverted data remains: (.*)'
unconverted_data_remains_match = re.findall(unconverted_data_remains_regex, error_message)
if unconverted_data_remains_match:
# found unconverted_data_remains
# example of caught error message:
# "unconverted data remains: 22Z"
sliced_time_data = unconverted_data_remains_match[0]
if not sliced_time_data:
# did not catch expected error
raise ValueError(e)
if '.' in sliced_time_data:
# found milliseconds - appending ".%f" to date format
date_format += '.%f'
timezone_regex = r'[Zz+-].*'
time_zone = re.findall(timezone_regex, sliced_time_data)
if time_zone:
# found timezone - appending it to the date format
date_format += time_zone[0]
# Make the fractions shorter less than 6 charactors
# e.g. '2022-01-23T12:34:56.123456789+09:00' to '2022-01-23T12:34:56.123456+09:00'
# '2022-01-23T12:34:56.123456789' to '2022-01-23T12:34:56.123456'
date_string = re.sub(r'([0-9]+\.[0-9]{6})[0-9]*([Zz]|[+-]\S+?)?', '\\1\\2', date_string)
return safe_strptime(date_string, date_format)
def build_dbot_entry(indicator, indicator_type, vendor, score, description=None, build_malicious=True):
"""Build a dbot entry. if score is 3 adds malicious
Examples:
>>> build_dbot_entry('user@example.com', 'Email', 'Vendor', 1)
{'DBotScore': {'Indicator': 'user@example.com', 'Type': 'email', 'Vendor': 'Vendor', 'Score': 1}}
>>> build_dbot_entry('user@example.com', 'Email', 'Vendor', 3, build_malicious=False)
{'DBotScore': {'Indicator': 'user@example.com', 'Type': 'email', 'Vendor': 'Vendor', 'Score': 3}}
>>> build_dbot_entry('user@example.com', 'email', 'Vendor', 3, 'Malicious email')
{'DBotScore': {'Vendor': 'Vendor', 'Indicator': 'user@example.com', 'Score': 3, 'Type': 'email'}, \
'Account.Email(val.Address && val.Address == obj.Address)': {'Malicious': {'Vendor': 'Vendor', 'Description': \
'Malicious email'}, 'Address': 'user@example.com'}}
>>> build_dbot_entry('md5hash', 'md5', 'Vendor', 1)
{'DBotScore': {'Indicator': 'md5hash', 'Type': 'file', 'Vendor': 'Vendor', 'Score': 1}}
:type indicator: ``str``
:param indicator: indicator field. if using file hashes, can be dict
:type indicator_type: ``str``
:param indicator_type:
type of indicator ('url, 'domain', 'ip', 'cve', 'email', 'md5', 'sha1', 'sha256', 'crc32', 'sha512', 'ctph')
:type vendor: ``str``
:param vendor: Integration ID
:type score: ``int``
:param score: DBot score (0-3)
:type description: ``str`` or ``None``
:param description: description (will be added to malicious if dbot_score is 3). can be None
:type build_malicious: ``bool``
:param build_malicious: if True, will add a malicious entry
:return: dbot entry
:rtype: ``dict``
"""
if not 0 <= score <= 3:
raise DemistoException('illegal DBot score, expected 0-3, got `{}`'.format(score))
indicator_type_lower = indicator_type.lower()
if indicator_type_lower not in INDICATOR_TYPE_TO_CONTEXT_KEY:
raise DemistoException('illegal indicator type, expected one of {}, got `{}`'.format(
INDICATOR_TYPE_TO_CONTEXT_KEY.keys(), indicator_type_lower
))
# handle files
if INDICATOR_TYPE_TO_CONTEXT_KEY[indicator_type_lower] == 'file':
indicator_type_lower = 'file'
dbot_entry = {
outputPaths['dbotscore']: {
'Indicator': indicator,
'Type': indicator_type_lower,
'Vendor': vendor,
'Score': score
}
}
if score == 3 and build_malicious:
dbot_entry.update(build_malicious_dbot_entry(indicator, indicator_type, vendor, description))
return dbot_entry
def build_malicious_dbot_entry(indicator, indicator_type, vendor, description=None):
""" Build Malicious dbot entry
Examples:
>>> build_malicious_dbot_entry('8.8.8.8', 'ip', 'Vendor', 'Google DNS')
{'IP(val.Address && val.Address == obj.Address)': {'Malicious': {'Vendor': 'Vendor', 'Description': 'Google DNS\
'}, 'Address': '8.8.8.8'}}
>>> build_malicious_dbot_entry('md5hash', 'MD5', 'Vendor', 'Malicious File')
{'File(val.MD5 && val.MD5 == obj.MD5 || val.SHA1 && val.SHA1 == obj.SHA1 || val.SHA256 && val.SHA256 == obj.SHA\
256 || val.SHA512 && val.SHA512 == obj.SHA512 || val.CRC32 && val.CRC32 == obj.CRC32 || val.CTPH && val.CTPH == obj.CTP\
H || val.SSDeep && val.SSDeep == obj.SSDeep)': {'Malicious': {'Vendor': 'Vendor', 'Description': 'Malicious File'}\
, 'MD5': 'md5hash'}}
:type indicator: ``str``
:param indicator: Value (e.g. 8.8.8.8)
:type indicator_type: ``str``
:param indicator_type: e.g. 'IP'
:type vendor: ``str``
:param vendor: Integration ID
:type description: ``str``
:param description: Why it's malicious
:return: A malicious DBot entry
:rtype: ``dict``
"""
indicator_type_lower = indicator_type.lower()
if indicator_type_lower in INDICATOR_TYPE_TO_CONTEXT_KEY:
key = INDICATOR_TYPE_TO_CONTEXT_KEY[indicator_type_lower]
# `file` indicator works a little different
if key == 'file':
entry = {
indicator_type.upper(): indicator,
'Malicious': {
'Vendor': vendor,
'Description': description
}
}
return {outputPaths[key]: entry}
else:
entry = {
key: indicator,
'Malicious': {
'Vendor': vendor,
'Description': description
}
}
return {outputPaths[indicator_type_lower]: entry}
else:
raise DemistoException('Wrong indicator type supplied: {}, expected {}'
.format(indicator_type, INDICATOR_TYPE_TO_CONTEXT_KEY.keys()))
# Will add only if 'requests' module imported
if 'requests' in sys.modules:
if IS_PY3 and PY_VER_MINOR >= 10:
from requests.packages.urllib3.util.ssl_ import create_urllib3_context
# The ciphers string used to replace default cipher string
CIPHERS_STRING = '@SECLEVEL=0:ECDHE+AESGCM:ECDHE+CHACHA20:DHE+AESGCM:DHE+CHACHA20:ECDH+AESGCM:DH+AESGCM:' \
'ECDH+AES:DH+AES:RSA+ANESGCM:RSA+AES:!aNULL:!eNULL:!MD5:!DSS'
class SSLAdapter(HTTPAdapter):
"""
A wrapper used for https communication to enable ciphers that are commonly used
and are not enabled by default
:return: No data returned
:rtype: ``None``
"""
context = create_urllib3_context(ciphers=CIPHERS_STRING)
def __init__(self, verify=True, **kwargs):
# type: (bool, dict) -> None
if not verify and IS_PY3:
self.context.check_hostname = False
if not verify and ssl.OPENSSL_VERSION_INFO >= (3, 0, 0, 0):
self.context.options |= 0x4
super().__init__(**kwargs) # type: ignore[arg-type]
def init_poolmanager(self, *args, **kwargs):
kwargs['ssl_context'] = self.context
return super(SSLAdapter, self).init_poolmanager(*args, **kwargs)
def proxy_manager_for(self, *args, **kwargs):
kwargs['ssl_context'] = self.context
return super(SSLAdapter, self).proxy_manager_for(*args, **kwargs)
class UcpRequestContext(object):
"""Mutable container for request components that the credential injection methods can modify.
:type headers: ``dict``
:param headers: The request headers. Example: {'Accept': 'application/json'}
:type params: ``dict``
:param params: The request URL query parameters. Example: {'limit': 50, 'sort': 'desc'}
:type auth: ``tuple``
:param auth: The basic authentication tuple. Example: ('admin_user', 'super_secret_password')
:type data: ``dict`` or ``str`` or ``bytes``
:param data: The body data for the request. Example: 'key=value&other=123'
:type json_data: ``dict``
:param json_data: The JSON payload for the request. Example: {'username': 'test_user', 'active': True}
"""
__slots__ = ('headers', 'params', 'auth', 'data', 'json_data')
def __init__(self, headers, params, auth, data, json_data):
# Automatically guarantee these are safe dictionary copies, even if None is passed
self.headers = dict(headers or {})
self.params = dict(params or {})
self.auth = auth
self.data = data
self.json_data = json_data
def __iter__(self):
"""Allows the object to be unpacked directly into variables."""
yield self.headers
yield self.params
yield self.auth
yield self.data
yield self.json_data
class BaseClient(object):
"""Client to use in integrations with powerful _http_request
:type base_url: ``str``
:param base_url: Base server address with suffix, for example: https://example.com/api/v2/.
:type verify: ``bool``
:param verify: Whether the request should verify the SSL certificate.
:type proxy: ``bool``
:param proxy: Whether to run the integration using the system proxy.
:type ok_codes: ``tuple`` or ``None``
:param ok_codes:
The request codes to accept as OK, for example: (200, 201, 204).
Will use requests.Response.ok if set to None.
:type headers: ``dict`` or ``None``
:param headers:
The request headers, for example: {'Accept`: `application/json`}.
Can be None.
:type auth: ``dict`` or ``tuple`` or ``None``
:param auth:
The request authorization, for example: (username, password).
Can be None.
:return: No data returned
:rtype: ``None``
"""
REQUESTS_TIMEOUT = 60
TIME_SENSITIVE_TOTAL_TIMEOUT = 15
def __init__(
self,
base_url,
verify=True,
proxy=False,
ok_codes=tuple(),
headers=None,
auth=None,
timeout=REQUESTS_TIMEOUT,
):
self._base_url = base_url
self._verify = verify
self._ok_codes = ok_codes
self._headers = headers
self._auth = auth
self._session = requests.Session()
# the following condition was added to overcome the security hardening happened in Python 3.10.
# https://github.com/python/cpython/pull/25778
# https://bugs.python.org/issue43998
if IS_PY3 and PY_VER_MINOR >= 10 and not verify:
self._session.mount('https://', SSLAdapter(verify=verify))
if proxy:
ensure_proxy_has_http_prefix()
else:
skip_proxy()
if not verify:
skip_cert_verification()
# removing trailing = char from env var value added by the server
entity_timeout = os.getenv('REQUESTS_TIMEOUT.' + (get_integration_name() or get_script_name()), '')
system_timeout = os.getenv('REQUESTS_TIMEOUT', '')
self.timeout = float(entity_timeout or system_timeout or timeout)
# Time-Sensitive Logic
self._time_sensitive_deadline = None
self._time_sensitive_total_timeout = self.TIME_SENSITIVE_TOTAL_TIMEOUT
if is_time_sensitive():
demisto.debug("Time-sensitive mode enabled. Setting execution time limit to {} seconds.".format(self._time_sensitive_total_timeout))
self._time_sensitive_deadline = time.time() + float(
self._time_sensitive_total_timeout
)
self.execution_metrics = ExecutionMetrics()
def _get_ucp_auth_error_codes(self):
# type: () -> tuple
"""Returns a tuple of HTTP status codes that indicate an expired UCP token.
Override in subclasses if the vendor API uses non-standard authentication error codes
(e.g., returning (401, 403) instead of just (401,) for an expired token).
Example::
class Client(BaseClient):
def _get_ucp_auth_error_codes(self):
return (401, 403)
"""
return (401,)
def _apply_ucp_credentials(self, credentials, ctx):
# type: (dict, UcpRequestContext) -> None
"""Overridable Method: Apply UCP credentials to the request context.
Dispatches to per-type methods that are individually overridable:
- _apply_ucp_oauth2() -- for OAuth2 tokens
- _apply_ucp_api_key() -- for API keys
- _apply_ucp_plain() -- for username/password (basic auth)
Override the specific per-type method if the vendor API uses a
non-standard mechanism (custom header, query param, request body).
Override this method only if you need to change the dispatch logic itself.
:type credentials: ``dict``
:param credentials: The credentials dictionary fetched from UCP.
Example: {'type': 'api_key', 'api_key': {'key': '12345'}}
:type ctx: ``UcpRequestContext``
:param ctx: The mutable request context object. The matching _apply_ucp_* method will mutate this object.
:return: None
:rtype: ``None``
"""
cred_type = credentials.get('type')
# Bug on UCP side where they return different types for the same credential type. To be fixed in July'26 version
if cred_type == 'oauth2_client_credentials' or cred_type == 'oauth2_authorization_code' or cred_type == 'oauth2':
self._apply_ucp_oauth2(credentials, ctx)
elif cred_type == 'api_key':
self._apply_ucp_api_key(credentials, ctx)
elif cred_type == 'plain':
self._apply_ucp_plain(credentials, ctx)
else:
demisto.error('[UCP][CommonServerPython.py] _apply_ucp_credentials: Unsupported credential type: "{}". '
'Supported types: oauth2, api_key, plain.'.format(cred_type))
raise UcpException()
def _apply_ucp_oauth2(self, credentials, ctx):
# type: (dict, UcpRequestContext) -> None
"""Apply OAuth2 credentials. Override in subclasses for custom OAuth2
token placement (e.g., custom header name or query parameter).
Default: sets ``Authorization`` header to ``'{token_type} {access_token}'``.
Args:
credentials: dict from getUCPCredentials().
ctx: UcpRequestContext to mutate.
"""
oauth2_data = credentials.get('oauth2', credentials)
token_type = oauth2_data.get('token_type', 'Bearer')
access_token = oauth2_data.get('access_token', '')
if not access_token:
demisto.error("[UCP][CommonServerPython.py] access_token is empty in UCP OAuth2 credentials")
raise UcpException()
ctx.headers['Authorization'] = '{} {}'.format(token_type, access_token)
def _apply_ucp_api_key(self, credentials, ctx):
# type: (dict, UcpRequestContext) -> None
"""Apply API key credentials. Override for custom API key placement.
Default: sets Authorization header to 'Bearer {key}'.
Args:
credentials: dict from getUCPCredentials().
ctx: UcpRequestContext to mutate.
Example::
class Client(BaseClient):
def _apply_ucp_api_key(self, credentials, ctx):
api_key_data = credentials.get('api_key', credentials)
ctx.headers['X-API-Key'] = api_key_data.get('key', '')
"""
# API key fields may be nested inside credentials['api_key'];
# fall back to top-level for backward compatibility.
api_key_data = credentials.get('api_key', credentials)
key = api_key_data.get('key', '')
if not key:
demisto.error("[UCP][CommonServerPython.py] API key is empty in UCP credentials")
raise UcpException()
ctx.headers['Authorization'] = 'Bearer {}'.format(key)
def _apply_ucp_plain(self, credentials, ctx):
# type: (dict, UcpRequestContext) -> None
"""Apply plain credentials (basic auth). Override for custom placement.
Default: sets ctx.auth tuple for requests basic auth.
Args:
credentials: dict from getUCPCredentials().
ctx: UcpRequestContext to mutate.
"""
# Plain fields may be nested inside credentials['plain'];
# fall back to top-level for backward compatibility.
plain_data = credentials.get('plain', credentials)
username = plain_data.get('username', '')
if not username:
demisto.error("[UCP][CommonServerPython.py] username is empty in UCP plain credentials")
raise UcpException()
password = plain_data.get('password', '')
ctx.auth = (username, password)
def _resolve_ucp_capability(self):
# type: () -> tuple
"""Resolve the (capability, sub_capability) for the current command.
Override in subclasses to provide integration-specific sub_capability
mapping. The default returns ``(resolve_ucp_capability(), None)``.
:return: Tuple of ``(capability, sub_capability)``.
:rtype: ``tuple``
Example::
class Client(BaseClient):
def _resolve_ucp_capability(self):
return resolve_ucp_capability(), 'my-sub-capability'
"""
return resolve_ucp_capability(), None
def _inject_ucp_credentials(self, ctx):
# type: (UcpRequestContext) -> str
"""Resolve UCP credentials and apply them to *ctx* in place.
Resolves capability -> profile -> credentials, then dispatches to
the appropriate ``_apply_ucp_*`` method to mutate *ctx*.
:type ctx: ``UcpRequestContext``
:param ctx: The request context to mutate with UCP credentials.
:return: The ``method_unique_id`` used to fetch credentials (needed
for cache invalidation on auth failure retry).
:rtype: ``str``
"""
capability, sub_capability = self._resolve_ucp_capability()
method_unique_id = get_ucp_method_unique_id(capability, sub_capability)
ucp_creds = get_ucp_credentials(method_unique_id)
self._apply_ucp_credentials(ucp_creds, ctx)
return method_unique_id
def __del__(self):
self._return_execution_metrics_results()
try:
self._session.close()
except AttributeError:
# we ignore exceptions raised due to session not used by the client and hence do not exist in __del__
pass
except Exception: # noqa
demisto.debug('failed to close BaseClient session with the following error:\n{}'.format(traceback.format_exc()))
def _implement_retry(self, retries=0,
status_list_to_retry=None,
backoff_factor=5,
backoff_jitter=0.0,
raise_on_redirect=False,
raise_on_status=False):
"""
Implements the retry mechanism.
In the default case where retries = 0 the request will fail on the first time
:type retries: ``int``
:param retries: How many retries should be made in case of a failure. when set to '0'- will fail on the first time
:type status_list_to_retry: ``iterable``
:param status_list_to_retry: A set of integer HTTP status codes that we should force a retry on.
A retry is initiated if the request method is in ['GET', 'POST', 'PUT']
and the response status code is in ``status_list_to_retry``.
:type backoff_factor ``float``
:param backoff_factor:
A backoff factor to apply between attempts after the second try
(most errors are resolved immediately by a second try without a
delay). urllib3 will sleep for::
{backoff factor} * (2 ** ({number of total retries} - 1))
seconds. If the backoff_factor is 0.1, then :func:`.sleep` will sleep
for [0.0s, 0.2s, 0.4s, ...] between retries. It will never be longer
than :attr:`Retry.BACKOFF_MAX`.
By default, backoff_factor set to 5
:type backoff_jitter ``float``
:param backoff_jitter: the sleep (backoff factor) is extended by
random.uniform(0, {backoff jitter})
:type raise_on_redirect ``bool``
:param raise_on_redirect: Whether, if the number of redirects is
exhausted, to raise a MaxRetryError, or to return a response with a
response code in the 3xx range.
:type raise_on_status ``bool``
:param raise_on_status: Similar meaning to ``raise_on_redirect``:
whether we should raise an exception, or return a response,
if status falls in ``status_forcelist`` range and retries have
been exhausted.
"""
try:
method_whitelist = "allowed_methods" if hasattr(
Retry.DEFAULT, "allowed_methods"
) else "method_whitelist" # type: ignore[attr-defined]
dynamic_kwargs = {
method_whitelist: frozenset(['GET', 'POST', 'PUT', 'PATCH', 'DELETE'])
}
if hasattr(Retry.DEFAULT, "backoff_jitter"):
dynamic_kwargs["backoff_jitter"] = backoff_jitter
retry = Retry(
total=retries,
read=retries,
connect=retries,
backoff_factor=backoff_factor,
status=retries,
status_forcelist=status_list_to_retry,
raise_on_status=raise_on_status,
raise_on_redirect=raise_on_redirect,
**dynamic_kwargs # type: ignore[arg-type]
)
http_adapter = HTTPAdapter(max_retries=retry)
# the following condition was added to overcome the security hardening happened in Python 3.10.
# https://github.com/python/cpython/pull/25778
# https://bugs.python.org/issue43998
if self._verify:
https_adapter = http_adapter
elif IS_PY3 and PY_VER_MINOR >= 10:
https_adapter = SSLAdapter(max_retries=retry, verify=self._verify) # type: ignore[arg-type]
else:
https_adapter = http_adapter
self._session.mount('https://', https_adapter)
except NameError:
pass
def _http_request(self, method, url_suffix='', full_url=None, headers=None, auth=None, json_data=None,
params=None, data=None, files=None, timeout=None, resp_type='json', ok_codes=None,
return_empty_response=False, retries=0, status_list_to_retry=None,
backoff_factor=5, backoff_jitter=0.0, raise_on_redirect=False, raise_on_status=False,
error_handler=None, empty_valid_codes=None, params_parser=None, with_metrics=False, **kwargs):
"""A wrapper for requests lib to send our requests and handle requests and responses better.
:type method: ``str``
:param method: The HTTP method, for example: GET, POST, and so on.
:type url_suffix: ``str``
:param url_suffix: The API endpoint.
:type full_url: ``str``
:param full_url:
Bypasses the use of self._base_url + url_suffix. This is useful if you need to
make a request to an address outside of the scope of the integration
API.
:type headers: ``dict``
:param headers: Headers to send in the request. If None, will use self._headers.
:type auth: ``tuple``
:param auth:
The authorization tuple (usually username/password) to enable Basic/Digest/Custom HTTP Auth.
if None, will use self._auth.
:type params: ``dict``
:param params: URL parameters to specify the query.
:type data: ``dict``
:param data: The data to send in a 'POST' request.
:type json_data: ``dict``
:param json_data: The dictionary to send in a 'POST' request.
:type files: ``dict``
:param files: The file data to send in a 'POST' request.
:type timeout: ``float`` or ``tuple``
:param timeout:
The amount of time (in seconds) that a request will wait for a client to
establish a connection to a remote machine before a timeout occurs.
can be only float (Connection Timeout) or a tuple (Connection Timeout, Read Timeout).
:type resp_type: ``str``
:param resp_type:
Determines which data format to return from the HTTP request. The default
is 'json'. Other options are 'text', 'content', 'xml' or 'response'. Use 'response'
to return the full response object.
:type ok_codes: ``tuple``
:param ok_codes:
The request codes to accept as OK, for example: (200, 201, 204). If you specify
"None", will use self._ok_codes.
:type retries: ``int``
:param retries: How many retries should be made in case of a failure. when set to '0'- will fail on the first time
:type status_list_to_retry: ``iterable``
:param status_list_to_retry: A set of integer HTTP status codes that we should force a retry on.
A retry is initiated if the request method is in ['GET', 'POST', 'PUT']
and the response status code is in ``status_list_to_retry``.
:type backoff_factor ``float``
:param backoff_factor:
A backoff factor to apply between attempts after the second try
(most errors are resolved immediately by a second try without a
delay). urllib3 will sleep for::
{backoff factor} * (2 ** ({number of total retries} - 1))
seconds. If the backoff_factor is 0.1, then :func:`.sleep` will sleep
for [0.0s, 0.2s, 0.4s, ...] between retries. It will never be longer
than :attr:`Retry.BACKOFF_MAX`.
By default, backoff_factor set to 5
:type backoff_jitter ``float``
:param backoff_jitter: the sleep (backoff factor) is extended by
random.uniform(0, {backoff jitter})
:type raise_on_redirect ``bool``
:param raise_on_redirect: Whether, if the number of redirects is
exhausted, to raise a MaxRetryError, or to return a response with a
response code in the 3xx range.
:type raise_on_status ``bool``
:param raise_on_status: Similar meaning to ``raise_on_redirect``:
whether we should raise an exception, or return a response,
if status falls in ``status_forcelist`` range and retries have
been exhausted.
:type error_handler ``callable``
:param error_handler: Given an error entry, the error handler outputs the
new formatted error message.
:type empty_valid_codes: ``list``
:param empty_valid_codes: A list of all valid status codes of empty responses (usually only 204, but
can vary)
:type params_parser: ``callable``
:param params_parser: How to quote the params. By default, spaces are replaced with `+` and `/` to `%2F`.
see here for more info: https://docs.python.org/3/library/urllib.parse.html#urllib.parse.urlencode
Note! supported only in python3.
:type with_metrics ``bool``
:param with_metrics: Whether or not to calculate execution metrics from the response
:return: Depends on the resp_type parameter
:rtype: ``dict`` or ``str`` or ``bytes`` or ``xml.etree.ElementTree.Element`` or ``requests.Response``
"""
# Time-Sensitive command Logic
request_timeout = timeout
request_retries = retries
remaining_time = None
# Time-Sensitive command mode
if self._time_sensitive_deadline:
remaining_time = self._time_sensitive_deadline - time.time()
if remaining_time <= 0:
raise DemistoException(
"Time-sensitive command execution time limit ({time_limit}s) reached before performing the API request."
.format(time_limit=self._time_sensitive_total_timeout)
)
request_retries = 0
request_timeout = remaining_time
# Set default timeout if one hasn't been set by user OR by time-sensitive logic
if request_timeout is None:
request_timeout = self.timeout
try:
# Replace params if supplied
address = full_url if full_url else urljoin(self._base_url, url_suffix)
headers = headers if headers else self._headers
auth = auth if auth else self._auth
if request_retries:
self._implement_retry(
request_retries,
status_list_to_retry,
backoff_factor,
backoff_jitter,
raise_on_redirect,
raise_on_status,
)
# -- UCP: Inject credentials into request --
current_method_unique_id = None
if should_use_ucp_auth():
demisto.debug("[UCP][CommonServerPython.py] _http_request: injecting UCP credentials")
ctx = UcpRequestContext(headers, params, auth, data, json_data)
current_method_unique_id = self._inject_ucp_credentials(ctx)
headers, params, auth, data, json_data = ctx
if (
IS_PY3 and params_parser
): # The `quote_via` parameter is supported only in python3.
params = urllib.parse.urlencode(params, quote_via=params_parser)
res = self._session.request(
method,
address,
verify=self._verify,
params=params,
data=data,
json=json_data,
files=files,
headers=headers,
auth=auth,
timeout=request_timeout,
**kwargs
)
# -- UCP: Invalidate credentials if we get an auth error --
if should_use_ucp_auth() and res.status_code in self._get_ucp_auth_error_codes():
invalidate_ucp_credentials(current_method_unique_id)
if not self._is_status_code_valid(res, ok_codes):
self._handle_error(error_handler, res, with_metrics)
return self._handle_success(
res,
resp_type,
empty_valid_codes,
return_empty_response,
with_metrics,
)
except requests.exceptions.ConnectTimeout as exception:
if with_metrics:
self.execution_metrics.timeout_error += 1
err_msg = (
"Connection Timeout Error - potential reasons might be that the Server URL parameter"
" is incorrect or that the Server is not accessible from your host."
)
if (
self._time_sensitive_deadline
and remaining_time is not None
and remaining_time <= request_timeout
):
err_msg = "Time-sensitive command execution time limit ({time_limit}s) exceeded. Original error: {original_msg}".format(
time_limit=self._time_sensitive_total_timeout,
original_msg=err_msg
)
raise DemistoException(err_msg, exception)
except requests.exceptions.SSLError as exception:
if with_metrics:
self.execution_metrics.ssl_error += 1
# in case the "Trust any certificate" is already checked
if not self._verify:
raise
err_msg = 'SSL Certificate Verification Failed - try selecting \'Trust any certificate\' checkbox in' \
' the integration configuration.'
raise DemistoException(err_msg, exception)
except requests.exceptions.ProxyError as exception:
if with_metrics:
self.execution_metrics.proxy_error += 1
err_msg = 'Proxy Error - if the \'Use system proxy\' checkbox in the integration configuration is' \
' selected, try clearing the checkbox.'
raise DemistoException(err_msg, exception)
except requests.exceptions.ConnectionError as exception:
if with_metrics:
self.execution_metrics.connection_error += 1
# Get originating Exception in Exception chain
error_class = str(exception.__class__)
err_type = '<' + error_class[error_class.find('\'') + 1: error_class.rfind('\'')] + '>'
err_msg = 'Verify that the server URL parameter' \
' is correct and that you have access to the server from your host.' \
'\nError Type: {}'.format(err_type)
if exception.errno and exception.strerror:
err_msg += '\nError Number: [{}]\nMessage: {}\n'.format(exception.errno, exception.strerror)
else:
err_msg += '\n{}'.format(str(exception))
raise DemistoException(err_msg, exception)
except requests.exceptions.RetryError as exception:
if with_metrics:
self.execution_metrics.retry_error += 1
try:
reason = 'Reason: {}'.format(exception.args[0].reason.args[0])
except Exception: # noqa: disable=broad-except
reason = ''
err_msg = 'Max Retries Error- Request attempts with {} retries failed. \n{}'.format(retries, reason)
raise DemistoException(err_msg, exception)
def _handle_error(self, error_handler, res, should_update_metrics):
""" Handles error response by calling error handler or default handler.
If an exception is raised, update metrics with failure. Otherwise, proceeds.
:type res: ``requests.Response``
:param res: Response from API after the request for which to check error type
:type error_handler ``callable``
:param error_handler: Given an error entry, the error handler outputs the
new formatted error message.
:type should_update_metrics ``bool``
:param should_update_metrics: Whether or not to update execution metrics according to response
"""
try:
if error_handler:
error_handler(res)
else:
self.client_error_handler(res)
except Exception:
if should_update_metrics:
self._update_metrics(res, success=False)
raise
def _handle_success(self, res, resp_type, empty_valid_codes, return_empty_response, should_update_metrics):
""" Handles successful response
:type res: ``requests.Response``
:param res: Response from API after the request for which to check error type
:type resp_type: ``str``
:param resp_type:
Determines which data format to return from the HTTP request. The default
is 'json'. Other options are 'text', 'content', 'xml' or 'response'. Use 'response'
to return the full response object.
:type empty_valid_codes: ``list``
:param empty_valid_codes: A list of all valid status codes of empty responses (usually only 204, but
can vary)
:type return_empty_response: ``bool``
:param response: Whether to return an empty response body if the response code is in empty_valid_codes
:type should_update_metrics ``bool``
:param should_update_metrics: Whether or not to update execution metrics according to response
"""
if should_update_metrics:
self._update_metrics(res, success=True)
if not empty_valid_codes:
empty_valid_codes = [204]
is_response_empty_and_successful = (res.status_code in empty_valid_codes)
if is_response_empty_and_successful and return_empty_response:
return res
return self.cast_response(res, resp_type)
def cast_response(self, res, resp_type, raise_on_error=True):
resp_type = resp_type.lower()
try:
if resp_type == 'json':
return res.json()
if resp_type == 'text':
return res.text
if resp_type == 'content':
return res.content
if resp_type == 'xml':
ET.fromstring(res.text)
if resp_type == 'response':
return res
return res
except ValueError as exception:
if raise_on_error:
raise DemistoException('Failed to parse {} object from response: {}' # type: ignore[str-bytes-safe]
.format(resp_type, res.content), exception, res)
def _update_metrics(self, res, success):
""" Updates execution metrics based on response and success flag.
:type response: ``requests.Response``
:param response: Response from API after the request for which to check error type
:type success: ``bool``
:param success: Wheter the request succeeded or failed
"""
if success:
if not self.is_polling_in_progress(res):
self.execution_metrics.success += 1
else:
error_type = self.determine_error_type(res)
if error_type == ErrorTypes.QUOTA_ERROR:
self.execution_metrics.quota_error += 1
elif error_type == ErrorTypes.AUTH_ERROR:
self.execution_metrics.auth_error += 1
elif error_type == ErrorTypes.SERVICE_ERROR:
self.execution_metrics.service_error += 1
elif error_type == ErrorTypes.GENERAL_ERROR:
self.execution_metrics.general_error += 1
def determine_error_type(self, response):
""" Determines the type of error based on response status code and content.
Note: this method can be overriden by subclass when implementing execution metrics.
:type response: ``requests.Response``
:param response: Response from API after the request for which to check error type
:return: The error type if found, otherwise None
:rtype: ``ErrorTypes``
"""
if response.status_code == 401:
return ErrorTypes.AUTH_ERROR
elif response.status_code == 429:
return ErrorTypes.QUOTA_ERROR
elif response.status_code == 500:
return ErrorTypes.SERVICE_ERROR
return ErrorTypes.GENERAL_ERROR
def is_polling_in_progress(self, response):
"""If thie response indicates polling operation in progress, return True.
Note: this method should be overriden by subclass when implementing polling reputation commands
with execution metrics.
:type response: ``requests.Response``
:param response: Response from API after the request for which to check the polling status
:return: Whether the response indicates polling in progress
:rtype: ``bool``
"""
return False
def _is_status_code_valid(self, response, ok_codes=None):
"""If the status code is OK, return 'True'.
:type response: ``requests.Response``
:param response: Response from API after the request for which to check the status.
:type ok_codes: ``tuple`` or ``list``
:param ok_codes:
The request codes to accept as OK, for example: (200, 201, 204). If you specify
"None", will use response.ok.
:return: Whether the status of the response is valid.
:rtype: ``bool``
"""
# Get wanted ok codes
status_codes = ok_codes if ok_codes else self._ok_codes
if status_codes:
return response.status_code in status_codes
return response.ok
def client_error_handler(self, res):
"""Generic handler for API call error
Constructs and throws a proper error for the API call response.
:type response: ``requests.Response``
:param response: Response from API after the request for which to check the status.
"""
err_msg = 'Error in API call [{}] - {}'.format(res.status_code, res.reason)
try:
# Try to parse json error response
error_entry = res.json()
err_msg += '\n{}'.format(json.dumps(error_entry))
raise DemistoException(err_msg, res=res)
except ValueError:
err_msg += '\n{}'.format(res.text)
raise DemistoException(err_msg, res=res)
def _return_execution_metrics_results(self):
""" Returns execution metrics results.
Might raise an AttributeError exception if execution_metrics is not initialized.
"""
try:
if self.execution_metrics.metrics:
return_results(cast(CommandResults, self.execution_metrics.metrics))
except AttributeError:
pass
def generic_http_request(method,
server_url,
timeout=60,
verify=True,
proxy=False,
client_headers=None,
headers=None,
url_suffix=None,
data=None,
ok_codes=None,
auth=None,
error_handler=None,
files=None,
params=None,
retries=0,
resp_type='json',
status_list_to_retry=None,
json_data=None,
return_empty_response=False,
backoff_factor=5,
raise_on_redirect=False,
raise_on_status=False,
empty_valid_codes=None,
params_parser=None,
with_metrics=False,
**kwargs
):
"""
A wrapper for the BaseClient._http_request() method, that allows performing HTTP requests without initiating a BaseClient object.
Note: Avoid using this method if unnecessary. It is more recommended to use the BaseClient class.
Args:
method (str): HTTP request method (e.g., GET, POST, PUT, DELETE).
server_url (str): Base URL of the server.
timeout (int, optional): Timeout in seconds for the request (defaults to 10).
verify (bool, optional): Whether to verify SSL certificates (defaults to True).
proxy (bool or str, optional): Use a proxy server. Can be a boolean (defaults to False)
or a proxy URL string.
client_headers (dict, optional): Additional headers to be included in all requests
made by the client (overrides headers argument).
headers (dict, optional): Additional headers for this specific request.
url_suffix (str, optional): Path suffix to be appended to the server URL.
data (object, optional): Data to be sent in the request body (e.g., dictionary for POST requests).
ok_codes (list of int, optional): A list of HTTP status codes that are considered successful responses
(defaults to [200]).
auth (tuple, optional): Authentication credentials (username, password) for the request.
error_handler (callable, optional): Function to handle request errors.
files (dict, optional): Dictionary of files to be uploaded (for multipart/form-data requests).
params (dict, optional): URL parameters to be included in the request.
retries (int, optional): Number of times to retry the request on failure (defaults to 0).
retries (int, optional): Number of times to retry the request on failure (defaults to 0).
status_list_to_retry (int, optional): A set of integer HTTP status codes that we should force a retry on.
A retry is initiated if the request method is in ['GET', 'POST', 'PUT']
and the response status code is in ``status_list_to_retry``.
resp_type (iterable, optional): Determines which data format to return from the HTTP request. The default
is 'json'.
json_data (dict, optional): The dictionary to send in a 'POST' request.
backoff_factor (float, optional): A backoff factor to apply between attempts after the second try
(most errors are resolved immediately by a second try without a
delay). urllib3 will sleep for::
{backoff factor} * (2 ** ({number of total retries} - 1))
seconds. If the backoff_factor is 0.1, then :func:`.sleep` will sleep
for [0.0s, 0.2s, 0.4s, ...] between retries. It will never be longer
than :attr:`Retry.BACKOFF_MAX`.
By default, backoff_factor set to 5
raise_on_redirect (bool, optional): Whether, if the number of redirects is
exhausted, to raise a MaxRetryError, or to return a response with a
response code in the 3xx range.
raise_on_status (bool,optional): Similar meaning to ``raise_on_redirect``:
whether we should raise an exception, or return a response,
if status falls in ``status_forcelist`` range and retries have
been exhausted.
empty_valid_codes (list, optional): A list of all valid status codes of empty responses (usually only 204, but
can vary)
return_empty_response (bool, optional): Whether to return an empty response body if the response code is in empty_valid_codes
params_parser (callable, optional): How to quote the params. By default, spaces are replaced with `+` and `/` to `%2F`.
see here for more info: https://docs.python.org/3/library/urllib.parse.html#urllib.parse.urlencode
Note! supported only in python3.
with_metrics (bool, optional): Whether or not to calculate execution metrics from the response
Returns:
:return: Depends on the resp_type parameter
:rtype: ``dict`` or ``str`` or ``bytes`` or ``xml.etree.ElementTree.Element`` or ``requests.Response``
Raises:
exceptions.RequestException: If an error occurs during the request.
"""
client = BaseClient(base_url=server_url,
verify=verify,
proxy=proxy,
ok_codes=ok_codes,
headers=client_headers,
auth=auth,
timeout=timeout
)
return client._http_request(method=method, url_suffix=url_suffix, data=data, ok_codes=ok_codes, error_handler=error_handler,
headers=headers, files=files, params=params, retries=retries, resp_type=resp_type,
status_list_to_retry=status_list_to_retry, json_data=json_data,
return_empty_response=return_empty_response, backoff_factor=backoff_factor,
raise_on_redirect=raise_on_redirect, raise_on_status=raise_on_status,
empty_valid_codes=empty_valid_codes, params_parser=params_parser, with_metrics=with_metrics,
**kwargs)
def batch(iterable, batch_size=1):
"""Gets an iterable and yields slices of it.
:type iterable: ``list``
:param iterable: list or other iterable object.
:type batch_size: ``int``
:param batch_size: the size of batches to fetch
:rtype: ``list``
:return:: Iterable slices of given
"""
current_batch = iterable[:batch_size]
not_batched = iterable[batch_size:]
while current_batch:
yield current_batch
current_batch = not_batched[:batch_size]
not_batched = not_batched[batch_size:]
def dict_safe_get(dict_object, keys, default_return_value=None, return_type=None, raise_return_type=True):
"""Recursive safe get query (for nested dicts and lists), If keys found return value otherwise return None or default value.
Example:
>>> data = {"something" : {"test": "A"}}
>>> dict_safe_get(data, ['something', 'test'])
>>> 'A'
>>> dict_safe_get(data, ['something', 'else'], 'default value')
>>> 'default value'
:type dict_object: ``dict``
:param dict_object: dictionary to query.
:type keys: ``list``
:param keys: keys for recursive get.
:type default_return_value: ``object``
:param default_return_value: Value to return when no key available.
:type return_type: ``type``
:param return_type: Excepted return type.
:type raise_return_type: ``bool``
:param raise_return_type: Whether to raise an error when the value didn't match the expected return type.
:rtype: ``object``
:return:: Value from nested query.
"""
return_value = dict_object
for key in keys:
try:
return_value = return_value[key]
except (KeyError, TypeError, IndexError, AttributeError):
return_value = default_return_value
break
if return_type and not isinstance(return_value, return_type):
if raise_return_type:
raise TypeError("Safe get Error:\nDetails: Return Type Error Excepted return type {0},"
" but actual type from nested dict/list is {1} with value {2}.\n"
"Query: {3}\nQueried object: {4}".format(return_type, type(return_value),
return_value, keys, dict_object))
return_value = default_return_value
return return_value
CONTEXT_UPDATE_RETRY_TIMES = 3
MIN_VERSION_FOR_VERSIONED_CONTEXT = '6.0.0'
def merge_lists(original_list, updated_list, key):
"""
Replace values in a list with those in an updated list.
Example:
>>> original = [{'id': '1', 'updated': 'n'}, {'id': '2', 'updated': 'n'}, {'id': '11', 'updated': 'n'}]
>>> updated = [{'id': '1', 'updated': 'y'}, {'id': '3', 'updated': 'y'}, {'id': '11', 'updated': 'n',
>>> 'remove': True}]
>>> result = [{'id': '1', 'updated': 'y'}, {'id': '2', 'updated': 'n'}, {'id': '3', 'updated': 'y'}]
:type original_list: ``list``
:param original_list: The original list.
:type updated_list: ``list``
:param updated_list: The updated list.
:type key: ``str``
:param key: The key to replace elements by.
:rtype: ``list``
:return: The merged list.
"""
original_dict = {element[key]: element for element in original_list}
updated_dict = {element[key]: element for element in updated_list}
original_dict.update(updated_dict)
removed = [obj for obj in original_dict.values() if obj.get('remove', False) is True]
for r in removed:
demisto.debug('Removing from integration context: {}'.format(str(r)))
merged_list = [obj for obj in original_dict.values() if obj.get('remove', False) is False]
return merged_list
def set_integration_context(context, sync=True, version=-1):
"""
Sets the integration context.
:type context: ``dict``
:param context: The context to set.
:type sync: ``bool``
:param sync: Whether to save the context directly to the DB.
:type version: ``Any``
:param version: The version of the context to set.
:rtype: ``dict``
:return: The new integration context
"""
demisto.debug('Setting integration context')
if is_versioned_context_available():
demisto.debug('Updating integration context with version {}. Sync: {}'.format(version, sync))
return demisto.setIntegrationContextVersioned(context, version, sync)
else:
return demisto.setIntegrationContext(context)
def get_integration_context(sync=True, with_version=False):
"""
Gets the integration context.
:type sync: ``bool``
:param sync: Whether to get the integration context directly from the DB.
:type with_version: ``bool``
:param with_version: Whether to return the version.
:rtype: ``dict``
:return: The integration context.
"""
if is_versioned_context_available():
integration_context = demisto.getIntegrationContextVersioned(sync)
if with_version:
return integration_context
else:
if isinstance(integration_context, list):
demisto.error("The integration context is a list with {} items".format(len(integration_context)))
return integration_context.get("context", {})
else:
return demisto.getIntegrationContext()
def is_versioned_context_available():
"""
Determines whether versioned integration context is available according to the server version.
:rtype: ``bool``
:return: Whether versioned integration context is available
"""
return is_demisto_version_ge(MIN_VERSION_FOR_VERSIONED_CONTEXT)
def set_to_integration_context_with_retries(context, object_keys=None, sync=True,
max_retry_times=CONTEXT_UPDATE_RETRY_TIMES):
"""
Update the integration context with a dictionary of keys and values with multiple attempts.
The function supports merging the context keys using the provided object_keys parameter.
If the version is too old by the time the context is set,
another attempt will be made until the limit after a random sleep.
:type context: ``dict``
:param context: A dictionary of keys and values to set.
:type object_keys: ``dict``
:param object_keys: A dictionary to map between context keys and their unique ID for merging them.
:type sync: ``bool``
:param sync: Whether to save the context directly to the DB.
:type max_retry_times: ``int``
:param max_retry_times: The maximum number of attempts to try.
:rtype: ``None``
:return: None
"""
attempt = 0
# do while...
while True:
if attempt == max_retry_times:
raise Exception('Failed updating integration context. Max retry attempts exceeded.')
# Update the latest context and get the new version
integration_context, version = update_integration_context(context, object_keys, sync)
demisto.debug('Attempting to update the integration context with version {}.'.format(version))
# Attempt to update integration context with a version.
# If we get a ValueError (DB Version), then the version was not updated and we need to try again.
attempt += 1
try:
set_integration_context(integration_context, sync, version)
demisto.debug('Successfully updated integration context with version {}.'
''.format(version))
break
except ValueError as ve:
demisto.debug('Failed updating integration context with version {}: {} Attempts left - {}'
''.format(version, str(ve), CONTEXT_UPDATE_RETRY_TIMES - attempt))
# Sleep for a random time
time_to_sleep = randint(1, 100) / 1000
time.sleep(time_to_sleep) # pylint: disable=E9003
def get_integration_context_with_version(sync=True):
"""
Get the latest integration context with version, if available.
:type sync: ``bool``
:param sync: Whether to get the context directly from the DB.
:rtype: ``tuple``
:return: The latest integration context with version.
"""
latest_integration_context_versioned = get_integration_context(sync, with_version=True)
version = -1
if is_versioned_context_available():
integration_context = latest_integration_context_versioned.get('context', {})
if sync:
version = latest_integration_context_versioned.get('version', 0)
else:
integration_context = latest_integration_context_versioned
return integration_context, version
def update_integration_context(context, object_keys=None, sync=True):
"""
Update the integration context with a given dictionary after merging it with the latest integration context.
:type context: ``dict``
:param context: The keys and values to update in the integration context.
:type object_keys: ``dict``
:param object_keys: A dictionary to map between context keys and their unique ID for merging them
with the latest context.
:type sync: ``bool``
:param sync: Whether to use the context directly from the DB.
:rtype: ``tuple``
:return: The updated integration context along with the current version.
"""
integration_context, version = get_integration_context_with_version(sync)
if not object_keys:
object_keys = {}
for key, _ in context.items():
latest_object = json.loads(integration_context.get(key, '[]'))
updated_object = context[key]
if key in object_keys:
merged_list = merge_lists(latest_object, updated_object, object_keys[key])
integration_context[key] = json.dumps(merged_list)
else:
integration_context[key] = json.dumps(updated_object)
return integration_context, version
class DemistoException(Exception):
def __init__(self, message, exception=None, res=None, error_type=None, *args):
self.res = res
self.message = message
self.exception = exception
self.error_type = error_type
super(DemistoException, self).__init__(message, exception, error_type, *args)
def __str__(self):
return str(self.message)
class UcpException(DemistoException):
"""Exception for UCP (Unified Connector Platform) authentication errors.
Inherits from ``DemistoException`` and provides a default, user-friendly
message that does **not** expose internal terminology (UCP, connector, etc.).
Callers should log diagnostics via ``demisto.error()`` *before* raising,
then simply ``raise UcpException()`` to surface the generic message.
A custom *message* can be passed when a more specific (but still
user-safe) description is appropriate.
"""
DEFAULT_MESSAGE = (
'An authentication configuration error occurred. '
'Please verify the integration instance configuration and try again. '
'If the problem persists, contact Cortex support.'
)
def __init__(self, message=None, exception=None, res=None, error_type=None, *args):
super(UcpException, self).__init__(
message or self.DEFAULT_MESSAGE, exception=exception, res=res, error_type=error_type, *args
)
# -- Agentix Standardized Error Hierarchy -----------------------------
#
# These exceptions provide structured error information (error_code,
# details dict) that ``return_error`` serializes via ``ExtendedPayload``
# so that LLM agents can programmatically distinguish error categories
# and decide whether to retry, fix an argument, or escalate.
#
# Hierarchy:
# DemistoException
# +-- CortexError (base - carries error_code + details)
# +-- CortexMissingArgError
# +-- CortexInvalidArgError
# +-- CortexConflictingArgsError
# +-- CortexResourceNotFoundError
# +-- CortexExternalApiError
# | +-- CortexAuthError
# | +-- CortexRateLimitError
# | +-- CortexTimeoutError
# | +-- CortexConnectionError
# +-- CortexParseError
# +-- CortexPermissionError
# +-- CortexExecutionError
class RetryGuidance(object):
"""Whether (and how) an operation that failed is worth retrying.
Used by :class:`CortexError` to append an automatic retry hint to the
error message and to expose machine-readable guidance via
``ExtendedPayload`` so that LLM agents can decide whether to retry, fix an
argument, or escalate.
* :data:`NOT_RETRYABLE` - retrying as-is will not help (e.g. invalid
credentials, missing permissions). Requires human/config intervention.
* :data:`RETRY_AFTER_FIX` - retrying can succeed only after the *input* is
corrected (e.g. a missing or invalid argument).
* :data:`RETRY_LATER` - the same request may succeed if retried later,
typically after a short wait (e.g. rate limiting, timeouts, transient
connection issues).
:return: None
:rtype: ``None``
"""
NOT_RETRYABLE = "not_retryable"
RETRY_AFTER_FIX = "retry_after_fix"
RETRY_LATER = "retry_later"
# Human-readable hint appended to the error message for each guidance value.
_HINTS = {
NOT_RETRYABLE: "Retrying will not help; this requires fixing the configuration or contacting support.",
RETRY_AFTER_FIX: "You can retry after correcting the input.",
RETRY_LATER: "This may be transient - you can retry the same request later.",
}
@classmethod
def hint(cls, guidance):
# type: (str) -> str
"""Return the human-readable retry hint for a guidance value.
:type guidance: ``str``
:param guidance: A :class:`RetryGuidance` value.
:return: The human-readable retry hint, or an empty string if unknown.
:rtype: ``str``
"""
return cls._HINTS.get(guidance, "")
@classmethod
def is_retryable(cls, guidance):
# type: (str) -> bool
"""Return whether the given guidance value indicates a retry may help.
:type guidance: ``str``
:param guidance: A :class:`RetryGuidance` value.
:return: ``True`` if a retry may help, ``False`` otherwise.
:rtype: ``bool``
"""
return guidance in (cls.RETRY_AFTER_FIX, cls.RETRY_LATER)
# Key used inside ``details`` (and thus ``ExtendedPayload``) to carry the
# retry guidance and the boolean retryable flag.
EXTENDED_PAYLOAD_RETRY_GUIDANCE_KEY = 'retry_guidance'
EXTENDED_PAYLOAD_RETRYABLE_KEY = 'retryable'
class CortexError(DemistoException):
"""Base exception for all content standardized errors.
Carries structured error information including ``error_code``,
``retry_guidance`` and ``details`` that are attached to the error entry via
``ExtendedPayload`` for LLM agent consumption.
+======================================================================+
| WHEN TO USE |
| Generic fallback for an error that fits none of the specific |
| subclasses below. PREFER a specific subclass whenever one applies - |
| it sets the correct error_code and retry_guidance automatically, |
| letting LLM agents (Agentix) classify the failure and decide whether |
| to retry. Use the base class directly only as a last resort. |
+======================================================================+
:type override_message: ``str``
:param override_message: An OPTIONAL, fully-custom message that REPLACES the
automatically-built message for human (non-Agentix) display. Its primary
purpose is **backward compatibility** - preserving an existing,
hand-written error string while still gaining the structured
``error_code`` / ``retry_guidance`` metadata. Leave it ``None`` to use
the standardized auto-built message (the recommended default).
Note: it is ignored for Agentix callers, who always receive the
machine-friendly :meth:`auto_message`. The structured arguments of each
subclass (e.g. the argument name, status code) build the auto message
and populate ``details`` regardless of this value.
:type error_code: ``str``
:param error_code: Machine-readable code from :class:`CortexErrorCode`.
:type details: ``dict``
:param details: Arbitrary key/value pairs providing context about the error.
:return: None
:rtype: ``None``
"""
error_code = CortexErrorCode.INTERNAL_ERROR # type: str
# Default, auto-generated message used by ``build_message`` when no explicit
# message is supplied. Subclasses override this (or ``build_message``) to
# provide a category-specific default.
_default_message = 'An error occurred.' # type: str
# Whether/how this error category is worth retrying. Subclasses override
# this with the appropriate :class:`RetryGuidance` value. ``None`` means
# "unknown" and no retry hint is added.
retry_guidance = None # type: Optional[str]
def __init__(self, override_message=None, error_code=None, details=None, **kwargs):
# Respect an instance-level error_code already set by a subclass
# __init__ (e.g. CortexExternalApiError status-code auto-classification) before
# delegating here; then the explicit param; then the class default.
self.error_code = error_code or self.__dict__.get('error_code') or self.__class__.error_code
self.details = details or {}
# Stores the optional, caller-supplied override message (if any) so that
# ``build_message`` can return it instead of the auto-built message.
# Subclasses that rely solely on the auto-built message pass
# ``override_message=None``.
self._custom_message = override_message
# Expose retry guidance (machine-readable) in details/ExtendedPayload.
if self.retry_guidance is not None:
self.details.setdefault(EXTENDED_PAYLOAD_RETRY_GUIDANCE_KEY, self.retry_guidance)
self.details.setdefault(
EXTENDED_PAYLOAD_RETRYABLE_KEY, RetryGuidance.is_retryable(self.retry_guidance)
)
super(CortexError, self).__init__(self.build_message(), error_type=self.error_code, **kwargs)
def _base_message(self):
"""Return the *auto-generated* message body, ignoring any custom message.
Subclasses override this to build their category-specific message from
their own arguments (e.g. the missing argument name, the resource
identifier, the HTTP status code). This method must NOT consult
:attr:`_custom_message` - that is handled centrally by
:meth:`build_message`. The retry hint is appended by
:meth:`build_message` / :meth:`auto_message`.
:return: The auto-generated error message body (without retry hint).
:rtype: ``str``
"""
return self._default_message
def _with_retry_hint(self, message):
"""Append the retry hint (if any) to the given message."""
hint = RetryGuidance.hint(self.retry_guidance) if self.retry_guidance is not None else ""
if hint:
return "{} {}".format(message, hint)
return message
def auto_message(self):
"""Build the automatic, unified message - always ignoring any custom message.
This is the machine-friendly, standardized message used for Agentix
callers. It is built purely from the error's structured arguments
(via :meth:`_base_message`) plus the automatic retry hint, regardless of
whether a custom ``message`` was supplied to ``__init__``.
:return: The automatic, unified error message.
:rtype: ``str``
"""
return self._with_retry_hint(self._base_message())
def build_message(self):
"""Build the human-readable message to display for this error.
When an explicit ``message`` was supplied by the caller, it is returned
verbatim (no retry hint) so callers retain full control over the
wording. Otherwise the automatic message (see :meth:`auto_message`) is
returned.
:return: The error message to display.
:rtype: ``str``
"""
if self._custom_message:
return self._custom_message
return self.auto_message()
class CortexMissingArgError(CortexError):
"""Raised when a required argument (or one of several) is not provided.
+======================================================================+
| WHEN TO USE |
| A mandatory command/script argument was not supplied at all (missing |
| or empty). Also for "at least one of these arguments is required". |
| DO NOT use for an argument that WAS provided but holds a bad value - |
| use CortexInvalidArgError instead. |
+======================================================================+
Supports three shapes, selected automatically:
* **Single required argument** - pass a single name as ``arg_name``::
CortexMissingArgError('hostname')
# "Required argument 'hostname' was not provided."
* **At least one of several** (``require_one=True``, the default when a
list/tuple of names is passed) - the user must supply at least one of
the listed arguments::
CortexMissingArgError(['endpoint_id', 'endpoint_ip', 'endpoint_hostname'])
# "At least one of the following arguments must be provided:
# 'endpoint_id', 'endpoint_ip', 'endpoint_hostname'."
* **All of several required** (``require_one=False``) - every listed
argument is mandatory::
CortexMissingArgError(['user_id', 'token'], require_one=False)
# "The following required arguments were not provided: 'user_id', 'token'."
:type arg_name: ``str`` or ``list``
:param arg_name: Name of the missing argument, or a list of argument names.
:type override_message: ``str``
:param override_message: Optional custom message that overrides the
auto-built one (for backward compatibility). Auto-generated if omitted.
:type require_one: ``bool``
:param require_one: When a list of names is provided, whether only *one* of
them is required (``True``, default) or *all* of them (``False``).
Ignored when a single argument name is provided.
:return: None
:rtype: ``None``
"""
error_code = CortexErrorCode.MISSING_ARGUMENT
retry_guidance = RetryGuidance.RETRY_AFTER_FIX
def __init__(self, arg_name, override_message=None, require_one=True):
# Normalize to a list internally while remembering whether the caller
# passed a single name or several.
if isinstance(arg_name, (list, tuple, set)):
self.arg_names = [str(a) for a in arg_name]
self._is_multi = True
else:
self.arg_names = [str(arg_name)]
self._is_multi = False
self.arg_name = self.arg_names[0] if not self._is_multi else None
self.require_one = require_one
details = {"arguments": self.arg_names} # type: dict
if not self._is_multi:
details["argument"] = self.arg_names[0]
else:
details["require_one"] = require_one
super(CortexMissingArgError, self).__init__(override_message, details=details)
def _quoted_args(self):
return ', '.join("'{}'".format(a) for a in self.arg_names)
def _base_message(self):
if not self._is_multi:
return "Required argument '{}' was not provided.".format(self.arg_names[0])
if self.require_one:
return "At least one of the following arguments must be provided: {}.".format(self._quoted_args())
return "The following required arguments were not provided: {}.".format(self._quoted_args())
class CortexInvalidArgError(CortexError):
"""Raised when an argument has an invalid value.
+======================================================================+
| WHEN TO USE |
| An argument WAS provided but its value is unacceptable - wrong |
| format, out of range, or not an allowed value (e.g. a non-numeric |
| "limit", an unknown enum option, a malformed date). |
| DO NOT use for: a missing argument (CortexMissingArgError), |
| contradicting arguments (CortexConflictingArgsError), or a parsing |
| failure of API response data (CortexParseError). |
+======================================================================+
:type arg_name: ``str``
:param arg_name: Name of the invalid argument.
:type value: ``any``
:param value: The invalid value that was provided.
:type reason: ``str``
:param reason: Explanation of why the value is invalid.
:type allowed_values: ``list``
:param allowed_values: List of acceptable values, if applicable.
:type override_message: ``str``
:param override_message: Optional custom message that overrides the
auto-built one (for backward compatibility). Auto-generated if omitted.
:return: None
:rtype: ``None``
"""
error_code = CortexErrorCode.INVALID_ARGUMENT
retry_guidance = RetryGuidance.RETRY_AFTER_FIX
def __init__(self, arg_name, value=None, reason=None, allowed_values=None, override_message=None):
self.arg_name = arg_name
self.value = value
self.reason = reason
self.allowed_values = allowed_values
details = {"argument": arg_name}
if value is not None:
details["value"] = str(value)
if allowed_values:
details["allowed_values"] = [str(v) for v in allowed_values]
super(CortexInvalidArgError, self).__init__(override_message, details=details)
def _base_message(self):
parts = ["Invalid value for argument '{}'".format(self.arg_name)]
if self.value is not None:
parts.append(": got '{}'".format(self.value))
if self.reason:
parts.append(". {}".format(self.reason))
if self.allowed_values:
parts.append(". Allowed values: {}".format(', '.join(str(v) for v in self.allowed_values)))
return "".join(parts)
class CortexConflictingArgsError(CortexError):
"""Raised when arguments contradict each other.
+======================================================================+
| WHEN TO USE |
| Two or more individually-valid arguments cannot be used together |
| (mutually exclusive), or the combination is contradictory (e.g. both |
| ip and hostname given when only one is allowed, or start_time later |
| than end_time). |
| DO NOT use for: a single bad value (CortexInvalidArgError) or a |
| missing argument (CortexMissingArgError). |
+======================================================================+
Builds a smart, actionable message that explains *which* arguments
conflict, *why* they conflict, and *what* a valid combination looks like.
There are two common conflict shapes, selected automatically:
* **Mutually exclusive** (``mutually_exclusive=True`` - the default when
``arguments`` are given): only one of the listed arguments may be
provided at a time. The message tells the user to pick exactly one.
* **Free-form**: when ``reason`` / ``resolution`` are supplied (or a custom
``message`` is passed), those are used to describe the conflict and the
valid path forward.
:type arguments: ``list``
:param arguments: Names of the conflicting arguments that were supplied together.
:type override_message: ``str``
:param override_message: Optional fully-custom message that overrides the
auto-built one (for backward compatibility).
:type reason: ``str``
:param reason: Optional explanation of *why* the arguments conflict.
:type resolution: ``str``
:param resolution: Optional explanation of *what* would be a valid combination.
When omitted and the conflict is mutually exclusive, a sensible
"provide only one of ..." resolution is generated automatically.
:type mutually_exclusive: ``bool``
:param mutually_exclusive: Whether the listed arguments are mutually
exclusive. Defaults to ``True`` when ``arguments`` are provided.
:return: None
:rtype: ``None``
"""
error_code = CortexErrorCode.CONFLICTING_ARGUMENTS
retry_guidance = RetryGuidance.RETRY_AFTER_FIX
def __init__(self, override_message=None, arguments=None, reason=None, resolution=None, mutually_exclusive=None):
self.arguments = list(arguments or [])
self.reason = reason
self.resolution = resolution
# Default to mutual exclusivity when a list of arguments is provided
# and the caller did not explicitly say otherwise.
self.mutually_exclusive = bool(self.arguments) if mutually_exclusive is None else mutually_exclusive
details = {"arguments": self.arguments} # type: dict
if reason:
details["reason"] = reason
if resolution:
details["resolution"] = resolution
super(CortexConflictingArgsError, self).__init__(override_message, details=details)
def _quoted_args(self):
return ', '.join("'{}'".format(a) for a in self.arguments)
def _base_message(self):
parts = []
# 1. What conflicts.
if self.arguments:
parts.append("Conflicting arguments: {}.".format(self._quoted_args()))
else:
parts.append("Conflicting arguments were provided.")
# 2. Why they conflict.
if self.reason:
parts.append(self.reason)
elif self.mutually_exclusive and self.arguments:
parts.append("These arguments are mutually exclusive and cannot be used together.")
# 3. What would be valid.
if self.resolution:
parts.append(self.resolution)
elif self.mutually_exclusive and self.arguments:
parts.append("Provide only one of: {}.".format(self._quoted_args()))
return ' '.join(parts)
class CortexResourceNotFoundError(CortexError):
"""Raised when a requested resource is not found.
+======================================================================+
| WHEN TO USE |
| A specific entity the user asked for does not exist on the remote |
| system (e.g. an incident/ticket/endpoint/user ID returns 404 or an |
| empty result). The identifier was syntactically valid - it just has |
| no match. |
| DO NOT use for: a malformed identifier value (CortexInvalidArgError) |
| or a generic non-2xx API failure (CortexExternalApiError). |
+======================================================================+
:type resource_type: ``str``
:param resource_type: Type of resource (e.g. "endpoint", "incident").
:type identifier: ``any``
:param identifier: The identifier that was looked up.
:type override_message: ``str``
:param override_message: Optional custom message that overrides the
auto-built one (for backward compatibility). Auto-generated if omitted.
:return: None
:rtype: ``None``
"""
error_code = CortexErrorCode.RESOURCE_NOT_FOUND
retry_guidance = RetryGuidance.RETRY_AFTER_FIX
def __init__(self, resource_type, identifier=None, override_message=None):
self.resource_type = resource_type
self.identifier = identifier
super(CortexResourceNotFoundError, self).__init__(override_message, details={
"resource_type": resource_type,
"identifier": str(identifier) if identifier else None,
})
def _base_message(self):
if self.identifier:
return "{} '{}' not found".format(self.resource_type, self.identifier)
return "{} not found".format(self.resource_type)
class CortexExternalApiError(CortexError):
"""Raised when an external API returns an error.
+========================================================================+
| WHEN TO USE |
| A request to the third-party/external service failed with a non-2xx |
| HTTP status (or an equivalent transport error) and no more specific |
| subclass applies. Pass status_code so the error is auto-classified |
| (auth / quota / service) and retry guidance is set correctly. |
| PREFER a specific subclass when it fits: CortexAuthError (401/403), |
| CortexRateLimitError (429), CortexTimeoutError, CortexConnectionError. |
| DO NOT use for: a valid response whose BODY cannot be parsed |
| (CortexParseError) or a 404 for a specific entity |
| (CortexResourceNotFoundError). |
+========================================================================+
Automatically classifies by HTTP status code when ``api_error_type``
is not provided:
- 401/403 -> ``AUTH_ERROR``
- 429 -> ``QUOTA_ERROR``
- 5xx -> ``SERVICE_ERROR``
:type override_message: ``str``
:param override_message: Optional custom message that overrides the
auto-built one (for backward compatibility). Auto-generated if omitted.
:type status_code: ``int``
:param status_code: HTTP status code from the API response.
:type api_error_type: ``str``
:param api_error_type: A :class:`CortexErrorCode` value for explicit
classification (e.g. ``CortexErrorCode.SERVICE_ERROR``). When provided,
it becomes the error's ``error_code`` directly.
:type response_body: ``str`` or ``bytes``
:param response_body: The original API error body (truncated to 500 chars,
bytes are decoded). It is appended to the built/auto message
(``... Original API error: <body>``) so the raw API error travels
together with the unified, human-readable message.
:return: None
:rtype: ``None``
"""
error_code = CortexErrorCode.API_ERROR
# API/network errors are commonly transient, so default to "retry later".
# Auth-related status codes (401/403) override this to NOT_RETRYABLE below.
retry_guidance = RetryGuidance.RETRY_LATER
# Default, auto-generated message used when no explicit message is supplied.
_default_message = "An error occurred while communicating with the external API."
def __init__(self, override_message=None, status_code=None, api_error_type=None, response_body=None):
self.status_code = status_code
# Original error body returned by the API. It is appended to the built
# (auto) message rather than exposed as a separate field, so the raw API
# error travels together with the unified, human-readable message.
if response_body:
# Decode bytes cleanly so the original API error reads as text
# (avoids leaking a b'...' repr into the message).
if isinstance(response_body, bytes):
response_body = response_body.decode("utf-8", errors="replace")
response_body = str(response_body)[:MAX_API_RESPONSE_BODY_LENGTH]
self.response_body = response_body
details = {} # type: dict
if status_code:
details["status_code"] = status_code
# When the caller passes an explicit api_error_type (a CortexErrorCode
# value) it becomes the error_code directly.
if api_error_type:
self.error_code = api_error_type
details["api_error_type"] = api_error_type
# Otherwise auto-classify based on the HTTP status code.
elif status_code:
if status_code in (401, 403):
self.error_code = CortexErrorCode.AUTH_ERROR
details["api_error_type"] = CortexErrorCode.AUTH_ERROR
# Auth failures won't be fixed by retrying as-is.
self.retry_guidance = RetryGuidance.NOT_RETRYABLE
elif status_code == 429:
self.error_code = CortexErrorCode.QUOTA_ERROR
details["api_error_type"] = CortexErrorCode.QUOTA_ERROR
elif status_code >= 500:
self.error_code = CortexErrorCode.SERVICE_ERROR
details["api_error_type"] = CortexErrorCode.SERVICE_ERROR
super(CortexExternalApiError, self).__init__(override_message, details=details)
def _base_message(self):
message = self._default_message
if self.status_code:
message = "{} (HTTP {})".format(message, self.status_code)
# Append the original API error body (when available) so the raw API
# error is conveyed within the message itself.
if self.response_body:
message = "{} Original API error: {}".format(message, self.response_body)
return message
class CortexAuthError(CortexExternalApiError):
"""Raised when authentication fails (401/403).
+======================================================================+
| WHEN TO USE |
| The external service rejected the request due to authentication |
| problems - invalid/expired credentials, API key, or token. |
| Marked not-retryable (retrying as-is won't help). |
+======================================================================+
:return: None
:rtype: ``None``
"""
error_code = CortexErrorCode.AUTH_ERROR
retry_guidance = RetryGuidance.NOT_RETRYABLE
_default_message = "Authentication failed. Check your credentials or API key."
def __init__(self, override_message=None, **kwargs):
super(CortexAuthError, self).__init__(override_message, api_error_type=CortexErrorCode.AUTH_ERROR, **kwargs)
class CortexRateLimitError(CortexExternalApiError):
"""Raised when API rate limit is exceeded (429).
+======================================================================+
| WHEN TO USE |
| The external service throttled the request (rate/quota limit). |
| Marked retryable-later; pass retry_after when the service indicates |
| how long to wait. |
+======================================================================+
:type retry_after: ``int``
:param retry_after: Seconds to wait before retrying, if known.
:return: None
:rtype: ``None``
"""
error_code = CortexErrorCode.QUOTA_ERROR
retry_guidance = RetryGuidance.RETRY_LATER
_default_message = "API rate limit exceeded. Please retry later."
def __init__(self, override_message=None, retry_after=None, **kwargs):
self.retry_after = retry_after
super(CortexRateLimitError, self).__init__(override_message, api_error_type=CortexErrorCode.QUOTA_ERROR, **kwargs)
if retry_after:
self.details["retry_after_seconds"] = retry_after
def _base_message(self):
if self.retry_after:
return "{} Retry after {} seconds.".format(self._default_message, self.retry_after)
return self._default_message
class CortexTimeoutError(CortexExternalApiError):
"""Raised when an API request times out.
+======================================================================+
| WHEN TO USE |
| The request was sent but the external service did not respond within |
| the allotted time (read/connect timeout). Marked retryable-later. |
+======================================================================+
:return: None
:rtype: ``None``
"""
error_code = CortexErrorCode.TIMEOUT_ERROR
retry_guidance = RetryGuidance.RETRY_LATER
_default_message = "The request timed out. Please try again."
def __init__(self, override_message=None, **kwargs):
super(CortexTimeoutError, self).__init__(override_message, api_error_type=CortexErrorCode.TIMEOUT_ERROR, **kwargs)
class CortexConnectionError(CortexExternalApiError):
"""Raised when unable to connect to the service.
+======================================================================+
| WHEN TO USE |
| The connection to the external service could not be established at |
| all (DNS failure, connection refused/reset, network unreachable) - |
| i.e. there was no HTTP response. Marked retryable-later. |
| DO NOT use for: a connection that succeeded but returned an error |
| status - use CortexExternalApiError (or a more specific subclass). |
+======================================================================+
:return: None
:rtype: ``None``
"""
error_code = CortexErrorCode.CONNECTION_ERROR
retry_guidance = RetryGuidance.RETRY_LATER
_default_message = "Unable to connect to the service. Check network connectivity."
def __init__(self, override_message=None, **kwargs):
super(CortexConnectionError, self).__init__(override_message, api_error_type=CortexErrorCode.CONNECTION_ERROR, **kwargs)
class CortexParseError(CortexError):
"""Raised when data returned by the external API/service cannot be parsed.
+=======================================================================+
| WHEN TO USE |
| A RESPONSE received from the external service could not be decoded or |
| interpreted (invalid JSON/XML, unexpected schema, missing fields). |
| The call itself succeeded - the problem is the payload. |
| Marked not-retryable. |
| DO NOT use for: invalid user input (CortexInvalidArgError) or a |
| non-2xx API status (CortexExternalApiError). |
+=======================================================================+
:return: None
:rtype: ``None``
"""
error_code = CortexErrorCode.API_RESPONSE_PARSE_ERROR
retry_guidance = RetryGuidance.NOT_RETRYABLE
_default_message = "Failed to parse the response received from the API."
class CortexPermissionError(CortexError):
"""Raised when the user lacks required permissions.
+======================================================================+
| WHEN TO USE |
| The action is understood and authenticated, but the user/account is |
| not authorized to perform it (authorization, not authentication). |
| Marked not-retryable. |
| DO NOT use for: failed authentication/credentials (CortexAuthError). |
+======================================================================+
:return: None
:rtype: ``None``
"""
error_code = CortexErrorCode.PERMISSION_ERROR
retry_guidance = RetryGuidance.NOT_RETRYABLE
_default_message = "Permission denied. You do not have the required permissions to perform this action."
class CortexExecutionError(CortexError):
"""Raised when a command or script execution fails.
+=======================================================================+
| WHEN TO USE |
| A command/script failed during its own logic for a reason not covered |
| by the other categories (e.g. a sub-command returned an error, a |
| business rule could not be satisfied, an internal step failed). |
| Retry guidance is left unset (retryability depends on the case). |
| DO NOT use for: input validation, external API failures, or parsing |
| problems - prefer the dedicated subclasses for those. |
+=======================================================================+
:return: None
:rtype: ``None``
"""
error_code = CortexErrorCode.EXECUTION_ERROR
_default_message = "The command or script execution failed."
class GetRemoteDataArgs:
"""get-remote-data args parser
:type args: ``dict``
:param args: arguments for the command.
:return: No data returned
:rtype: ``None``
"""
def __init__(self, args):
self.remote_incident_id = args['id']
self.last_update = args['lastUpdate']
class GetModifiedRemoteDataArgs:
"""get-modified-remote-data args parser
:type args: ``dict``
:param args: arguments for the command.
:return: No data returned
:rtype: ``None``
"""
def __init__(self, args):
self.last_update = args['lastUpdate']
class UpdateRemoteSystemArgs:
"""update-remote-system args parser
:type args: ``dict``
:param args: arguments for the command of the command.
:return: No data returned
:rtype: ``None``
"""
def __init__(self, args):
self.data = args.get('data') # type: ignore
self.entries = args.get('entries')
self.incident_changed = args.get('incidentChanged')
self.remote_incident_id = args.get('remoteId')
self.inc_status = args.get('status')
self.delta = args.get('delta')
class GetRemoteDataResponse:
"""get-remote-data response parser
:type mirrored_object: ``dict``
:param mirrored_object: The object you are mirroring, in most cases the incident.
:type entries: ``list``
:param entries: The entries you want to add to the war room.
:return: No data returned
:rtype: ``None``
"""
def __init__(self, mirrored_object, entries):
self.mirrored_object = mirrored_object
self.entries = entries
def extract_for_local(self):
"""Extracts the response into the mirrored incident.
:return: List of details regarding the mirrored incident.
:rtype: ``list``
"""
if self.mirrored_object:
return [self.mirrored_object] + self.entries
class GetModifiedRemoteDataResponse:
"""get-modified-remote-data response parser
:type modified_incident_ids: ``list``
:param modified_incident_ids: The incidents that were modified since the last check.
:return: No data returned
:rtype: ``None``
"""
def __init__(self, modified_incident_ids):
self.modified_incident_ids = modified_incident_ids
def to_entry(self):
"""Extracts the response
:return: List of incidents to run the get-remote-data command on.
:rtype: ``list``
"""
demisto.info('Modified incidents: {}'.format(self.modified_incident_ids))
return {'Contents': self.modified_incident_ids, 'Type': EntryType.NOTE, 'ContentsFormat': EntryFormat.JSON}
class SchemeTypeMapping:
"""Scheme type mappings builder.
:type type_name: ``str``
:param type_name: The name of the remote incident type.
:type fields: ``dict``
:param fields: The dict of fields to their description.
:return: No data returned
:rtype: ``None``
"""
def __init__(self, type_name='', fields=None):
self.type_name = type_name
self.fields = fields if fields else {}
def add_field(self, name, description=''):
"""Adds a field to the incident type mapping.
:type name: ``str``
:param name: The name of the field.
:type description: ``str``
:param description: The description for that field.a
:return: No data returned
:rtype: ``None``
"""
self.fields.update({
name: description
})
def extract_mapping(self):
"""Extracts the mapping into XSOAR mapping screen.
:return: the mapping object for the current field.
:rtype: ``dict``
"""
return {
self.type_name: self.fields
}
class GetMappingFieldsResponse:
"""Handler for the mapping fields object.
:type scheme_types_mapping: ``list``
:param scheme_types_mapping: List of all the mappings in the remote system.
:return: No data returned
:rtype: ``None``
"""
def __init__(self, scheme_types_mapping=None):
self.scheme_types_mappings = scheme_types_mapping if scheme_types_mapping else []
def add_scheme_type(self, scheme_type_mapping):
"""Add another incident type mapping.
:type scheme_type_mapping: ``dict``
:param scheme_type_mapping: mapping of a singular field.
:return: No data returned
:rtype: ``None``
"""
self.scheme_types_mappings.append(scheme_type_mapping)
def extract_mapping(self):
"""Extracts the mapping into XSOAR mapping screen.
:return: the mapping object for the current field.
:rtype: ``dict``
"""
all_mappings = {}
for scheme_types_mapping in self.scheme_types_mappings:
all_mappings.update(scheme_types_mapping.extract_mapping())
return all_mappings
def get_x_content_info_headers():
"""Get X-Content-* headers to send in outgoing requests to use when performing requests to
external services such as oproxy.
:return: headers dict
:rtype: ``dict``
"""
calling_context = demisto.callingContext.get('context', {})
brand_name = calling_context.get('IntegrationBrand', '')
instance_name = calling_context.get('IntegrationInstance', '')
headers = {
'X-Content-Version': CONTENT_RELEASE_VERSION,
'X-Content-Name': brand_name or instance_name or 'Name not found',
'X-Content-LicenseID': demisto.getLicenseID(),
'X-Content-Branch': CONTENT_BRANCH_NAME,
'X-Content-Server-Version': get_demisto_version_as_str(),
}
return headers
class BaseWidget:
@abstractmethod
def to_display(self):
pass
class TextWidget(BaseWidget):
"""Text Widget representation
:type text: ``str``
:param text: The text for the widget to display
:return: No data returned
:rtype: ``None``
"""
def __init__(self, text):
# type: (str) -> None
self.text = text
def to_display(self):
"""Text Widget representation
:type text: ``str``
:param text: The text for the widget to display
:return: No data returned
:rtype: ``None``
"""
return self.text
class TrendWidget(BaseWidget):
"""Trend Widget representation
:type current_number: ``int``
:param current_number: The Current number in the trend.
:type previous_number: ``int``
:param previous_number: The previous number in the trend.
:return: No data returned
:rtype: ``None``
"""
def __init__(self, current_number, previous_number):
# type: (int, int) -> None
self.current_number = current_number
self.previous_number = previous_number
def to_display(self):
return json.dumps({
'currSum': self.current_number,
'prevSum': self.previous_number
})
class NumberWidget(BaseWidget):
"""Number Widget representation
:type number: ``int``
:param number: The number for the widget to display.
:return: No data returned
:rtype: ``None``
"""
def __init__(self, number):
# type: (int) -> None
self.number = number
def to_display(self):
return self.number
class BarColumnPieWidget(BaseWidget):
"""Bar/Column/Pie Widget representation
:type categories: ``list``
:param categories: a list of categories to display(Better use the add_category function to populate the data.
:return: No data returned
:rtype: ``None``
"""
def __init__(self, categories=None):
# type: (list) -> None
self.categories = categories if categories else [] # type: List[dict]
def add_category(self, name, number):
"""Add a category to widget.
:type name: ``str``
:param name: the name of the category to add.
:type number: ``int``
:param number: the number value of the category.
:return: No data returned.
:rtype: ``None``
"""
self.categories.append({
'name': name,
'data': [number]
})
def to_display(self):
return json.dumps(self.categories)
class LineWidget(BaseWidget):
"""Line Widget representation
:type categories: ``Any``
:param categories: a list of categories to display(Better use the add_category function to populate the data.
:return: No data returned
:rtype: ``None``
"""
def __init__(self, categories=None):
# type: (list) -> None
self.categories = categories if categories else [] # type: List[dict]
def add_category(self, name, number, group):
"""Add a category to widget.
:type name: ``str``
:param name: the name of the category to add.
:type number: ``int``
:param number: the number value of the category.
:type group: ``str``
:param group: the name of the relevant group.
:return: No data returned
:rtype: ``None``
"""
self.categories.append({
'name': name,
'data': [number],
'groups': [
{
'name': group,
'data': [number]
},
]
})
def to_display(self):
processed_names = [] # type: List[str]
processed_categories = [] # type: List[dict]
for cat in self.categories:
if cat['name'] in processed_names:
for processed_category in processed_categories:
if cat['name'] == processed_category['name']:
processed_category['data'] = [processed_category['data'][0] + cat['data'][0]]
processed_category['groups'].extend(cat['groups'])
break
else:
processed_categories.append(cat)
processed_names.append(cat['name'])
return json.dumps(processed_categories)
class TableOrListWidget(BaseWidget):
"""Table/List Widget representation
:type data: ``Any``
:param data: a list of data to display(Better use the add_category function to populate the data.
:return: No data returned
:rtype: ``None``
"""
def __init__(self, data=None):
# type: (Any) -> None
self.data = data if data else []
if not isinstance(self.data, list):
self.data = [data]
def add_row(self, data):
"""Add a row to the widget.
:type data: ``Any``
:param data: the data to add to the list/table.
:return: No data returned
:rtype: ``None``
"""
self.data.append(data)
def to_display(self):
return json.dumps({
'total': len(self.data),
'data': self.data
})
class IndicatorsSearcher:
"""Used in order to search indicators by the paging or serachAfter param
:type page: ``int``
:param page: the number of page from which we start search indicators from.
:type filter_fields: ``Optional[str]``
:param filter_fields: comma separated fields to filter (e.g. "value,type")
:type from_date: ``Optional[str]``
:param from_date: the start date to search from.
:type query: ``Optional[str]``
:param query: indicator search query
:type to_date: ``Optional[str]``
:param to_date: the end date to search until to.
:type value: ``str``
:param value: the indicator value to search.
:type limit: ``Optional[int]``
:param limit: the current upper limit of the search (can be updated after init)
:type sort: ``List[Dict]``
:param sort: An array of sort params ordered by importance. Item structure: {"field": string, "asc": boolean}
:return: No data returned
:rtype: ``None``
"""
SEARCH_AFTER_TITLE = 'searchAfter'
def __init__(self,
page=0,
filter_fields=None,
from_date=None,
query=None,
size=100,
to_date=None,
value='',
limit=None,
sort=None,
search_after=None
):
# searchAfter is available in searchIndicators from version 6.1.0
self._can_use_search_after = True
# populateFields merged in https://github.com/demisto/server/pull/18398
self._can_use_filter_fields = True
self._search_after_param = search_after
self._page = page
self._filter_fields = filter_fields
self._total = None
self._from_date = from_date
self._query = query
self._size = size
self._to_date = to_date
self._value = value
self._limit = limit
self._total_iocs_fetched = 0
self._sort = sort
def __iter__(self):
return self
# python2
def next(self):
return self.__next__()
def __next__(self):
if self.is_search_done():
raise StopIteration
res = self.search_indicators_by_version(from_date=self._from_date,
query=self._query,
size=self._size,
to_date=self._to_date,
value=self._value)
fetched_len = len(res.get('iocs') or [])
if fetched_len == 0:
raise StopIteration
self._total_iocs_fetched += fetched_len
return res
@property
def page(self):
return self._page
@property
def total(self):
return self._total
@property
def limit(self):
return self._limit
@limit.setter
def limit(self, value):
self._limit = value
def is_search_done(self):
"""
Return True if one of these conditions is met (else False):
1. self.limit is set, and it's updated to be less or equal to zero - return True
2. for search_after if self.total was populated by a previous search, but no self._search_after_param
3. for page if self.total was populated by a previous search, but page is too large
"""
reached_limit = self.limit is not None and self.limit <= self._total_iocs_fetched
if reached_limit:
demisto.debug("IndicatorsSearcher has reached its limit: {}".format(self.limit))
# update limit to match _total_iocs_fetched value
if self._total_iocs_fetched > self.limit:
self.limit = self._total_iocs_fetched
return True
else:
if self.total is None:
return False
no_more_indicators = self.total and self._search_after_param is None
if no_more_indicators:
demisto.debug("IndicatorsSearcher can not fetch anymore indicators")
return no_more_indicators
def search_indicators_by_version(self, from_date=None, query='', size=100, to_date=None, value=''):
"""There are 2 cases depends on the sever version:
1. Search indicators using paging, raise the page number in each call.
2. Search indicators using searchAfter param, update the _search_after_param in each call.
:type from_date: ``Optional[str]``
:param from_date: the start date to search from.
:type query: ``Optional[str]``
:param query: indicator search query
:type size: ``int``
:param size: limit the number of returned results.
:type to_date: ``Optional[str]``
:param to_date: the end date to search until to.
:type value: ``str``
:param value: the indicator value to search.
:return: object contains the search results
:rtype: ``dict``
"""
search_args = assign_params(
fromDate=from_date,
toDate=to_date,
query=query,
size=size,
value=value,
searchAfter=self._search_after_param,
populateFields=self._filter_fields,
)
if is_demisto_version_ge('6.6.0'):
search_args['sort'] = self._sort
demisto.debug('IndicatorsSearcher: page {}, search_args: {}'.format(self._page, search_args))
res = demisto.searchIndicators(**search_args)
self._total = res.get('total')
demisto.debug('IndicatorsSearcher: page {}, result size: {}'.format(self._page, self._total))
# when total is None, there is a problem with the server for returning indicators, hence need to restart the container,
# see XSUP-26699
if self._total is None:
raise SystemExit(
"Encountered issue when trying to fetch indicators for integration in instance {integration}. "
"Restarting container and trying again.".format(integration=get_integration_instance_name())
)
if isinstance(self._page, int):
self._page += 1 # advance pages
self._search_after_param = res.get(self.SEARCH_AFTER_TITLE)
return res
class AutoFocusKeyRetriever:
"""AutoFocus API Key management class
:type api_key: ``str``
:param api_key: Auto Focus API key coming from the integration parameters
:type override_default_credentials: ``bool``
:param override_default_credentials: Whether to override the default credentials and use the
Cortex XSOAR given AutoFocus API Key
:return: No data returned
:rtype: ``None``
"""
def __init__(self, api_key):
# demisto.getAutoFocusApiKey() is available from version 6.2.0
if not api_key:
try:
api_key = demisto.getAutoFocusApiKey() # is not available on tenants
except ValueError as err:
raise DemistoException('AutoFocus API Key is only available on the main account for TIM customers. ' + str(err))
self.key = api_key
def get_feed_last_run():
"""
This function gets the feed's last run: using `demisto.getLastRun()`.
:rtype: ``dict``
:return: All indicators from the feed's last run
"""
feed_last_run = demisto.getLastRun() or {}
if not feed_last_run:
integration_ctx = demisto.getIntegrationContext()
if integration_ctx:
feed_last_run = integration_ctx
demisto.setLastRun(feed_last_run)
demisto.setIntegrationContext({})
return feed_last_run
def set_feed_last_run(last_run_indicators):
"""
This function sets the feed's last run: using `demisto.setLastRun()`.
:type last_run_indicators: ``dict``
:param last_run_indicators: Indicators to save in "lastRun" object.
:rtype: ``None``
:return: None
"""
demisto.setLastRun(last_run_indicators)
def set_last_mirror_run(last_mirror_run): # type: (Dict[Any, Any]) -> None
"""
This function sets the last run of the mirror, from XSOAR version 6.6.0, by using `demisto.setLastMirrorRun()`.
Before XSOAR version 6.6.0, we don't set the given data and an exception will be raised.
:type last_mirror_run: ``dict``
:param last_mirror_run: Data to save in the "LastMirrorRun" object.
:rtype: ``None``
:return: None
"""
if is_demisto_version_ge('6.6.0'):
try:
demisto.setLastMirrorRun(last_mirror_run)
except json.JSONDecodeError as e:
# see XSUP-24343
if not isinstance(last_mirror_run, dict):
raise TypeError("non-dictionary passed to set_last_mirror_run")
demisto.debug(
"encountered JSONDecodeError from server during setLastMirrorRun. As long as the value passed can be converted to json, this error can be ignored.")
demisto.debug(str(e))
else:
raise DemistoException("You cannot use setLastMirrorRun as your version is below 6.6.0")
def get_last_mirror_run(): # type: () -> Optional[Dict[Any, Any]]
"""
This function gets the last run of the mirror, from XSOAR version 6.6.0, using `demisto.getLastMirrorRun()`.
Before XSOAR version 6.6.0, the given data is not returned and an exception will be raised.
:rtype: ``dict``
:return: A dictionary representation of the data that was already set in the previous runs (or an empty dict if
we did not set anything yet).
"""
if is_demisto_version_ge('6.6.0'):
return demisto.getLastMirrorRun() or {}
raise DemistoException("You cannot use getLastMirrorRun as your version is below 6.6.0")
def support_multithreading(): # pragma: no cover
"""Adds lock on the calls to the Cortex XSOAR server from the Demisto object to support integration which use multithreading.
:return: No data returned
:rtype: ``None``
"""
global HAVE_SUPPORT_MULTITHREADING_CALLED_ONCE
if HAVE_SUPPORT_MULTITHREADING_CALLED_ONCE:
return
HAVE_SUPPORT_MULTITHREADING_CALLED_ONCE = True
global demisto
prev_do = demisto._Demisto__do # type: ignore[attr-defined]
demisto.lock = Lock() # type: ignore[attr-defined]
def locked_do(cmd):
if demisto.lock.acquire(timeout=60): # type: ignore[call-arg,attr-defined]
try:
return prev_do(cmd) # type: ignore[call-arg]
finally:
demisto.lock.release() # type: ignore[attr-defined]
else:
raise RuntimeError('Failed acquiring lock')
demisto._Demisto__do = locked_do # type: ignore[attr-defined]
def get_tenant_account_name():
"""Gets the tenant name from the server url.
:return: The account name.
:rtype: ``str``
"""
urls = demisto.demistoUrls()
server_url = urls.get('server', '')
account_name = ''
if '/acc_' in server_url:
tenant_name = server_url.split('acc_')[-1]
account_name = "acc_{}".format(tenant_name) if tenant_name != "" else ""
return account_name
def indicators_value_to_clickable(indicators):
"""
Function to get the indicator url link for indicators
:type indicators: ``dict`` + List[dict]
:param indicators: An indicator or a list of indicators
:rtype: ``dict``
:return: Key is the indicator, and the value is it's url in the server
"""
if not isinstance(indicators, (list, dict)):
return {}
if not isinstance(indicators, list):
indicators = [indicators]
res = {}
query = ' or '.join(['value:{indicator}'.format(indicator=indicator) for indicator in indicators])
indicator_searcher = IndicatorsSearcher(query=query)
for ioc_res in indicator_searcher:
for inidicator_data in ioc_res.get('iocs', []):
indicator = inidicator_data.get('value')
indicator_id = inidicator_data.get('id')
if not indicator or not indicator_id:
raise DemistoException('The response of indicator searcher is invalid')
indicator_url = os.path.join('#', 'indicator', indicator_id)
res[indicator] = '[{indicator}]({indicator_url})'.format(indicator=indicator, indicator_url=indicator_url)
return res
def get_message_threads_dump(_sig, _frame):
"""
Listener function to dump the threads to log info
:type _sig: ``int``
:param _sig: The signal number
:type _frame: ``Any``
:param _frame: The current stack frame
:return: Message to print.
:rtype: ``str``
"""
code = []
for threadId, stack in sys._current_frames().items():
code.append("\n# ThreadID: %s" % threadId)
for filename, lineno, name, line in traceback.extract_stack(stack):
code.append('File: "%s", line %d, in %s' % (filename, lineno, name))
if line:
code.append(" %s" % (line.strip()))
ret_value = '\n\n--- Start Threads Dump ---\n' \
+ '\n'.join(code) \
+ '\n\n--- End Threads Dump ---\n'
return ret_value
def get_message_memory_dump(_sig, _frame):
"""
Listener function to dump the memory to log info
:type _sig: ``int``
:param _sig: The signal number
:type _frame: ``Any``
:param _frame: The current stack frame
:return: Message to print.
:rtype: ``str``
"""
classes_dict = {} # type: Dict[str, Dict]
for obj in gc.get_objects():
size = sys.getsizeof(obj, 0)
if hasattr(obj, '__class__'):
cls = str(obj.__class__)[8:-2]
if cls in classes_dict:
current_class = classes_dict.get(cls, {}) # type: Dict
current_class['count'] += 1 # type: ignore
current_class['size'] += size # type: ignore
classes_dict[cls] = current_class
else:
current_class = {
'name': cls,
'count': 1,
'size': size,
}
classes_dict[cls] = current_class
classes_as_list = list(classes_dict.values())
ret_value = '\n\n--- Start Variables Dump ---\n'
ret_value += get_message_classes_dump(classes_as_list)
ret_value += '\n--- End Variables Dump ---\n\n'
ret_value = '\n\n--- Start Variables Dump ---\n'
ret_value += get_message_local_vars()
ret_value += get_message_global_vars()
ret_value += '\n--- End Variables Dump ---\n\n'
ret_value += get_message_modules_sizes()
return ret_value
def get_message_classes_dump(classes_as_list):
"""
A function that returns the printable message about classes dump
:type classes_as_list: ``list``
:param classes_as_list: The classes to print to the log
:return: Message to print.
:rtype: ``str``
"""
ret_value = '\n\n--- Start Memory Dump ---\n'
ret_value += '\n--- Start Top {} Classes by Count ---\n\n'.format(PROFILING_DUMP_ROWS_LIMIT)
classes_sorted_by_count = sorted(classes_as_list, key=lambda d: d['count'], reverse=True)
ret_value += 'Count\t\tSize\t\tName\n'
for current_class in classes_sorted_by_count[:PROFILING_DUMP_ROWS_LIMIT]:
ret_value += '{}\t\t{}\t\t{}\n'.format(current_class["count"], current_class["size"], current_class["name"])
ret_value += '\n--- End Top {} Classes by Count ---\n'.format(PROFILING_DUMP_ROWS_LIMIT)
ret_value += '\n--- Start Top {} Classes by Size ---\n'.format(PROFILING_DUMP_ROWS_LIMIT)
classes_sorted_by_size = sorted(classes_as_list, key=lambda d: d['size'], reverse=True)
ret_value += 'Size\t\tCount\t\tName\n'
for current_class in classes_sorted_by_size[:PROFILING_DUMP_ROWS_LIMIT]:
ret_value += '{}\t\t{}\t\t{}\n'.format(current_class["size"], current_class["count"], current_class["name"])
ret_value += '\n--- End Top {} Classes by Size ---\n'.format(PROFILING_DUMP_ROWS_LIMIT)
ret_value += '\n--- End Memory Dump ---\n\n'
return ret_value
def get_message_local_vars():
"""
A function that returns the printable message about local variables
:return: Message to print.
:rtype: ``str``
"""
local_vars = list(locals().items())
ret_value = '\n\n--- Start Local Vars ---\n\n'
for current_local_var in local_vars:
ret_value += shorten_string_for_printing(str(current_local_var)) + '\n'
ret_value += '\n--- End Local Vars ---\n\n'
return ret_value
def get_size_of_object(input_object):
"""
A function that recursively iterate to sum size of object & members.
:type input_object: ``Any``
:param input_object: The object to calculate its memory footprint
:return: Size of input_object in bytes, or -1 if cannot determine the size.
:rtype: ``int``
"""
if IS_PY3 and PY_VER_MINOR >= 10:
from collections.abc import Mapping
else:
from collections import Mapping # type: ignore[no-redef, attr-defined]
from collections import deque
from numbers import Number
# IS_PY3:
# ZERO_DEPTH_BASES = (str, bytes, Number, range, bytearray) if IS_PY3 else (str, bytes, Number, bytearray)
ZERO_DEPTH_BASES = (str, bytes, Number, bytearray)
MAX_LEVEL = 20
_seen_ids = set()
def inner(obj, level):
"""
A recursion that goes deep into objects, to calculate their deep memory footprint.
:type level: ``int``
:param level: Current level of the recursion (object)
:return: Size of obj in bytes, or -1 if cannot determine the size.
:rtype: ``int``
"""
if level == MAX_LEVEL:
# Stop at MAX_LEVEL
return sys.getsizeof(obj)
obj_id = id(obj)
if obj_id in _seen_ids:
return 0
_seen_ids.add(obj_id)
size = sys.getsizeof(obj)
if isinstance(obj, ZERO_DEPTH_BASES):
pass
elif isinstance(obj, (tuple, list, set, deque)):
size += sum(inner(i, level + 1) for i in obj)
elif isinstance(obj, Mapping):
mapping_items_keys = list(getattr(obj, 'keys')())
for current_key in mapping_items_keys:
if current_key in obj:
size += (inner(current_key, level + 1) + inner(obj[current_key], level + 1))
# Check for custom object instances - may subclass above too
if hasattr(obj, '__dict__'):
size += inner(vars(obj), level + 1)
return size
try:
return inner(input_object, 0)
except RuntimeError:
# A RuntimeError can occur, for example, in flask apps: it's forbidden to perform
# some functionality outside the flask app context, which is unreachable from the
# signal handler. Therefore, we just skip calculating the size in this case.
demisto.debug('Skipping flask app internal objects in size calculation')
return -1
excluded_globals = ['__name__', '__doc__', '__package__', '__loader__',
'__spec__', '__annotations__', '__builtins__',
'__file__', '__cached__', '_Feature',
]
excluded_types_names = ['MagicMock', # When running tests locally
]
def get_message_global_vars():
"""
A function that returns the printable message about global variables
:return: Message to print.
:rtype: ``str``
"""
excluded_types = [types.ModuleType, types.FunctionType,
]
globals_dict = dict(globals())
globals_dict_full = {}
for current_key in globals_dict.keys():
current_value = globals_dict[current_key]
if not type(current_value) in excluded_types \
and current_key not in excluded_globals \
and type(current_value).__name__ not in excluded_types_names:
globals_dict_full[current_key] = {
'name': current_key,
'value': current_value,
# Deep calculation. Better than sys.getsizeof(current_value)
'size': get_size_of_object(current_value)
}
ret_value = '\n\n--- Start Top {} Globals by Size ---\n'.format(PROFILING_DUMP_ROWS_LIMIT)
globals_sorted_by_size = sorted(globals_dict_full.values(), key=lambda d: d['size'], reverse=True)
ret_value += 'Size\t\tName\t\tValue\n'
for current_global in globals_sorted_by_size[:PROFILING_DUMP_ROWS_LIMIT]:
ret_value += '{}\t\t{}\t\t{}\n'.format(current_global["size"], current_global["name"],
shorten_string_for_printing(str(current_global["value"])))
ret_value += '\n--- End Top {} Globals by Size ---\n'.format(PROFILING_DUMP_ROWS_LIMIT)
return ret_value
def get_message_modules_sizes():
"""
A function that returns the printable message about the loaded modules by size
:return: Message to print.
:rtype: ``str``
"""
globals_dict = dict(globals())
globals_dict_full = {}
for current_key in globals_dict.keys():
current_value = globals_dict[current_key]
if isinstance(current_value, types.ModuleType) \
and current_key not in excluded_globals \
and type(current_value).__name__ not in excluded_types_names:
globals_dict_full[current_key] = {
'name': current_key,
'value': current_value,
# Deep calculation. Better than sys.getsizeof(current_value)
'size': get_size_of_object(current_value)
}
ret_value = '\n\n--- Start Top {} Modules by Size ---\n'.format(PROFILING_DUMP_ROWS_LIMIT)
globals_sorted_by_size = sorted(globals_dict_full.values(), key=lambda d: d['size'], reverse=True)
ret_value += 'Size\t\tName\t\tValue\n'
for current_global in globals_sorted_by_size[:PROFILING_DUMP_ROWS_LIMIT]:
ret_value += '{}\t\t{}\t\t{}\n'.format(current_global["size"], current_global["name"],
shorten_string_for_printing(str(current_global["value"])))
ret_value += '\n--- End Top {} Modules by Size ---\n'.format(PROFILING_DUMP_ROWS_LIMIT)
return ret_value
def signal_handler_profiling_dump(_sig, _frame):
"""
Listener function to dump the threads and memory to log info
:type _sig: ``int``
:param _sig: The signal number
:type _frame: ``Any``
:param _frame: The current stack frame
:return: No data returned
:rtype: ``None``
"""
msg = '\n\n--- Start Profiling Dump ---\n'
msg += get_message_threads_dump(_sig, _frame)
msg += get_message_memory_dump(_sig, _frame)
msg += '\n--- End Profiling Dump ---\n\n'
LOG(msg)
LOG.print_log()
def register_signal_handler_profiling_dump(signal_type=None,
profiling_dump_rows_limit=PROFILING_DUMP_ROWS_LIMIT): # pragma: no cover
"""
Function that registers the threads and memory dump signal listener
:type profiling_dump_rows_limit: ``int``
:param profiling_dump_rows_limit: The max number of profiling related rows to print to the log
:type profiling_dump_rows_limit: ``int``
:param profiling_dump_rows_limit: The max number of profiling related rows to print to the log
:return: No data returned
:rtype: ``None``
"""
if OS_LINUX or OS_MAC:
import signal
globals_ = globals()
globals_['PROFILING_DUMP_ROWS_LIMIT'] = profiling_dump_rows_limit
requested_signal = signal_type if signal_type else signal.SIGUSR1
signal.signal(requested_signal, signal_handler_profiling_dump)
else:
demisto.info('Not a Linux or Mac OS, profiling using a signal is not supported.')
def shorten_string_for_printing(source_string, max_length=64):
"""
Function that removes the middle of a long str, for printint or logging.
If needed, it will replace the middle with '...',
Examples:
>>> shorten_string_for_printing('123456789', 9)
'123456789'
>>> shorten_string_for_printing('1234567890', 9)
'abc...890'
>>> shorten_string_for_printing('123456789012', 10)
'1234...012'
:type source_string: ``str``
:param source_string: A long str that needs shortening.
:type max_length: ``int``
:param max_length: Maximum length of the returned str, should be higher than 0. Default is 64.
:return:: A string no longer than max_length.
:rtype: ``str``
"""
if not source_string or max_length < 1 or len(source_string) <= max_length:
return source_string
extremeties_length = int((max_length - 3) / 2)
if max_length % 2 == 0:
# even max_length. Start with one more char than at the beginning
ret_value = source_string[:extremeties_length + 1] \
+ '...' \
+ source_string[-extremeties_length:]
return ret_value
else:
# odd max_length
ret_value = source_string[:extremeties_length] \
+ '...' \
+ source_string[-extremeties_length:]
return ret_value
class PollResult:
"""The response object for polling functions. This object contains information about whether to run again, and what
the CommandResults are in case of success, or failure.
:return: PollResult
:rtype: ``PollResult``
"""
def __init__(self, response, continue_to_poll=False, args_for_next_run=None, partial_result=None):
"""
Constructor for PollResult
:type response: ``Any``
:param response: The response of the command in the event of success,
or in case of failure but Polling is false
:type continue_to_poll: ``Union[bool, Callable]``
:param continue_to_poll: An iterable of relevant keys list from the json. Notice we save it as a set in the class
:type args_for_next_run: ``Dict``
:param args_for_next_run: The arguments to use in the next iteration. Will use the input args in case of None
:type partial_result: ``CommandResults``
:param partial_result: CommandResults to return, even though we will poll again
"""
self.response = response
self.continue_to_poll = continue_to_poll
self.args_for_next_run = args_for_next_run
self.partial_result = partial_result
def polling_function(name, interval=30, timeout=600, poll_message='Fetching Results:', polling_arg_name="polling",
requires_polling_arg=True): # pragma: no cover
"""
To use on a function that should rerun itself
Commands that use this decorator must have a Polling argument, polling: true in yaml,
and a hidden hide_polling_output argument.
Commands that use this decorator should return a PollResult.
Will raise an DemistoException if the server version doesn't support Scheduled Commands (< 6.2.0)
:type name: ``str``
:param name: The name of the command
:type interval: ``int``
:param interval: How many seconds until the next run. Recommended range: 30-60 seconds.
:type timeout: ``int``
:param timeout: How long
:type poll_message: ``str``
:param poll_message: The message to display in the war room while polling
:type requires_polling_arg: ``bool``
:param requires_polling_arg: Whether a polling argument should be expected as one of the demisto args
:return: Decorator for polling functions
:rtype: ``Function``
"""
def dec(func):
def inner(args, *arguments, **kwargs):
"""
Args:
args (dict): command arguments (demisto.args()).
*arguments: any additional arguments to the command function.
**kwargs: additional keyword arguments to the command function.
"""
if not isinstance(args, dict):
demisto.debug("Detected args of type {args_type}. Casting to dict.".format(args_type=type(args)))
args = dict(args or {})
if not requires_polling_arg or argToBoolean(args.get(polling_arg_name, False)):
ScheduledCommand.raise_error_if_not_supported()
poll_result = func(args, *arguments, **kwargs)
should_poll = poll_result.continue_to_poll if isinstance(poll_result.continue_to_poll, bool) \
else poll_result.continue_to_poll()
if not should_poll:
return poll_result.response
readable_output = poll_message if not args.get('hide_polling_output') else None
poll_args = poll_result.args_for_next_run or args
poll_args['hide_polling_output'] = True
poll_response = poll_result.partial_result or CommandResults(readable_output=readable_output)
poll_response.scheduled_command = ScheduledCommand(command=name, next_run_in_seconds=interval,
args=poll_args, timeout_in_seconds=timeout)
return poll_response
else:
return func(args, *arguments, **kwargs).response
return inner
return dec
def get_pack_version():
"""
Get the pack version.
The version can be retrieved only for the pack that contains the running script or integration.
:return: The pack version in which the integration/script is part of, in case not found returns empty string.
:rtype: ``str``
"""
global_name = "CONSTANT_PACK_VERSION"
global_vars = globals()
if global_name in global_vars:
return global_vars[global_name]
else:
return ''
def create_indicator_result_with_dbotscore_unknown(indicator, indicator_type, reliability=None,
context_prefix=None, address_type=None, relationships=None):
'''
Used for cases where the api response to an indicator is not found,
returns CommandResults with readable_output generic in this case, and indicator with DBotScore unknown
:type indicator: ``str``
:param indicator: The value of the indicator
:type indicator_type: ``DBotScoreType``
:param indicator_type: use DBotScoreType class [Unsupport in types CVE and ATTACKPATTERN]
:type reliability: ``DBotScoreReliability``
:param reliability: use DBotScoreReliability class
:type context_prefix: ``str``
:param context_prefix: Use only in case that the indicator is CustomIndicator
:type address_type: ``str``
:param address_type: Use only in case that the indicator is Cryptocurrency
:type relationships: ``list of EntityRelationship``
:param relationships: List of relationships of the indicator.
:rtype: ``CommandResults``
:return: CommandResults
'''
if not context_prefix and (indicator_type is DBotScoreType.CUSTOM or not DBotScoreType.is_valid_type(indicator_type)):
raise ValueError('Indicator type is invalid')
if indicator_type in [DBotScoreType.CVE, DBotScoreType.ATTACKPATTERN]:
# not supportted, because they have a fixed dbotscore
msg_error = 'DBotScoreType.{} is unsupported'.format(indicator_type.upper())
raise ValueError(msg_error)
dbot_score = Common.DBotScore(indicator=indicator,
indicator_type=indicator_type
if DBotScoreType.is_valid_type(indicator_type) else DBotScoreType.CUSTOM,
score=Common.DBotScore.NONE,
reliability=reliability,
message='No results found.')
integration_name = dbot_score.integration_name or 'Results'
indicator_ = None # type: Any
if indicator_type is DBotScoreType.FILE:
if sha1Regex.match(indicator):
indicator_ = Common.File(dbot_score=dbot_score, sha1=indicator)
indicator_type = 'sha1'
elif sha256Regex.match(indicator):
indicator_ = Common.File(dbot_score=dbot_score, sha256=indicator)
indicator_type = 'sha256'
elif sha512Regex.match(indicator):
indicator_ = Common.File(dbot_score=dbot_score, sha512=indicator)
indicator_type = 'sha512'
elif md5Regex.match(indicator):
indicator_ = Common.File(dbot_score=dbot_score, md5=indicator)
indicator_type = 'md5'
else:
raise ValueError('This indicator -> {} is incorrect'.format(indicator))
elif indicator_type is DBotScoreType.IP:
indicator_ = Common.IP(ip=indicator, dbot_score=dbot_score)
elif indicator_type is DBotScoreType.URL:
indicator_ = Common.URL(url=indicator, dbot_score=dbot_score)
elif indicator_type is DBotScoreType.DOMAIN:
indicator_ = Common.Domain(domain=indicator, dbot_score=dbot_score)
elif indicator_type is DBotScoreType.EMAIL:
indicator_ = Common.EMAIL(address=indicator, dbot_score=dbot_score)
elif indicator_type is DBotScoreType.CERTIFICATE:
indicator_ = Common.Certificate(subject_dn=indicator, dbot_score=dbot_score)
elif indicator_type is DBotScoreType.ACCOUNT:
indicator_ = Common.Account(id=indicator, dbot_score=dbot_score)
elif indicator_type is DBotScoreType.CRYPTOCURRENCY:
if not address_type:
raise ValueError('Missing address_type parameter')
indicator_ = Common.Cryptocurrency(address=indicator, address_type=address_type, dbot_score=dbot_score)
indicator_type = address_type
else:
indicator_ = Common.CustomIndicator(indicator_type=indicator_type,
value=indicator,
dbot_score=dbot_score,
data={},
context_prefix=context_prefix,
relationships=relationships,
)
indicator_type = indicator_type.upper()
readable_output = tableToMarkdown(name='{}:'.format(integration_name),
t={indicator_type: indicator, 'Result': 'Not found'},
headers=[indicator_type, 'Result'])
return CommandResults(readable_output=readable_output, indicator=indicator_)
def get_fetch_run_time_range(last_run, first_fetch, look_back=0, timezone=0, date_format='%Y-%m-%dT%H:%M:%S'):
"""
Calculates the time range for fetch depending the look_back argument and the previous fetch start time
given from the last_run object.
:type last_run: ``dict``
:param last_run: The LastRun object
:type first_fetch: ``str``
:param first_fetch: The first time to fetch, used in the first fetch of an instance
:type look_back: ``int``
:param look_back: The time to look back in fetch in minutes
:type timezone: ``int``
:param timezone: The time zone offset in hours
:type date_format: ``str``
:param date_format: The date format
:return: The time range (start_time, end_time) of the creation date for the incidents to fetch in the current run.
:rtype: ``Tuple``
"""
last_run_time = last_run and 'time' in last_run and last_run['time']
now = get_current_time(timezone)
if not last_run_time:
last_run_time = dateparser.parse(first_fetch, settings={'TIMEZONE': 'UTC', 'RETURN_AS_TIMEZONE_AWARE': True})
if last_run_time:
last_run_time += timedelta(hours=timezone)
else:
last_run_time = dateparser.parse(last_run_time, settings={'TIMEZONE': 'UTC', 'RETURN_AS_TIMEZONE_AWARE': True})
if look_back and look_back > 0:
if now - last_run_time < timedelta(minutes=look_back):
last_run_time = now - timedelta(minutes=look_back)
demisto.debug("lb: fetch start time: {}, fetch end time: {}".format(
last_run_time.strftime(date_format), now.strftime(date_format)))
return last_run_time.strftime(date_format), now.strftime(date_format)
def get_current_time(time_zone=0):
"""
Gets the current time in a given timezone, as time awared datetime.
:type time_zone: ``int``
:param time_zone: The time zone offset in hours.
:return: The current time.
:rtype: ``datetime``
"""
now = datetime.utcnow() + timedelta(hours=time_zone)
try:
import pytz
return now.replace(tzinfo=pytz.UTC)
except ImportError:
demisto.debug('pytz is missing, will not return timeaware object.')
return now
def calculate_new_offset(old_offset, num_incidents, total_incidents):
""" This calculates the new offset based on the response
:type old_offset: ``int``
:param old_offset: The offset from the previous run
:type num_incidents: ``int``
:param num_incidents: The number of incidents returned by the API.
:type total_incidents: ``int``
:param total_incidents: The total number of incidents returned by the API.
:return: The new offset for the next run.
:rtype: ``int``
"""
if not num_incidents:
return 0
if total_incidents and num_incidents + old_offset >= total_incidents:
return 0
return old_offset + num_incidents
def filter_incidents_by_duplicates_and_limit(incidents_res, last_run, fetch_limit, id_field):
"""
Removes duplicate incidents from response and returns the incidents till limit.
The function should be called after getting the get-incidents API response,
and by passing the id_field it will filter out the incidents that were already fetched
by checking the incident IDs that are saved from the previous fetch in the last run object
:type incidents_res: ``list``
:param incidents_res: The incidents from the API response
:type last_run: ``dict``
:param last_run: The LastRun object
:type fetch_limit: ``int``
:param fetch_limit: The incidents limit to return
:type id_field: ``str``
:param id_field: The incident id field
:return: List of incidents after filtering duplicates when len(incidents) <= limit
:rtype: ``list``
"""
demisto.debug('lb: Filtering incidents by duplicates and limit')
found_incidents = last_run.get('found_incident_ids', {})
incidents = []
demisto.debug('lb: Number of incidents before filtering: {}, their ids: {}'.format(len(incidents_res),
[incident_res[id_field] for incident_res in
incidents_res]))
for incident in incidents_res:
if incident[id_field] not in found_incidents:
incidents.append(incident)
demisto.debug('lb: Number of incidents after filtering: {}, their ids: {}'.format(len(incidents),
[incident[id_field] for incident in
incidents]))
return incidents[:fetch_limit]
def get_latest_incident_created_time(incidents, created_time_field, date_format='%Y-%m-%dT%H:%M:%S',
increase_last_run_time=False):
"""
Gets the latest incident created time
:type incidents: ``list``
:param incidents: List of incidents
:type created_time_field: ``str``
:param created_time_field: The incident created time field
:type date_format: ``str``
:param date_format: The date format
:type increase_last_run_time: ``bool``
:param increase_last_run_time: Whether to increase the last run time with one millisecond
:return: The latest incident time
:rtype: ``str``
"""
demisto.debug('lb: Getting latest incident created time')
latest_incident_time = safe_strptime(incidents[0][created_time_field], date_format)
for incident in incidents:
incident_time = safe_strptime(incident[created_time_field], date_format)
if incident_time > latest_incident_time:
latest_incident_time = incident_time
if increase_last_run_time:
latest_incident_time = latest_incident_time + timedelta(milliseconds=1)
demisto.debug("lb: latest_incident_time is {}".format(latest_incident_time))
return latest_incident_time.strftime(date_format)
def remove_old_incidents_ids(found_incidents_ids, current_time, look_back):
"""
Removes old incident ids from the last run object to avoid overloading.
:type found_incidents_ids: ``dict``
:param found_incidents_ids: Dict of incidents ids
:type current_time: ``int``
:param current_time: The current epoch time to compare with the existing IDs added time
:type look_back: ``int``
:param look_back: The look back time in minutes
:return: The new incidents ids
:rtype: ``dict``
"""
demisto.debug('lb: Remove old incidents ids, current time is {}'.format(current_time))
look_back_in_seconds = look_back * 60
deletion_threshold_in_seconds = look_back_in_seconds * 2
new_found_incidents_ids = {}
latest_incident_time = max(found_incidents_ids.values() or [current_time])
demisto.debug('lb: latest_incident_time is {}'.format(latest_incident_time))
for inc_id, addition_time in found_incidents_ids.items():
if (
current_time - addition_time <= deletion_threshold_in_seconds
or addition_time == latest_incident_time # The latest IDs must be kept to avoid duplicate incidents
):
new_found_incidents_ids[inc_id] = addition_time
demisto.debug('lb: Adding incident id: {}, its addition time: {}, deletion_threshold_in_seconds: {}'.format(
inc_id, addition_time, deletion_threshold_in_seconds))
else:
demisto.debug('lb: Removing incident id: {}, its addition time: {}, deletion_threshold_in_seconds: {}'.format(
inc_id, addition_time, deletion_threshold_in_seconds))
demisto.debug('lb: Number of new found ids: {}, their ids: {}'.format(
len(new_found_incidents_ids), new_found_incidents_ids.keys()))
return new_found_incidents_ids
def get_found_incident_ids(last_run, incidents, look_back, id_field, remove_incident_ids):
"""
Gets the found incident ids from the last run object and adds the new fetched incident IDs.
:type last_run: ``dict``
:param last_run: The LastRun object
:type incidents: ``list``
:param incidents: List of incidents to add
:type look_back: ``int``
:param look_back: The look back time in minutes
:type id_field: ``str``
:param id_field: The incident id field
:return: The new incident ids.
:rtype: ``dict``
"""
demisto.debug('lb: Get found incident ids')
found_incidents = last_run.get('found_incident_ids', {})
current_time = int(time.time())
for incident in incidents:
found_incidents[incident[id_field]] = current_time
if remove_incident_ids:
found_incidents = remove_old_incidents_ids(found_incidents, current_time, look_back)
return found_incidents
def create_updated_last_run_object(last_run, incidents, fetch_limit, look_back, start_fetch_time, end_fetch_time,
created_time_field, date_format='%Y-%m-%dT%H:%M:%S', increase_last_run_time=False,
new_offset=None):
"""
Calculates the next fetch time and limit depending the incidents result and creates an updated LastRun object
with the new time and limit.
:type last_run: ``dict``
:param last_run: The LastRun object
:type incidents: ``list``
:param incidents: List of the incidents result
:type fetch_limit: ``int``
:param fetch_limit: The fetch limit
:type look_back: ``int``
:param look_back: The time to look back in fetch in minutes
:type start_fetch_time: ``str``
:param start_fetch_time: The time the fetch started to fetch from
:type end_fetch_time: ``str``
:param end_fetch_time: The end time in which the fetch incidents ended
:type created_time_field: ``str``
:param created_time_field: The incident created time field
:type date_format: ``str``
:param date_format: The date format
:type increase_last_run_time: ``bool``
:param increase_last_run_time: Whether to increase the last run time with one millisecond
:type new_offset: ``int | None``
:param new_offset: The new offset to set in the last run
:return: The new LastRun object
:rtype: ``Dict``
"""
demisto.debug("lb: Create updated last run object, len(incidents) is {},"
"look_back is {}, fetch_limit is {}, new_offset is {}".format(len(incidents), look_back, fetch_limit,
new_offset))
remove_incident_ids = True
new_limit = len(last_run.get('found_incident_ids', [])) + len(incidents) + fetch_limit
if new_offset:
# if we need to update the offset, we need to keep the old time and just update the offset
new_last_run = {
'time': last_run.get("time"),
}
elif len(incidents) == 0:
new_last_run = {
'time': end_fetch_time,
'limit': new_limit if look_back > 0 else fetch_limit,
}
else:
latest_incident_fetched_time = get_latest_incident_created_time(incidents, created_time_field, date_format,
increase_last_run_time)
new_last_run = {
'time': latest_incident_fetched_time,
'limit': new_limit if look_back > 0 else fetch_limit
}
if latest_incident_fetched_time == start_fetch_time:
# we are still on the same time, no need to remove old incident ids
remove_incident_ids = False
if new_offset is not None:
new_last_run['offset'] = new_offset
new_last_run['limit'] = fetch_limit
demisto.debug("lb: The new_last_run is: {}, the remove_incident_ids is: {}".format(new_last_run,
remove_incident_ids))
return new_last_run, remove_incident_ids
def update_last_run_object(last_run, incidents, fetch_limit, start_fetch_time, end_fetch_time, look_back,
created_time_field, id_field, date_format='%Y-%m-%dT%H:%M:%S', increase_last_run_time=False,
new_offset=None):
"""
Updates the LastRun object with the next fetch time and limit and with the new fetched incident IDs.
:type last_run: ``dict``
:param last_run: The LastRun object
:type incidents: ``list``
:param incidents: List of the incidents result
:type fetch_limit: ``int``
:param fetch_limit: The fetch limit
:type start_fetch_time: ``str``
:param start_fetch_time: The time the fetch started to fetch from
:type end_fetch_time: ``str``
:param end_fetch_time: The end time in which the fetch incidents ended
:type look_back: ``int``
:param look_back: The time to look back in fetch in minutes
:type created_time_field: ``str``
:param created_time_field: The incident created time field
:type id_field: ``str``
:param id_field: The incident id field
:type date_format: ``str``
:param date_format: The date format
:type increase_last_run_time: ``bool``
:param increase_last_run_time: Whether to increase the last run time with one millisecond
:type new_offset: ``int | None``
:param new_offset: The new offset to set in the last run
:return: The updated LastRun object
:rtype: ``Dict``
"""
if not look_back:
look_back = 0
updated_last_run, remove_incident_ids = create_updated_last_run_object(last_run,
incidents,
fetch_limit,
look_back,
start_fetch_time,
end_fetch_time,
created_time_field,
date_format,
increase_last_run_time,
new_offset
)
found_incidents = get_found_incident_ids(last_run, incidents, look_back, id_field, remove_incident_ids)
updated_last_run['found_incident_ids'] = found_incidents
last_run.update(updated_last_run)
return last_run
# YML metadata collector mocked classes.
class ParameterTypes:
"""YML ConfKey key_type type.
This is an empty class, used for code autocompletion when using
demisto-sdk generate_yml_from_python command syntax. For more information,
visit the command's README.md.
:return: The ParameterTypes enum
:rtype: ``ParameterTypes``
"""
STRING = 0
NUMBER = 1
ENCRYPTED = 4
BOOLEAN = 8
AUTH = 9
DOWNLOAD_LINK = 11
TEXT_AREA = 12
INCIDENT_TYPE = 13
TEXT_AREA_ENCRYPTED = 14
SINGLE_SELECT = 15
MULTI_SELECT = 16
class OutputArgument:
"""YML output argument.
This is an empty class, used for code autocompletion when using
demisto-sdk generate_yml_from_python command syntax. For more information,
visit the command's README.md.
:return: The OutputArgument object
:rtype: ``OutputArgument``
"""
def __init__(self,
name,
output_type=dict,
description=None,
prefix=None):
pass
class InputArgument:
"""YML input argument for a command.
This is an empty class, used for code autocompletion when using
demisto-sdk generate_yml_from_python command syntax. For more information,
visit the command's README.md.
:return: The InputArgument object
:rtype: ``InputArgument``
"""
def __init__(self,
name=None,
description=None,
required=False,
default=None,
is_array=False,
secret=False,
execution=False,
options=None,
input_type=None):
pass
class ConfKey:
"""YML configuration key fields.
This is an empty class, used for code autocompletion when using
demisto-sdk generate_yml_from_python command syntax. For more information,
visit the command's README.md.
:return: The ConfKey object
:rtype: ``ConfKey``
"""
def __init__(self,
name,
display=None,
default_value=None,
key_type=0, # Expects ParameterType
required=False,
additional_info=None,
options=None,
input_type=None):
pass
class YMLMetadataCollector:
"""The YMLMetadataCollector class provides decorators for integration
functions which contain details relevant to yml generation.
This is an empty class, used for code autocompletion when using
demisto-sdk generate_yml_from_python command syntax. For more information,
visit the command's README.md.
:return: The YMLMetadataCollector object
:rtype: ``YMLMetadataCollector``
"""
def __init__(self, integration_name, docker_image="demisto/python3:latest",
description=None, category="Utilities", conf=None,
is_feed=False, is_fetch=False, is_runonce=False,
detailed_description=None, image=None, display=None,
tests=["No tests"], fromversion="6.0.0",
long_running=False, long_running_port=False, integration_type="python",
integration_subtype="python3", deprecated=None, systemd=None,
timeout=None, default_classifier=None,
default_mapper_in=None, integration_name_x2=None, default_enabled_x2=None,
default_enabled=None, verbose=False):
pass
def command(self, command_name, outputs_prefix=None,
outputs_list=None, inputs_list=None,
execution=None, file_output=False,
multiple_output_prefixes=False, deprecated=False, restore=False,
description=None):
def command_wrapper(func):
def get_out_info(*args, **kwargs):
if restore:
kwargs['command_name'] = command_name
kwargs['outputs_prefix'] = outputs_prefix
kwargs['execution'] = execution
return func(*args, **kwargs)
return get_out_info
return command_wrapper
def xsiam_api_call_with_retries(
client,
xsiam_url,
zipped_data,
headers,
num_of_attempts,
events_error_handler=None,
error_msg='',
is_json_response=False,
data_type=EVENTS
): # pragma: no cover
"""
Send the fetched events or assests into the XDR data-collector private api.
:type client: ``BaseClient``
:param client: base client containing the XSIAM url.
:type xsiam_url: ``str``
:param xsiam_url: The URL of XSIAM to send the api request.
:type zipped_data: ``bytes``
:param zipped_data: encoded events
:type headers: ``dict``
:param headers: headers for the request
:type error_msg: ``str``
:param error_msg: The error message prefix in case of an error.
:type num_of_attempts: ``int``
:param num_of_attempts: The num of attempts to do in case there is an api limit (429 error codes).
:type events_error_handler: ``callable``
:param events_error_handler: error handler function
:type data_type: ``str``
:param data_type: events or assets
:return: Response object or DemistoException
:rtype: ``requests.Response`` or ``DemistoException``
"""
# retry mechanism in case there is a rate limit (429) from xsiam.
status_code = None
attempt_num = 1
response = None
while status_code != 200 and attempt_num < num_of_attempts + 1:
demisto.debug('Sending {data_type} into xsiam, attempt number {attempt_num}'.format(
data_type=data_type, attempt_num=attempt_num))
# in the last try we should raise an exception if any error occurred, including 429
ok_codes = (200, 429) if attempt_num < num_of_attempts else None
response = client._http_request(
method='POST',
full_url=urljoin(xsiam_url, '/logs/v1/xsiam'),
data=zipped_data,
headers=headers,
error_handler=events_error_handler,
ok_codes=ok_codes,
resp_type='response'
)
status_code = response.status_code
demisto.debug('received status code: {status_code}'.format(status_code=status_code))
if status_code == 429:
time.sleep(1)
attempt_num += 1
if is_json_response and response:
response = response.json()
if response.get('error', '').lower() != 'false':
raise DemistoException(error_msg + response.get('error'))
return response
def split_data_to_chunks(data, target_chunk_size):
"""
Splits a string of data into chunks of an approximately specified size.
The actual size can be lower.
:type data: ``list`` or a ``string``
:param data: A list of data or a string delimited with \n to split to chunks.
:type target_chunk_size: ``int``
:param target_chunk_size: The maximum size of each chunk. The maximal size allowed is 9MB.
:return: An iterable of lists where each list contains events with approx size of chunk size.
:rtype: ``collections.Iterable[list]``
"""
target_chunk_size = min(target_chunk_size, XSIAM_EVENT_CHUNK_SIZE_LIMIT)
chunk = [] # type: ignore[var-annotated]
chunk_size = 0
if isinstance(data, str):
data = data.split('\n')
for data_part in data:
if chunk_size >= target_chunk_size:
demisto.debug("reached max chunk size, sending chunk with size: {size}".format(size=chunk_size))
yield chunk
chunk = []
chunk_size = 0
data_part_size = sys.getsizeof(data_part)
if data_part_size >= MAX_ALLOWED_ENTRY_SIZE:
demisto.error(
"entry size {} is larger than the maximum allowed entry size {}, skipping this entry".format(data_part_size,
MAX_ALLOWED_ENTRY_SIZE))
continue
chunk.append(data_part)
chunk_size += data_part_size
if chunk_size != 0:
demisto.debug("sending the remaining chunk with size: {size}".format(size=chunk_size))
yield chunk
def send_events_to_xsiam(events, vendor, product, data_format=None, url_key='url', num_of_attempts=3,
chunk_size=XSIAM_EVENT_CHUNK_SIZE, should_update_health_module=True,
add_proxy_to_request=False, multiple_threads=False, client_class=None,
use_streaming_send=False):
"""
Send the fetched events into the XDR data-collector private api.
:type events: ``Union[str, list]``
:param events: The events to send to XSIAM server. Should be of the following:
1. List of strings or dicts where each string or dict represents an event.
2. String containing raw events separated by a new line.
:type vendor: ``str``
:param vendor: The vendor corresponding to the integration that originated the events.
:type product: ``str``
:param product: The product corresponding to the integration that originated the events.
:type data_format: ``str``
:param data_format: Should only be filled in case the 'events' parameter contains a string of raw
events in the format of 'leef' or 'cef'. In other cases the data_format will be set automatically.
:type url_key: ``str``
:param url_key: The param dict key where the integration url is located at. the default is 'url'.
:type num_of_attempts: ``int``
:param num_of_attempts: The num of attempts to do in case there is an api limit (429 error codes)
:type chunk_size: ``int``
:param chunk_size: Advanced - The maximal size of each chunk size we send to API. Limit of 9 MB will be inforced.
:type should_update_health_module: ``bool``
:param should_update_health_module: whether to trigger the health module showing how many events were sent to xsiam
:type add_proxy_to_request :``bool``
:param add_proxy_to_request: whether to add proxy to the send evnets request.
:type multiple_threads: ``bool``
:param multiple_threads: whether to use multiple threads to send the events to xsiam or not.
:type client_class: ``BaseClient``
:param client_class: The client class to use for the request.
:type use_streaming_send: ``bool``
:param use_streaming_send: Feature flag (default False). When True, serializes and gzips the data one item at a time
(streaming) instead of building full copies of the whole batch, keeping peak memory ~flat; the bytes sent to
XSIAM are equivalent to the legacy path. Ignored when multiple_threads=True or when data is already a raw string.
:return: Either None if running in a single thread or a list of future objects if running in multiple threads.
In case of running with multiple threads, the list of futures will hold the number of events sent and can be accessed by:
for future in concurrent.futures.as_completed(futures):
data_size += future.result()
:rtype: ``List[Future]`` or ``None``
"""
return send_data_to_xsiam(
events,
vendor,
product,
data_format,
url_key,
num_of_attempts,
chunk_size,
data_type="events",
should_update_health_module=should_update_health_module,
add_proxy_to_request=add_proxy_to_request,
multiple_threads=multiple_threads,
client_class=client_class if client_class else BaseClient,
use_streaming_send=use_streaming_send,
)
def is_scheduled_command_retry():
"""
Determines if the current command is a polling retry command. This is useful if some actions should not be performed
when a command is polling for a response such as submitting data for processing.
:returns: True if the command is part of a polling retry, otherwise false
:rtype: ``Bool``
"""
calling_context = demisto.callingContext.get('context', {})
sm = get_schedule_metadata(context=calling_context)
return True if sm.get('is_polling', False) else False
def retry(
times=3,
delay=1,
exceptions=Exception,
):
"""
retries to execute a function until an exception isn't raised anymore.
:type times: ``int``
:param times: The number of times to trigger the retry mechanism.
:type delay: ``int``
:param delay: The time in seconds to sleep between each time
:type exceptions: ``Exception``
:param exceptions: The exceptions that should be caught when executing
the function (Union[tuple[type[Exception], ...], type[Exception]])
:return: Any
:rtype: ``Any``
"""
def _retry(func):
func_name = func.__name__
@wraps(func)
def wrapper(*args, **kwargs):
for i in range(1, times + 1):
demisto.debug("Running func {func_name} for the {time} time".format(func_name=func_name, time=i))
try:
return func(*args, **kwargs)
except exceptions as error:
demisto.debug(
"Error when executing func {func_name}, error: {error}, time {time}".format(
func_name=func_name, error=error, time=i
)
)
if i == times:
raise
time.sleep(delay) # pylint: disable=sleep-exists
return wrapper
return _retry
def replace_spaces_in_credential(credential):
"""
This function is used in case of credential from type: 9 is in the wrong format
of one line with spaces instead of multiple lines.
:type credential: ``str`` or ``None``
:param credential: the credential to replace spaces in.
:return: the credential with spaces replaced with new lines if the credential is in the correct format,
otherwise the credential will be returned as is.
:rtype: ``str`` or ``None``
"""
if not credential:
return credential
match_begin = re.search("-----BEGIN(.*?)-----", credential)
match_end = re.search("-----END(.*?)-----", credential)
if match_begin and match_end:
return re.sub("(?<={0})(.*?)(?={1})".format(match_begin.group(0), match_end.group(0)),
lambda match: match.group(0).replace(' ', '\n'), credential)
return credential
def has_passed_time_threshold(timestamp_str, seconds_threshold):
"""
Checks if the time difference between the current time and the timestamp is greater than the threshold.
:type timestamp_str: ``str``
:param timestamp_str: The timestamp to compare the current time to.
:type seconds_threshold: ``int``
:param seconds_threshold: The threshold in seconds.
:return: True if the time difference is greater than the threshold, otherwise False.
:rtype: ``bool``
"""
import pytz
to_utc_timestamp = dateparser.parse(timestamp_str, settings={'TIMEZONE': 'UTC'})
# using astimezone since utcnow() returns a naive datetime object when unitesting
current_time = datetime.now().astimezone(pytz.utc)
if to_utc_timestamp:
time_difference = current_time - to_utc_timestamp
return time_difference.total_seconds() > seconds_threshold
else:
raise ValueError("Failed to parse timestamp: {timestamp_str}".format(timestamp_str=timestamp_str))
def send_data_to_xsiam(data, vendor, product, data_format=None, url_key='url', num_of_attempts=3,
chunk_size=XSIAM_EVENT_CHUNK_SIZE, data_type=EVENTS, should_update_health_module=True,
add_proxy_to_request=False, snapshot_id='', items_count=None, multiple_threads=False,
client_class=None, use_streaming_send=False):
"""
Send the supported fetched data types into the XDR data-collector private api.
:type data: ``Union[str, list]``
:param data: The data to send to XSIAM server. Should be of the following:
1. List of strings or dicts where each string or dict represents an event or asset.
2. String containing raw events separated by a new line.
:type vendor: ``str``
:param vendor: The vendor corresponding to the integration that originated the data.
:type product: ``str``
:param product: The product corresponding to the integration that originated the data.
:type data_format: ``str``
:param data_format: Should only be filled in case the 'events' parameter contains a string of raw
events in the format of 'leef' or 'cef'. In other cases the data_format will be set automatically.
:type url_key: ``str``
:param url_key: The param dict key where the integration url is located at. the default is 'url'.
:type num_of_attempts: ``int``
:param num_of_attempts: The num of attempts to do in case there is an api limit (429 error codes)
:type chunk_size: ``int``
:param chunk_size: Advanced - The maximal size of each chunk size we send to API. Limit of 9 MB will be inforced.
:type data_type: ``str``
:param data_type: Type of data to send to Xsiam, events or assets.
:type should_update_health_module: ``bool``
:param should_update_health_module: whether to trigger the health module showing how many events were sent to xsiam
This can be useful when using send_data_to_xsiam in batches for the same fetch.
:type add_proxy_to_request: ``bool``
:param add_proxy_to_request: whether to add proxy to the send evnets request.
:type snapshot_id: ``str``
:param snapshot_id: the snapshot id.
:type items_count: ``str``
:param items_count: the asset snapshot items count.
:type multiple_threads: ``bool``
:param multiple_threads: whether to use multiple threads to send the events to xsiam or not.
Note that when set to True, the updateModuleHealth should be done from the itnegration itself.
:type client_class: ``BaseClient``
:param client_class: The client class to use for the request.
:type use_streaming_send: ``bool``
:param use_streaming_send: Feature flag (default False). When True, serializes and gzips the data one item at a time
(streaming) instead of building full copies of the whole batch, keeping peak memory ~flat; the bytes sent to
XSIAM are equivalent to the legacy path. Ignored when multiple_threads=True or when data is already a raw string.
:return: Either None if running in a single thread or a list of future objects if running in multiple threads.
In case of running with multiple threads, the list of futures will hold the number of events sent and can be accessed by:
for future in concurrent.futures.as_completed(futures):
data_size += future.result()
:rtype: ``List[Future]`` or ``None```
"""
data_size = 0
params = demisto.params()
url = params.get(url_key)
calling_context = demisto.callingContext.get('context', {})
instance_name = calling_context.get('IntegrationInstance', '')
collector_name = calling_context.get('IntegrationBrand', '')
if not items_count:
items_count = len(data) if isinstance(data, list) else 1
if data_type not in DATA_TYPES:
demisto.debug("data type must be one of these values: {types}".format(types=DATA_TYPES))
return
if not data:
demisto.debug('send_data_to_xsiam function received no {data_type}, '
'skipping the API call to send {data} to XSIAM'.format(data_type=data_type, data=data_type))
demisto.updateModuleHealth({'{data_type}Pulled'.format(data_type=data_type): data_size})
return
# Feature flag (CIAC-16981): stream-serialize one item at a time (list-of-items, single-thread path only).
streaming_send = bool(use_streaming_send) and isinstance(data, list) and not multiple_threads
# Decide JSON-encoding once on the first item, like the legacy list path, so the payload is identical.
streaming_items_are_json = streaming_send and bool(data) and isinstance(data[0], dict)
if streaming_items_are_json:
data_format = 'json'
# only in case we have data to send to XSIAM we continue with this flow.
# Correspond to case 1: List of strings or dicts where each string or dict represents an one event or asset or snapshot.
if streaming_send:
demisto.debug("Sending {size} {data_type} to XSIAM (streaming send)".format(size=len(data), data_type=data_type))
elif isinstance(data, list):
# In case we have list of dicts we set the data_format to json and parse each dict to a stringify each dict.
demisto.debug("Sending {size} {data_type} to XSIAM".format(size=len(data), data_type=data_type))
if isinstance(data[0], dict):
data = [json.dumps(item) for item in data]
data_format = 'json'
# Separating each event with a new line
data = '\n'.join(data)
elif not isinstance(data, str):
raise DemistoException('Unsupported type: {data} for the {data_type} parameter.'
' Should be a string or list.'.format(data=type(data), data_type=data_type))
if not data_format:
data_format = 'text'
xsiam_api_token = demisto.getLicenseCustomField('Http_Connector.token')
xsiam_domain = demisto.getLicenseCustomField('Http_Connector.url')
xsiam_url = 'https://api-{xsiam_domain}'.format(xsiam_domain=xsiam_domain)
headers = {
'authorization': xsiam_api_token,
'format': data_format,
'product': product,
'vendor': vendor,
'content-encoding': 'gzip',
'collector-name': collector_name,
'instance-name': instance_name,
'final-reporting-device': url,
'collector-type': ASSETS if data_type == ASSETS else EVENTS
}
if data_type == ASSETS:
if not snapshot_id:
snapshot_id = str(round(time.time() * 1000))
# We are setting a time stamp ahead of the instance name since snapshot-ids must be configured in ascending
# alphabetical order such that first_snapshot < second_snapshot etc.
headers['snapshot-id'] = snapshot_id + instance_name
headers['total-items-count'] = str(items_count)
header_msg = 'Error sending new {data_type} into XSIAM.\n'.format(data_type=data_type)
def data_error_handler(res):
"""
Internal function to parse the XSIAM API errors
"""
try:
response = res.json()
error = res.reason
if response.get('error').lower() == 'false':
xsiam_server_err_msg = response.get('error')
error += ": " + xsiam_server_err_msg
except ValueError:
if res.text:
error = '\n{}'.format(res.text)
else:
error = "Received empty response from the server"
api_call_info = (
'Parameters used:\n'
'\tURL: {xsiam_url}\n'
'\tHeaders: {headers}\n\n'
'Response status code: {status_code}\n'
'Error received:\n\t{error}'
).format(xsiam_url=xsiam_url, headers=json.dumps(headers, indent=8), status_code=res.status_code, error=error)
demisto.error(header_msg + api_call_info)
raise DemistoException(header_msg + error, DemistoException)
if client_class is None:
client_class = BaseClient
client = client_class(base_url=xsiam_url, proxy=add_proxy_to_request)
if streaming_send:
# Streaming path: serialize+gzip one event at a time, freeing each as we go, so peak
# memory stays ~flat. At the target chunk size we close the stream, POST it, and open a fresh one.
target_chunk_size = min(chunk_size, XSIAM_EVENT_CHUNK_SIZE_LIMIT)
demisto.info("Sending events to xsiam with a single thread (streaming, free-as-you-go).")
def _post_zipped(zipped_data):
xsiam_api_call_with_retries(client=client, events_error_handler=data_error_handler,
error_msg=header_msg, headers=headers,
num_of_attempts=num_of_attempts, xsiam_url=xsiam_url,
zipped_data=zipped_data, is_json_response=True, data_type=data_type)
buf = _io.BytesIO()
gz = gzip.GzipFile(fileobj=buf, mode='wb')
chunk_uncompressed = 0 # uncompressed bytes written into the current gzip stream
chunk_items = 0 # items written into the current gzip stream
for index in range(len(data)):
serialized = json.dumps(data[index]) if streaming_items_are_json else data[index]
data[index] = None # free the source item as soon as it is serialized (keeps peak ~one event)
# Match legacy split_data_to_chunks: skip and log any single entry larger than the allowed size,
# measuring with sys.getsizeof on the serialized string exactly as the legacy path does.
entry_size = sys.getsizeof(serialized)
if entry_size >= MAX_ALLOWED_ENTRY_SIZE:
demisto.error("entry size {size} is larger than the maximum allowed entry size {max_size}, "
"skipping this entry".format(size=entry_size, max_size=MAX_ALLOWED_ENTRY_SIZE))
continue
line = serialized.encode('utf-8')
gz.write((b'\n' if chunk_items else b'') + line) # newline-separate items, like legacy '\n'.join(...)
chunk_uncompressed += len(line) + (1 if chunk_items else 0)
chunk_items += 1
if chunk_uncompressed >= target_chunk_size:
gz.close()
_post_zipped(buf.getvalue())
data_size += chunk_items
buf = _io.BytesIO()
gz = gzip.GzipFile(fileobj=buf, mode='wb')
chunk_uncompressed = 0
chunk_items = 0
# flush the final (partial) chunk
gz.close()
if chunk_items:
_post_zipped(buf.getvalue())
data_size += chunk_items
if should_update_health_module:
demisto.updateModuleHealth({'{data_type}Pulled'.format(data_type=data_type): data_size})
return
data_chunks = split_data_to_chunks(data, chunk_size)
def send_events(data_chunk):
chunk_size = len(data_chunk)
data_chunk = '\n'.join(data_chunk)
zipped_data = gzip.compress(data_chunk.encode('utf-8')) # type: ignore[AttributeError,attr-defined]
xsiam_api_call_with_retries(client=client, events_error_handler=data_error_handler,
error_msg=header_msg, headers=headers,
num_of_attempts=num_of_attempts, xsiam_url=xsiam_url,
zipped_data=zipped_data, is_json_response=True, data_type=data_type)
return chunk_size
if multiple_threads:
demisto.info("Sending events to xsiam with multiple threads.")
all_chunks = [chunk for chunk in data_chunks]
demisto.info("Finished appending all data_chunks to a list.")
support_multithreading()
futures = []
executor = concurrent.futures.ThreadPoolExecutor(max_workers=NUM_OF_WORKERS)
for chunk in all_chunks:
future = executor.submit(send_events, chunk)
futures.append(future)
demisto.info('Finished submiting {} Futures.'.format(len(futures)))
return futures
else:
demisto.info("Sending events to xsiam with a single thread.")
for chunk in data_chunks:
data_size += send_events(chunk)
if should_update_health_module:
demisto.updateModuleHealth({'{data_type}Pulled'.format(data_type=data_type): data_size})
return
def comma_separated_mapping_to_dict(raw_text):
"""
Transforming a textual comma-separated mapping into a dictionary object.
:type raw_text: ``str``
:param raw_text: Comma-separated mapping e.g ('key1=value1', 'key2=value2', ...)
:rtype: ``dict``
:return: Validated dictionary of the raw mapping e.g {'key1': 'value1', 'key2': 'value2', ...}
"""
demisto.debug("comma_separated_mapping_to_dict "
">> Resolving comma-separated input mapping: {raw_text}".format(raw_text=raw_text))
mapping_dict = {} # type: Dict[str, str]
# If a proper mapping was not provided, return an empty dict.
if not raw_text:
return mapping_dict
key_value_pairs = raw_text.split(',')
for pair in key_value_pairs:
# Trimming trailing whitespace
pair = pair.strip()
try:
key, value = pair.split('=')
except ValueError:
demisto.error("Error: Invalid mapping was provided. "
"Expected comma-separated mapping of format `key1=value1, key2=value2, ...`")
key = value = ''
if key in mapping_dict:
demisto.debug(
"comma_separated_mapping_to_dict "
"Warning: duplicate key provided for {key}: using latter value: {value}".format(key=key, value=value)
)
mapping_dict[key] = value
demisto.debug("comma_separated_mapping_to_dict << Resolved mapping: {mapping_dict}".format(mapping_dict=mapping_dict))
return mapping_dict
def safe_sleep(duration_seconds):
"""
Sleeps for the given duration, but raises an error if it would exceed the TTL.
:type duration_seconds: ``float``
:param duration_seconds: The desired sleep duration in seconds.
:return: None
:rtype: ``None``
"""
context = demisto.callingContext.get('context', {})
if 'runDuration' in context:
run_duration = int(context.get('runDuration')) * 60
time_left = run_duration - (datetime.now() - SAFE_SLEEP_START_TIME).total_seconds()
if duration_seconds > time_left:
raise ValueError("Requested a sleep of {} seconds, but time left until docker timeout is {} seconds."
.format(duration_seconds, run_duration))
else:
demisto.info('Safe sleep is not supported in this server version, sleeping for the requested time.')
time.sleep(duration_seconds) # pylint: disable=E9003
def is_time_sensitive():
"""
Checks if the command reputation (auto-enrichment) is called as auto-extract=inline.
This function checks if the 'isTimeSensitive' attribute exists in the 'demisto' object and if it's set to True.
:return: bool
:rtype: ``bool``
"""
return hasattr(demisto, 'isTimeSensitive') and demisto.isTimeSensitive()
def parse_json_string(json_string):
"""
Parse a JSON string into a Python dictionary.
:type json_string: ``str``
:param json_string: The JSON string to be parsed.
:rtype: ``dict``
:return: A Python dictionary representing the parsed JSON data.
"""
try:
data = json.loads(json_string)
return data
except json.JSONDecodeError as error: # type: ignore[attr-defined]
demisto.error("Error decoding JSON: {error}".format(error=error))
return {}
def get_server_config():
"""
Retrieves XSOAR server configuration.
:rtype: ``dict``
:return: The XSOAR server configuration.
"""
response = demisto.internalHttpRequest(method='GET', uri='/system/config')
body = parse_json_string(response.get('body'))
server_config = body.get('sysConf', {})
return server_config
def content_profiler(func):
"""
A decorator for profiling the execution time and performance of a function.
This decorator is useful for identifying performance bottlenecks and understanding the time complexity of your code.
It collects and displays detailed profiling information, including the total execution time, the number of calls,
and the average time per call.
When to use it:
- When you need to debug and optimize the performance of your functions or methods.
- When you want to identify slow or inefficient parts of your code.
- During the development and testing phases to ensure that your code meets performance requirements.
To use, decorate the function that calls the function you want to profile with @content_profiler.
Example: I want to profile the function_to_profile() function:
```
@content_profiler
function_to_profile():
# some code
```
Analyze the Profiling Data with SnakeViz:
Download the <automation_name>.prof from the war room and run:
pip install snakeviz; snakveiz <automation_name>.prof
**Tested with Python 3.10**
:type func: ``function``
:param func: The function to be profiled.
:return: The profiled function.
:rtype: ``any``
"""
import cProfile
import threading
def profiler_wrapper(*args, **kwargs):
"""
A wrapper function that profiles the execution of the decorated function.
:param args: The positional arguments to be passed to the decorated function.
:param kwargs: The keyword arguments to be passed to the decorated function.
:return: The result of the decorated function.
"""
def profiler_function(signal_event, profiler):
"""
A helper function that runs the profiled function. When the profiled function is finished
a signal is received and the profiling data is dumped to a temporary file.
Otherwise, the function will stop when reaching to timeout.
:param signal_event: A signal that will be set when the profiled function is finished.
:param profiler: The Profile object to use for profiling.
"""
def dump_result():
"""
Helper function to dump the profiling results to a file and return the file.
"""
import tempfile
# Create a temporary file to store profiling data
with tempfile.NamedTemporaryFile(delete=False) as temp_file:
profiler.dump_stats(temp_file.name)
temp_file_path = temp_file.name
# Read the profiling data from the temporary file
with open(temp_file_path, 'rb') as f:
profiling_results = f.read()
# Delete the temporary file
os.remove(temp_file_path)
context = demisto.callingContext
executed_commands = context.get("context", {}).get("ExecutedCommands", {})
automation_name = context.get("command") or executed_commands[0].get("name",
"stats") if executed_commands else "stats"
# Use Demisto's fileResult to create a file from the profiling stats
demisto.results(fileResult('{}.prof'.format(automation_name), profiling_results))
# 5 minutes in nanoseconds
default_timeout = 60 * 5 * 1e9
# The system timeout configuration.
timeout_nanoseconds = demisto.callingContext["context"].get("TimeoutDuration") or default_timeout
timeout_seconds = timeout_nanoseconds / 1e9
event_set = signal_event.wait(timeout_seconds - 5)
profiler.disable()
dump_result()
demisto.debug("Profiler finished.")
if not event_set:
return_error("The profiled function '{}' failed due to a timeout.".format(func.__name__))
def function_runner(func, profiler, signal_event,
results, *args, **kwargs):
"""
A wrapper function that runs the targeted profiled function and captures the results in the results list.
:param func: The function to be profiled.
:param profiler: The Profile object to use for profiling.
:param signal_event: The event to signal when profiling is complete.
:param results: A shared object to store the results of the profiled function.
:param args: The positional arguments to be passed to the profiled function.
:param kwargs: The keyword arguments
"""
profiler.enable()
demisto.debug("Profiler started.")
try:
results["function_results"] = func(*args, **kwargs)
finally:
# Signal the profiling thread that the command has completed.
signal_event.set()
support_multithreading()
results = {}
profiler = cProfile.Profile()
signal_event = threading.Event()
function_thread = threading.Thread(target=function_runner, args=(func, profiler, signal_event, results) + args,
kwargs=kwargs)
function_thread.start()
profiler_function(signal_event, profiler)
return results.get("function_results")
return profiler_wrapper
def find_and_remove_sensitive_text(text, pattern):
r"""
Finds all appearances of sensitive information in a string using regex and adds the sensitive
information to the list of strings that should not appear in any logs.
The regex pattern can be used to search for a specific word, or a pattern such as a word after a given word.
Examples:
>>> text = "first secret is ID123 and the second secret is id321 and the token: ABC"
>>> pattern = r'(token:\s*)(\S+)' # Capturing groups: (token:\s*) and (\S+)
>>> find_and_remove_sensitive_text(text, pattern)
Sensitive text added to be masked in the logs: ABC
>>> pattern = r'\bid\w*\b' # Match words starting with "id", case insensitive
>>> find_and_remove_sensitive_text(text, pattern)
Sensitive text added to be masked in the logs: ID123 and id321
:param text: The input text containing the sensitive information.
:type text: str
:param pattern: The regex pattern to match the sensitive information.
:type pattern: str
:return: None
:rtype: ``None``
"""
sensitive_pattern = re.compile(pattern)
matches = sensitive_pattern.findall(text)
if not matches:
return
for match in matches:
# in case the regex serches for a group pattern
if isinstance(match, tuple):
sensitive_text = match[1]
else:
# in case the regex serches for a specific word
sensitive_text = match
add_sensitive_log_strs(sensitive_text)
return
def execute_polling_command(
command_name,
args,
using_brand=None,
polling_interval_arg_name="interval_in_seconds",
default_polling_interval=30,
polling_timeout_arg_name="timeout_in_seconds",
default_polling_timeout=600,
):
r"""
###########################################
DO NOT USE THIS UNLESS ABOLUTLY NECESSERY!
###########################################
This is intended only for special cases where polling is not supported.
Continuously executes a specified command and sleeps until the command indicates polling is done or a
timeout is reached.
:param command_name: The name of the initial Demisto command to execute.
:type command_name: ``str``
:param args: A dictionary of arguments to pass to the command.
:type args: ``dict``
:param using_brand: The optional ID of the integration to use.
:type using_brand: ``str``
:param polling_interval_arg_name: The name of the argument in 'args' that specifies the polling interval in seconds.
:type polling_interval_arg_name: ``str``
:param default_polling_interval: The default polling interval in seconds if not specified in 'args'.
:type default_polling_interval: ``int``
:param polling_timeout_arg_name: The name of the argument in 'args' that specifies the polling timeout in seconds.
:type polling_timeout_arg_name: ``str``
:param default_polling_timeout: The default polling timeout in seconds if not specified in 'args'.
:type default_polling_timeout: ``int``
:return: A list of CommandResults objects containing the human-readable output and context outputs from each
successful command execution. If an error occurs, a single CommandResults object with an error entry type is returned.
:rtype: ``list[CommandResults]`` or ``CommandResults``
:raises DemistoException: If no command result entry is received, or if the polling times out.
"""
polling_interval = arg_to_number(args.get(polling_interval_arg_name)) or default_polling_interval
polling_timeout = arg_to_number(args.get(polling_timeout_arg_name)) or default_polling_timeout
if using_brand:
args["using-brand"] = using_brand
command_results = []
times_ran = 0
start_time = time.time()
demisto.debug("Executing command: {command_name} with polling interval: {polling_interval} and timeout: {polling_timeout}.".format(
command_name=command_name, polling_interval=polling_interval, polling_timeout=polling_timeout))
while time.time() - start_time < polling_timeout:
execution_results = demisto.executeCommand(command_name, args)
times_ran += 1
demisto.debug("Executed {times_ran} command {command_name} with args: {args}. Got results: {execution_results}.".format(
command_name=command_name, args=args, times_ran=times_ran, execution_results=execution_results))
if not execution_results:
raise DemistoException("Got no command result entry for command: {command_name} with args: {args}.".format(
command_name=command_name, args=args))
execution_result = execution_results[0]
if is_error(execution_result):
error_message = get_error(execution_result)
demisto.debug("Failed to run command: {command_name} with args: {args}. Error: {error_message}.".format(
command_name=command_name, args=args, error_message=error_message))
return CommandResults(readable_output=error_message, entry_type=EntryType.ERROR)
context_output = execution_result.get("EntryContext")
human_readable = execution_result.get("HumanReadable", "")
if context_output or human_readable:
demisto.debug("Appending command results with context: {context_output} and human-readable: {human_readable}.".format(
context_output=context_output, human_readable=human_readable))
command_results.append(CommandResults(readable_output=human_readable, outputs=context_output))
metadata = execution_result.get("Metadata", {})
demisto.debug("Command: {command_name} has entry metadata: {metadata}.".format(
command_name=command_name, metadata=metadata))
if not metadata.get("polling"):
demisto.debug("Finished running command: {command_name} with args: {args}. Returning results to war room.".format(
command_name=command_name, args=args))
return command_results
command_name = metadata.get("pollingCommand", command_name)
args = metadata.get("pollingArgs", {})
if using_brand:
args["using-brand"] = using_brand
demisto.debug("Sleeping {polling_interval} seconds before next command execution.".format(
polling_interval=polling_interval))
time.sleep(polling_interval) # pylint: disable=E9003
raise DemistoException("Timed out waiting for command: {command_name}.".format(command_name=command_name))
class SignalTimeoutError(Exception):
"""
Custom exception raised when the execution timeout is reached.
:return: None
:rtype: ``None``
"""
pass
class ExecutionTimeout(object):
"""
Context manager to limit the execution time of a code block.
:return: None
:rtype: ``None``
"""
def __init__(self, seconds):
"""
Initializes the ExecutionTimeout context manager.
:param seconds: The maximum execution time in seconds.
:type seconds: int or float
:return: None
:rtype: ``None``
"""
self.seconds = int(seconds)
def _timeout_handler(self, signum, frame):
"""
Signal handler that raises a `SignalTimeoutError`.
:param signum: The number of the signal received.
:type signum: int
:param frame: The current stack frame.
:type frame: frame object
:return: None
:rtype: ``None``
"""
raise SignalTimeoutError
def __enter__(self):
"""
Enters the context manager by setting up the signal handler for
SIGALRM and starting the timer.
:return: None
:rtype: ``None``
"""
demisto.debug("Running with execution timeout: {seconds}.".format(seconds=self.seconds))
signal.signal(signal.SIGALRM, self._timeout_handler) # Set handler for SIGALRM
signal.alarm(self.seconds) # start countdown for SIGALRM to be raised
def __exit__(self, exc_type, exc_val, exc_tb):
"""
Exits the context manager by canceling the SIGALARM and
suppressing the `SignalTimeoutError`.
:param exc_type: The type of the exception that occurred, if any.
:param exc_val: The instance of the exception that occurred, if any.
:param exc_tb: A traceback object showing where the exception occurred, if any.
:return: True if the `SignalTimeoutError` was raised and suppressed, False otherwise.
:rtype: bool
"""
demisto.debug("Resetting timed signal")
signal.alarm(0) # Cancel SIGALRM if it's scheduled
return exc_type is SignalTimeoutError # True if a timeout is reacched, False otherwise
@classmethod
def limit_time(cls, seconds, default_return_value=None):
"""
Decorator method to limit the execution time of a function.
:param seconds: The maximum execution time in seconds.
:type seconds: int or float
:param default_return_value: The value to return if the function times out.
:return: Any
:rtype: ``any``
"""
def wrapper(func):
@wraps(func)
def wrapped(*args, **kwargs):
return_value = default_return_value
with cls(seconds):
return_value = func(*args, **kwargs)
demisto.debug("The function: '{func_name}' finished in time.".format(func_name=func.__name__))
return return_value
return wrapped
return wrapper
class ISOEncoder(json.JSONEncoder):
"""
A custom JSONEncoder that converts datetime objects to ISO 8601 strings.
:return: The ISOEncoder object
:rtype: ``ISOEncoder``
"""
def default(self, obj):
if isinstance(obj, datetime):
return obj.isoformat()
# Let the base class handle other objects
return json.JSONEncoder.default(self, obj)
###########################################
# UCP Functions #
###########################################
_UCP_AUTH_PARAMS_INJECTED = False
# Seconds before token expiry to consider the cache stale and re-fetch.
_UCP_REFRESH_THRESHOLD_SECONDS = 30
# In-process TTL cache for UCP credentials, keyed by method_unique_id.
_ucp_creds_cache = {} # type: Dict[str, Dict[str, Any]]
# Command-to-capability mapping. Default: 'automation-and-remediation'.
_UCP_DEFAULT_CAPABILITY = 'automation-and-remediation'
_UCP_COMMAND_CAPABILITIES = {
'fetch-incidents': 'fetch-issues',
'fetch-events': 'log-collection',
'fetch-credentials': 'fetch-secrets',
'fetch-indicators': 'threat-intelligence-and-enrichment',
'fetch-assets': 'fetch-assets-and-vulnerabilities',
}
# Canonical credential-envelope schema per profile type.
#
# ``api_key`` and ``plain`` profiles have FIXED envelope schemas: the secret
# always lives under a known key inside ``creds[creds["type"]]`` regardless of
# what ``interpolation_mapping`` left-hand id (``field_id``) the manifest emits.
# The common scripts therefore OWN this knowledge and resolve those values from
# the canonical location, rather than trusting the generator-emitted field_id.
#
# Each entry maps the mapping's left-hand ``field_id`` (which equals the field's
# ``metadata.auth.parameter``) to the actual key inside the flattened envelope.
# Note the api_key alias: ``auth.parameter`` is ``api_key`` (per the connection
# schema / OPA contract) but the runtime envelope stores the value under ``key``.
#
# ``passthrough`` is the free-form escape hatch and intentionally has NO entry:
# its values are looked up generically by field_id.
_UCP_CANONICAL_FIELD_KEYS = {
'api_key': {'api_key': 'key'},
'plain': {'username': 'username', 'password': 'password'},
}
# -- UCP helper: interpolate connector field values into demisto.params() --
def _place_by_path(target, path, value):
# type: (dict, str, Any) -> None
"""Place ``value`` into ``target`` at the dotted ``path``, creating
intermediate dicts as needed.
The destination string is split on ``.``; each segment except the last
becomes (or reuses) a nested dict, and the final segment receives the
value. Two paths that share a parent therefore merge into a single nested
dict -- this is how multiple connector fields fold into one structured param
(e.g. ``credentials.identifier`` + ``credentials.password`` ->
``{"credentials": {"identifier": ..., "password": ...}}``).
A single-segment path (e.g. ``"url"``) places a flat scalar.
:type target: ``dict``
:param target: The destination params dict (mutated in place).
:type path: ``str``
:param path: A non-empty dotted destination path (e.g. ``"a.b.c"``).
:type value: ``Any``
:param value: The value to set at the leaf of ``path``.
:return: ``None``
:rtype: ``None``
"""
segments = [seg for seg in path.split('.') if seg != '']
if not segments:
return
cursor = target
for segment in segments[:-1]:
existing = cursor.get(segment)
if not isinstance(existing, dict):
existing = {}
cursor[segment] = existing
cursor = existing
cursor[segments[-1]] = value
def _parse_param_map(param_map):
# type: (Any) -> list
"""Parse a UCP ``param_map`` into a list of ``(field_id, destination)`` pairs.
The canonical form is a **single comma-separated string** of
``field_id:dotted.destination`` entries, e.g.::
"username:credentials.identifier,password:credentials.password,server_url:server_url"
Empty/invalid entries are skipped.
:type param_map: ``Any``
:param param_map: The raw ``param_map`` value from the profile (a
comma-separated string).
:return: List of ``(field_id, destination)`` tuples (order preserved).
:rtype: ``list``
"""
pairs = [] # type: list
if not param_map:
return pairs
items = []
for entry in str(param_map).split(','):
entry = entry.strip()
if not entry:
continue
if ':' not in entry:
demisto.error(
"[UCP][CommonServerPython.py] _parse_param_map: malformed entry '{}' (no ':'); skipping.".format(entry)
)
continue
field_id, destination = entry.split(':', 1)
items.append((field_id, destination))
for field_id, destination in items:
field_id = (field_id or '').strip()
destination = (destination or '').strip()
if not field_id or not destination:
demisto.debug(
"[UCP][CommonServerPython.py] _parse_param_map: empty field id or destination "
"('{}' -> '{}'); skipping.".format(field_id, destination)
)
continue
pairs.append((field_id, destination))
return pairs
def _select_ucp_profiles(profiles, capability):
# type: (list, str) -> list
"""Select ALL connection profiles in scope for the current command.
Interpolation is **metadata-first and capability-scoped**: starting from the
connector metadata's ``connectionProfiles``, it keeps **every** profile whose
``capability`` matches ``capability``. More than one profile may be active at
a time, so this returns a list (not a single profile). When no profile matches
``capability``, falls back to the first profile that has an
``interpolation_mapping``.
:type profiles: ``list``
:param profiles: The ``connectionProfiles`` from the connector metadata.
:type capability: ``str``
:param capability: The resolved capability for the current command.
:return: List of matching profile dicts (possibly empty).
:rtype: ``list``
"""
if not profiles:
demisto.debug('[UCP][CommonServerPython.py] _select_ucp_profiles: no connectionProfiles in metadata.')
return []
matched = [p for p in profiles if p.get('capability') == capability]
demisto.debug('[UCP][CommonServerPython.py] _select_ucp_profiles: found {} profile(s) with capability {}.'.format(
len(matched), capability))
if matched:
return matched
for p in profiles:
if ((p.get('metadata') or {}).get('xsoar') or {}).get('interpolation_mapping'):
demisto.debug('[UCP][CommonServerPython.py] _select_ucp_profiles: no capability match; '
'falling back to first profile with an interpolation_mapping.')
return [p]
return matched
def build_ucp_params(connector_metadata, capability=None):
# type: (Optional[dict], Optional[str]) -> dict
"""Build the reshaped params dict from UCP connector metadata.
Pure, side-effect-free core of UCP param interpolation. It is
**metadata-first**: it scans ``connectionProfiles`` to find every profile in
scope for the current capability/sub_capability, then for each such profile
reads its ``param_map`` (a mapping of connector field id -> dotted destination
path) together with the platform-supplied field values, and *interpolates*
them into the nested shape integrations expect -- most importantly folding
flat fields (e.g. ``username`` / ``password``) into a single structured param
(e.g. a ``credentials`` dict, the classic XSOAR ``type 9`` shape).
**Multiple profiles may be active at once.** Each matching profile that
carries a ``param_map`` contributes its values; results are merged into one
params dict (**last-wins** when two profiles target the same destination
path, in ``connectionProfiles`` order).
This function does NOT touch ``demisto`` state, so it can be reused from
either ``interpolate_ucp_params()`` (the CommonServerPython applier) or a
Demisto class ``params()`` override.
Expected metadata shape (from ``demisto.unifiedConnectorMetadata()``)::
{
"connectionProfiles": [
{
"method_unique_id": "...",
"capability": "automation-and-remediation",
"sub_capabilities": ["salesforce-iam"],
"type": "oauth2",
"param_map": {"username": "credentials.identifier", ...},
"fields": {"username": "alice", "password": "s3cr3t", ...}
}
]
}
:type connector_metadata: ``Optional[dict]``
:param connector_metadata: The connector metadata, as returned by
``demisto.unifiedConnectorMetadata()``. When falsy, an empty dict is
returned.
:type capability: ``Optional[str]``
:param capability: The resolved capability for the current command. When
``None``, resolved automatically via ``resolve_ucp_capability()``.
:return: A new params dict containing the interpolated values from all
matching profiles, merged in ``connectionProfiles`` order
(**last-wins** on conflicting destination paths). Returns ``{}`` when
``connector_metadata`` is falsy or when nothing is interpolated.
:rtype: ``dict``
"""
result = {} # type: Dict[str, Any]
if not connector_metadata:
return result
if capability is None:
capability = resolve_ucp_capability()
profiles = connector_metadata.get('connectionProfiles') or []
selected = _select_ucp_profiles(profiles, capability)
demisto.debug('[UCP][CommonServerPython.py] build_ucp_params: capability={!r}, selected {} of {} profile(s).'.format(
capability, len(selected), len(profiles)))
for profile in selected:
method_unique_id = profile.get('method_unique_id')
# The interpolation mapping lives under the profile's module-namespaced
# metadata: profile['metadata']['xsoar']['interpolation_mapping'].
interpolation_mapping = ((profile.get('metadata') or {}).get('xsoar') or {}).get('interpolation_mapping')
demisto.debug('[UCP][CommonServerPython.py] build_ucp_params: profile id {} interpolation_mapping={!r}'.format(
method_unique_id, interpolation_mapping))
pairs = _parse_param_map(interpolation_mapping)
if not pairs:
demisto.debug('[UCP][CommonServerPython.py] build_ucp_params: no interpolation pairs for profile id {}; skipping.'.format(method_unique_id))
continue
credentials = get_ucp_credentials(method_unique_id)
# The credentials envelope is nested under a type key, e.g.
# {"type": "plain", "plain": {"username": "...", "password": "..."}}.
# Flatten: look up the field inside creds[creds["type"]], with a
# top-level fallback for already-flat envelopes.
cred_values = {} # type: Dict[str, Any]
if isinstance(credentials, dict):
cred_type = credentials.get('type')
type_data = credentials.get(cred_type) if cred_type else None
if isinstance(type_data, dict):
cred_values = type_data
else:
cred_values = credentials
# Some envelope types (e.g. passthrough) wrap the actual field
# values one level deeper under a "parameters" sub-dict:
# {"type": "passthrough", "passthrough": {"parameters": {...}}}
# while others (e.g. plain) place them directly under the type key.
# Descend into "parameters" when present.
inner_params = cred_values.get('parameters') if isinstance(cred_values, dict) else None
if isinstance(inner_params, dict):
cred_values = inner_params
cred_type = credentials.get('type') if isinstance(credentials, dict) else None
# Field names only (never values) to keep credential material out of logs.
demisto.debug('[UCP][CommonServerPython.py] build_ucp_params: cred_type={!r}, flattened cred keys={}.'.format(
cred_type, sorted(cred_values.keys())))
# For fixed-schema types (api_key, plain) resolve the value from the
# canonical envelope key, aliasing the mapping's field_id as needed
# (e.g. api_key -> "key"). Free-form types (passthrough) fall back to a
# generic field_id lookup.
canonical_keys = _UCP_CANONICAL_FIELD_KEYS.get(cred_type, {})
for field_id, destination in pairs:
lookup_key = canonical_keys.get(field_id, field_id)
field_value = cred_values.get(lookup_key)
if field_value is None:
demisto.debug('[UCP][CommonServerPython.py] build_ucp_params: missing value for field {} for profile id {}.'.format(field_id, method_unique_id))
continue
_place_by_path(result, destination, field_value)
demisto.debug('[UCP][CommonServerPython.py] build_ucp_params: interpolated {} top-level param(s).'.format(len(result)))
return result
def _deep_merge_dicts(target, source):
# type: (dict, dict) -> dict
"""Recursively merge ``source`` into ``target`` in place.
For every key in ``source``:
* If both the existing value in ``target`` and the incoming value in
``source`` are dicts, the two are merged recursively (deep-merge).
* Otherwise the incoming value overwrites the existing one (this includes
the cases where types differ, e.g. a dict overwrites a scalar or a scalar
overwrites a dict).
Keys present only in ``target`` are preserved; keys present only in
``source`` are added. ``target`` is mutated in place (its object identity,
and the identity of any nested dicts that are recursed into, is preserved)
and returned.
:type target: ``dict``
:param target: The dict to merge into (mutated in place).
:type source: ``dict``
:param source: The dict whose values take precedence on conflicts.
:return: The same ``target`` dict, mutated in place.
:rtype: ``dict``
"""
for key, source_value in source.items():
target_value = target.get(key)
if isinstance(target_value, dict) and isinstance(source_value, dict):
_deep_merge_dicts(target_value, source_value)
else:
target[key] = source_value
return target
def interpolate_ucp_params(connector_metadata=None):
# type: (Optional[dict]) -> bool
"""Interpolate UCP connector field values into ``demisto.params()``.
Thin CommonServerPython-side applier around the pure
:func:`build_ucp_params`. Fetches the connector metadata (if not provided),
builds the reshaped params, and merges them into
``demisto.callingContext['params']`` so that subsequent ``demisto.params()``
calls observe them. When no ``param_map`` is present the function is a no-op
and the legacy params are left untouched.
On success the module-level ``_UCP_AUTH_PARAMS_INJECTED`` flag is set to
``True`` so that ``should_use_ucp_auth()`` knows credentials have already
been pre-injected.
:type connector_metadata: ``Optional[dict]``
:param connector_metadata: The connector metadata. If ``None``, fetched via
``demisto.unifiedConnectorMetadata()``.
:return: ``True`` if any params were interpolated, ``False`` otherwise.
:rtype: ``bool``
"""
global _UCP_AUTH_PARAMS_INJECTED
try:
if connector_metadata is None:
connector_metadata = demisto.unifiedConnectorMetadata()
if connector_metadata is None:# we arent in ucpland
return False
except AttributeError:
demisto.debug("[UCP][CommonServerPython.py] interpolate_ucp_params: unifiedConnectorMetadata() not available.")
return False
except Exception as e:
demisto.error("[UCP][CommonServerPython.py] interpolate_ucp_params: unifiedConnectorMetadata() error: {}".format(e))
return False
# Capability-scoped, multi-profile: resolve the capability for the current
# command and interpolate every active profile that carries a param_map.
capability = None
try:
capability = resolve_ucp_capability()
except Exception as e:
demisto.debug(
"[UCP][CommonServerPython.py] interpolate_ucp_params: could not resolve capability ({}).".format(e)
)
interpolated = build_ucp_params(connector_metadata, capability=capability)
if not interpolated:
return False
params = demisto.callingContext.setdefault('params', {})
_deep_merge_dicts(params, interpolated)
_UCP_AUTH_PARAMS_INJECTED = True
demisto.debug(
"[UCP][CommonServerPython.py] interpolate_ucp_params: interpolated {} top-level param(s) "
"for capability={}.".format(len(interpolated), capability)
)
return True
# -- UCP helper: extract expiry from credentials response --
def _extract_ucp_expiry(creds):
# type: (dict) -> Optional[float]
"""Extract expiry as a Unix epoch float from a UCP credential dict.
Looks up ``expires_at`` (ISO-8601) at the top level first, then inside
``creds[creds['type']]``. Falls back to ``time.time() + 300`` on parse
errors. Returns ``None`` when no ``expires_at`` exists.
:type creds: ``dict``
:param creds: Credential dict from ``demisto.getUCPCredentials()``.
Example::
{"type": "oauth2", "oauth2": {"access_token": "...", "expires_at": "2026-04-19T18:00:00+00:00"}}
:return: Unix epoch expiry, or ``None`` if absent.
:rtype: ``Optional[float]``
"""
cred_type = creds.get('type', '')
type_data = creds.get(cred_type, {}) if cred_type else {}
expires_at_str = creds.get('expires_at') or (
type_data.get('expires_at') if isinstance(type_data, dict) else None
)
if not expires_at_str:
return None
try:
dt = datetime.fromisoformat(expires_at_str.replace('Z', '+00:00'))
return dt.timestamp()
except (ValueError, AttributeError):
demisto.error("[UCP][CommonServerPython.py] _extract_ucp_expiry: Failed to parse UCP credentials expiry time. Defaulting to 5 minutes from now.")
# Unparseable -- fall back to 5 minutes from now.
# If this happens, it means there is an error in the response of demisto.getCredentials(). Reach out to XSOAR Backend
return time.time() + 300
# -- Public UCP functions --
def is_ucp_enabled():
# type: () -> bool
"""Check whether this integration instance is running in UCP (ConnectUs) mode.
UCP mode is active when ``demisto.unifiedConnectorMetadata()`` returns a
non-empty connector descriptor. The backend caches this value, so repeated
calls are cheap. This assumes that demisto.unifiedConnectorMetadata will never raise an exception.
:return: ``True`` if UCP metadata is present, ``False`` otherwise.
:rtype: ``bool``
"""
try:
connector_info = demisto.unifiedConnectorMetadata()
if connector_info:
return True
return False
except AttributeError:
demisto.debug("demisto.unifiedConnectorMetadata() is not available in this version of the server.")
return False
except Exception as e:
demisto.error("demisto.unifiedConnectorMetadata() has returned an error: {}".format(e))
return False
def should_use_ucp_auth():
# type: () -> bool
"""Determine whether UCP credentials should be used for authentication.
Returns ``True`` when UCP is enabled **and** credentials have not already
been pre-injected into ``demisto.params()`` via ``interpolate_ucp_params()``.
:return: ``True`` if per-request UCP credential injection should be used.
:rtype: ``bool``
"""
return is_ucp_enabled() and not _UCP_AUTH_PARAMS_INJECTED
def resolve_ucp_capability(command=None):
# type: (Optional[str]) -> str
"""Resolve the UCP capability for the current (or given) command.
Uses ``_UCP_COMMAND_CAPABILITIES`` for known commands, falling back to
``_UCP_DEFAULT_CAPABILITY`` (``'automation-and-remediation'``).
Integrations can override this function if they need custom mapping logic.
:type command: ``str`` or ``None``
:param command: The command name. Defaults to ``demisto.command()``.
:return: The capability string (e.g. ``'automation-and-remediation'``).
:rtype: ``str``
"""
if command is None:
command = demisto.command()
return _UCP_COMMAND_CAPABILITIES.get(command, _UCP_DEFAULT_CAPABILITY)
# -- Profile matching building blocks --
def _get_ucp_profiles():
# type: () -> list
"""Return the list of connection profiles from UCP metadata.
:return: List of profile dicts from ``unifiedConnectorMetadata()``.
:rtype: ``list``
:raises UcpException: If UCP is not enabled or no profiles exist.
"""
connector_info = demisto.unifiedConnectorMetadata()
if not connector_info:
demisto.error('[UCP][CommonServerPython.py] _get_ucp_profiles: unifiedConnectorMetadata() returned empty.')
raise UcpException()
profiles = connector_info.get('connectionProfiles', [])
if not profiles:
demisto.error("[UCP][CommonServerPython.py] _get_ucp_profiles: No connection profiles found in connector metadata.")
raise UcpException()
return profiles
def _find_ucp_profile_by_sub_capability(profiles, sub_capability):
# type: (list, str) -> Optional[str]
"""Find the ``method_unique_id`` of the first profile matching a sub-capability.
:type profiles: ``list``
:param profiles: List of profile dicts.
:type sub_capability: ``str``
:param sub_capability: The sub-capability to match against each profile's
``sub_capabilities`` list.
:return: The ``method_unique_id`` of the matched profile, or ``None``.
:rtype: ``Optional[str]``
"""
matches = [
p for p in profiles
if sub_capability in (p.get('sub_capabilities') or [])
]
if not matches:
return None
if len(matches) > 1:
demisto.info(
"[UCP][CommonServerPython.py] _find_ucp_profile_by_sub_capability: Multiple profiles ({}) match sub_capability='{}'. Using first.".format(len(matches), sub_capability)
)
return matches[0].get('method_unique_id')
def _find_ucp_profile_by_capability(profiles, capability):
# type: (list, str) -> Optional[str]
"""Find the ``method_unique_id`` of the first profile matching a capability.
:type profiles: ``list``
:param profiles: List of profile dicts.
:type capability: ``str``
:param capability: The capability string to match (e.g.
``'automation-and-remediation'``).
:return: The ``method_unique_id`` of the matched profile, or ``None``.
:rtype: ``Optional[str]``
"""
matches = [p for p in profiles if p.get('capability') == capability]
if not matches:
return None
if len(matches) > 1:
demisto.debug(
'[UCP][CommonServerPython.py] _find_ucp_profile_by_capability: Multiple profiles ({}) match capability="{}". '
'Using first.'.format(len(matches), capability)
)
return matches[0].get('method_unique_id')
def get_ucp_method_unique_id(capability=None, sub_capability=None):
# type: (Optional[str], Optional[str]) -> str
"""Resolve which connection profile's ``method_unique_id`` to use.
Resolution priority:
1. Match by *sub_capability* (via ``_find_ucp_profile_by_sub_capability``).
2. Match by *capability* (via ``_find_ucp_profile_by_capability``).
3. Fall back to the first profile in the list.
:type capability: ``str`` or ``None``
:param capability:
The capability to match. If ``None``, resolved via
``resolve_ucp_capability()``.
:type sub_capability: ``str`` or ``None``
:param sub_capability:
Optional sub-capability to match (e.g. ``'salesforce-iam'``).
:return: The ``method_unique_id`` string from the matched profile.
:rtype: ``str``
:raises DemistoException: If no connector metadata or no profiles exist.
"""
profiles = _get_ucp_profiles()
# Priority 1: match by sub_capability
if sub_capability:
method_id = _find_ucp_profile_by_sub_capability(profiles, sub_capability)
if method_id:
return method_id
# Priority 2: match by capability
if not capability:
capability = resolve_ucp_capability()
method_id = _find_ucp_profile_by_capability(profiles, capability)
if method_id:
return method_id
# Priority 3: fallback to first profile
return profiles[0].get('method_unique_id', '')
# -- Credential fetching with in-process TTL cache --
def get_ucp_credentials(method_unique_id=None, body=None):
# type: (Optional[str], Optional[dict]) -> dict
"""Fetch UCP credentials for a connection profile, with in-process TTL caching.
Results are cached keyed by ``method_unique_id``. The TTL is derived from
the ``expiresAt`` field in the response via ``_extract_ucp_expiry``. A
refresh is triggered ``_UCP_REFRESH_THRESHOLD_SECONDS`` before expiry.
Use ``invalidate_ucp_credentials(method_unique_id)`` to force a fresh
fetch on the next call.
Credentials whose response contains no ``expires_at`` field are cached
indefinitely within the process lifetime.
:type method_unique_id: ``Optional[str]``
:param method_unique_id: The profile's ``method_unique_id``. If ``None``,
resolves automatically via ``get_ucp_method_unique_id()``.
:type body: ``Optional[dict]``
:param body: Optional request body passed through to
``demisto.getUCPCredentials``. Use this to forward credential-fetch
parameters (e.g., a specific grant type or scopes) required by the
backend for this profile. When ``None``, no body is sent and the
backend uses its profile defaults.
:return: Credential dict from the backend (may be served from cache).
Example::
{
"type": "oauth2",
"oauth2": {
"access_token": "eyJhbGci...",
"expires_at": "2026-04-19T18:00:00+00:00"
}
}
:rtype: ``dict``
"""
if not method_unique_id:
method_unique_id = get_ucp_method_unique_id()
now = time.time()
entry = _ucp_creds_cache.get(method_unique_id)
if entry is not None:
expiry = entry.get('expiry')
if expiry is None or now < (expiry - _UCP_REFRESH_THRESHOLD_SECONDS):
return entry.get('result')
# Stale -- fall through to re-fetch
creds = demisto.getUCPCredentials(method_unique_id, from_cache=False, body=body)
demisto.debug("[UCP][CommonServerPython.py] Fetched fresh credentials for method_unique_id={}".format(method_unique_id))
expiry = _extract_ucp_expiry(creds)
_ucp_creds_cache[method_unique_id] = {'result': creds, 'expiry': expiry}
return creds
def invalidate_ucp_credentials(method_unique_id):
# type: (str) -> None
"""Remove a specific entry from the UCP credentials cache.
Public helper that any integration (or ``BaseClient``) can call when an
HTTP request returns an authentication error (e.g., 401 Unauthorized) to
force ``get_ucp_credentials()`` to fetch fresh credentials on its next
invocation.
:type method_unique_id: ``str``
:param method_unique_id: The cache key to invalidate. Must match the
``method_unique_id`` used when the credentials were originally fetched.
"""
_ucp_creds_cache.pop(method_unique_id, None)
demisto.debug("[UCP][CommonServerPython.py] Invalidated cached credentials for method_unique_id={}".format(method_unique_id))
# -- Auto-run interpolation at import time --
# Placed at the end of the UCP section so every UCP helper it depends on
# (get_ucp_method_unique_id -> resolve_ucp_capability, profile matching) is
# already defined. When the integration is not running under UCP, this is a
# cheap no-op (unifiedConnectorMetadata() is empty / unavailable).
try:
interpolate_ucp_params()
except Exception as _ucp_import_err:
# Import-time safety net: never let interpolation break module import.
demisto.error(
"[UCP][CommonServerPython.py] import-time interpolate_ucp_params() swallowed error: {}".format(_ucp_import_err)
)
###########################################
# End of UCP Functions #
###########################################
###########################################
# Safe Pickle Loading #
###########################################
# Two-layer defense against insecure deserialization (RCE via malicious pickle payloads).
# Layer 1 RestrictedUnpickler: allowlist of safe (module, class) pairs + safe module prefixes.
# Layer 2 Opcode validator: blocks legacy/rare pickle opcodes that legitimate models never use.
# Common (module, class) pairs shared by all ML-model loading sites.
# Each script extends this base with its own site-specific entries via set union.
BASE_PICKLE_ALLOWED_CLASSES = {
# Numpy
("numpy.core.multiarray", "_reconstruct"),
("numpy", "ndarray"),
("numpy", "dtype"),
("numpy.core.multiarray", "scalar"),
# Pandas
("pandas.core.frame", "DataFrame"),
("pandas.core.series", "Series"),
("pandas.core.indexes.base", "Index"),
("pandas.core.indexes.range", "RangeIndex"),
# Python builtins
("builtins", "dict"),
("builtins", "list"),
("builtins", "tuple"),
("builtins", "set"),
("builtins", "frozenset"),
("builtins", "str"),
("builtins", "int"),
("builtins", "float"),
("builtins", "bool"),
("builtins", "bytes"),
("builtins", "bytearray"),
("builtins", "complex"),
("builtins", "slice"),
("builtins", "range"),
# Python internals used by pickle protocol
("collections", "OrderedDict"),
("_codecs", "encode"),
("copyreg", "_reconstructor"),
}
class UnsafePickleError(Exception):
"""Raised when a pickle payload contains unsafe opcodes or classes.
:return: None
:rtype: ``None``
"""
# Legacy/rare opcodes that legitimate ML models never use.
_BLOCKED_PICKLE_OPCODES = {
"INST", "OBJ", "NEWOBJ_EX",
"EXT1", "EXT2", "EXT4",
"PERSID", "BINPERSID",
}
def validate_pickle_opcodes(payload):
# type: (bytes) -> None
"""
Layer 2 defense-in-depth: reject pickle payloads containing
legacy/rare opcodes that legitimate ML models never use.
:type payload: ``bytes``
:param payload: Raw pickle byte stream to validate.
:raises UnsafePickleError: If a blocked opcode is found or the payload is malformed.
:return: None
:rtype: ``None``
"""
try:
for opcode, _arg, pos in pickletools.genops(payload):
if opcode.name in _BLOCKED_PICKLE_OPCODES:
raise UnsafePickleError(
"Blocked unsafe pickle opcode {!r} at byte {}".format(opcode.name, pos)
)
except (UnsafePickleError, Exception) as e:
if isinstance(e, UnsafePickleError):
raise
raise UnsafePickleError("Invalid or malformed pickle payload: {}".format(e))
def _make_restricted_unpickler(allowed_classes, safe_module_prefixes):
# type: (set, set) -> type
"""
Factory that creates a RestrictedUnpickler class with the given allowlists.
:type allowed_classes: ``set``
:param allowed_classes: Set of ``(module, name)`` tuples for exact class matches.
:type safe_module_prefixes: ``set``
:param safe_module_prefixes: Set of top-level module names (e.g. ``"sklearn"``, ``"numpy"``)
whose submodules are all considered safe data-science code.
:rtype: ``type``
:return: A subclass of ``pickle.Unpickler`` with restricted ``find_class``.
"""
class RestrictedUnpickler(_pickle.Unpickler):
"""Strict whitelist-based unpickler (Layer 1 primary defense)."""
def find_class(self, module, name):
# type: (str, str) -> type
if (module, name) in allowed_classes:
return _pickle.Unpickler.find_class(self, module, name)
top_module = module.split(".")[0]
if top_module in safe_module_prefixes:
return _pickle.Unpickler.find_class(self, module, name)
raise _pickle.UnpicklingError(
"Blocked unauthorized class: '{}.{}'".format(module, name)
)
return RestrictedUnpickler
def safe_pickle_loads(data, allowed_classes, safe_module_prefixes):
# type: (bytes, set, set) -> object
"""
Drop-in replacement for ``pickle.loads()`` / ``dill.loads()`` with two-layer security.
Layer 1 (primary): ``RestrictedUnpickler`` only allows classes in *allowed_classes*
or from modules whose top-level name is in *safe_module_prefixes*.
Layer 2 (defense-in-depth): opcode validator blocks legacy/rare pickle opcodes
that legitimate ML models never use.
:type data: ``bytes``
:param data: The raw pickle payload to deserialize.
:type allowed_classes: ``set``
:param allowed_classes: Set of ``(module, name)`` tuples for exact class matches
(e.g. ``{("builtins", "dict"), ("__main__", "MyModel")}``).
:type safe_module_prefixes: ``set``
:param safe_module_prefixes: Set of top-level module names whose submodules are
all considered safe (e.g. ``{"sklearn", "numpy", "pandas"}``).
:rtype: ``object``
:return: The deserialized Python object.
:raises UnsafePickleError: If the payload contains blocked opcodes or is malformed.
:raises pickle.UnpicklingError: If the payload references an unauthorized class.
"""
validate_pickle_opcodes(data)
unpickler_cls = _make_restricted_unpickler(allowed_classes, safe_module_prefixes)
return unpickler_cls(_io.BytesIO(data)).load()
from DemistoClassApiModule import * # type:ignore [no-redef] # noqa:E402
###########################################
# DO NOT ADD LINES AFTER THIS ONE #
###########################################
register_module_line('CommonServerPython', 'end', __line__())
register_module_line('CustomScriptIntegration', 'start', __line__())